python.observation.field

  1# -*- coding: utf-8 -*-
  2"""
  3Created on Thu May 26 20:30:00 2022
  4
  5@author: philippe@loco-labs.io
  6
  7The `python.observation.field` module contains the `Field` class.
  8
  9Documentation is available in other pages :
 10
 11- The Json Standard for Field is defined [here](https://github.com/loco-philippe/
 12Environmental-Sensing/tree/main/documentation/IlistJSON-Standard.pdf)
 13- The concept of 'indexed list' is described in
 14[this page](https://github.com/loco-philippe/Environmental-Sensing/wiki/Indexed-list).
 15- The non-regression tests are at [this page](https://github.com/loco-philippe/
 16Environmental-Sensing/blob/main/python/Tests/test_iindex.py)
 17- The [examples](https://github.com/loco-philippe/Environmental-Sensing/tree/main/
 18python/Examples/Field) are :
 19    - [creation](https://github.com/loco-philippe/Environmental-Sensing/blob/main/
 20    python/Examples/Field/Field_creation.ipynb)
 21    - [value](https://github.com/loco-philippe/Environmental-Sensing/blob/main/
 22    python/Examples/Field/Field_value.ipynb)
 23    - [update](https://github.com/loco-philippe/Environmental-Sensing/blob/main/
 24    python/Examples/Field/Field_update.ipynb)
 25    - [structure](https://github.com/loco-philippe/Environmental-Sensing/blob/main/
 26    python/Examples/Field/Field_structure.ipynb)
 27    - [structure-analysis](https://github.com/loco-philippe/Environmental-Sensing/
 28    blob/main/python/Examples/Field/Field_structure-analysis.ipynb)
 29
 30---
 31"""
 32# %% declarations
 33from copy import copy, deepcopy
 34from abc import ABC, abstractmethod
 35
 36from observation.esconstante import ES
 37from observation.field_interface import FieldInterface, FieldError
 38from observation.field_structure import FieldStructure
 39from observation.util import util
 40from json_ntv import Ntv, NtvList
 41
 42
 43
 44class Field(FieldStructure, FieldInterface, ABC):
 45    # %% intro
 46    '''
 47    An `Field` is a representation of an index list .
 48
 49    *Attributes (for dynamic attributes see @property methods)* :
 50
 51    - **name** : name of the Field
 52    - **codec** : list of values for each key
 53    - **keys** : list of code values
 54
 55    The methods defined in this class are :
 56
 57    *constructor (@classmethod)*
 58
 59    - `Field.bol`
 60    - `Field.ntv`
 61    - `Field.from_parent`
 62    - `Field.from_ntv`
 63    - `Field.merging`
 64
 65    *conversion abstract static methods (@abstractmethod, @staticmethod)*
 66
 67    - `Field.l_to_i`
 68    - `Field.s_to_i`
 69    - `Field.l_to_e`
 70    - `Field.s_to_e`
 71    - `Field.i_to_n`
 72    - `Field.n_to_i`
 73    - `Field.i_to_name`
 74
 75    *dynamic value (getters @property)*
 76
 77    - `Field.values`
 78    - `Field.val`
 79    - `Field.cod`
 80    - `Field.codec`
 81    - `Field.infos`
 82    - `Field.keys`
 83
 84    *add - update methods (`observation.field_structure.FieldStructure`)*
 85
 86    - `Field.append`
 87    - `Field.setcodecvalue`
 88    - `Field.setcodeclist`
 89    - `Field.setname`
 90    - `Field.setkeys`
 91    - `Field.setlistvalue`
 92    - `Field.setvalue`
 93
 94    *transform methods (`observation.field_structure.FieldStructure`)*
 95
 96    - `Field.coupling`
 97    - `Field.extendkeys`
 98    - `Field.full`
 99    - `Field.reindex`
100    - `Field.reorder`
101    - `Field.sort`
102    - `Field.tocoupled`
103    - `Field.tostdcodec`
104
105    *getters methods (`observation.field_structure.FieldStructure`)*
106
107    - `Field.couplinginfos`
108    - `Field.derkeys`
109    - `Field.getduplicates`
110    - `Field.iscrossed`
111    - `Field.iscoupled`
112    - `Field.isderived`
113    - `Field.islinked`
114    - `Field.isvalue`
115    - `Field.iskeysfromderkeys`
116    - `Field.keysfromderkeys`
117    - `Field.keytoval`
118    - `Field.loc`
119    - `Field.recordfromkeys`
120    - `Field.recordfromvalue`
121    - `Field.valtokey`
122
123    *export methods (`observation.field_interface.FieldInterface`)*
124
125    - `Field.json`
126    - `Field.to_obj`
127    - `Field.to_dict_obj`
128    - `Field.to_numpy`
129    - `Field.to_pandas`
130    - `Field.vlist`
131    - `Field.vName`
132    - `Field.vSimple`
133    '''
134
135    def __init__(self, codec=None, name=None, keys=None,
136                 lendefault=0, reindex=False, fast=False):
137        '''
138        Field constructor.
139
140        *Parameters*
141
142        - **codec** :  list (default None) - external different values of index (see data model)
143        - **keys** :  list (default None)  - key value of index (see data model)
144        - **name** : string (default None) - name of index (see data model)
145        - **lendefault** : integer (default 0) - default len if no keys is defined
146        - **reindex** : boolean (default True) - if True, default codec is apply
147        - **fast**: boolean (default False) - if True, codec is created without conversion'''
148        if isinstance(codec, Field):
149            self._keys = copy(codec._keys)
150            self._codec = deepcopy(codec._codec)
151            self.name = copy(codec.name)
152            return
153        if codec is None:
154            codec = []
155        if not isinstance(codec, list):
156            codec = [codec]
157        codec = list(codec)
158        leng = lendefault
159        if codec and len(codec) > 0 and not leng:
160            leng = len(codec)
161        if not keys is None:
162            leng = len(keys)
163        if not name:
164            name = ES.defaultindex
165        if not (keys is None or isinstance(keys, list)):
166            raise FieldError("keys not list")
167        if keys is None and leng == 0:
168            keys = []
169        elif keys is None:
170            keys = [(i*len(codec))//leng for i in range(leng)]
171        if not isinstance(codec, list):
172            raise FieldError("codec not list")
173        if codec == []:
174            keysset = util.tocodec(keys)
175            #codec = [Ntv.obj(key) for key in keysset]
176            codec = self.l_to_i(keysset, fast=True)
177        codec = self.l_to_i(codec, fast=fast)
178        self._keys = keys
179        self._codec = codec
180        self.name = name
181        if reindex:
182            self.reindex()
183
184    @classmethod
185    def bol(cls, leng, notdef=None, name=None, default=True):
186        '''
187        Field constructor (boolean value).
188
189        *Parameters*
190
191        - **leng** : integer - length of the Field
192        - **notdef** : list (default None) - list of records without default value
193        - **default** : boolean (default True) - default value
194        - **name** : string (default None) - name of Field'''
195        values = [default] * leng
196        if notdef:
197            for item in notdef:
198                values[item] = not default        
199        return cls.ntv({name: values})
200        
201    @classmethod
202    def from_parent(cls, codec, parent, name=None, reindex=False):
203        '''Generate an Field Object from specific codec and parent keys.
204
205        *Parameters*
206
207        - **codec** : list of objects
208        - **name** : string (default None) - name of index (see data model)
209        - **parent** : Field, parent of the new Field
210        - **reindex** : boolean (default True) - if True, default codec is apply
211
212        *Returns* : Field '''
213        if isinstance(codec, Field):
214            return copy(codec)
215        return cls(codec=codec, name=name, keys=parent._keys, reindex=reindex)
216
217    @classmethod 
218    def ntv(cls, ntv_value=None, extkeys=None, reindex=True, decode_str=False):
219        '''Generate an Field Object from a Ntv field object'''
220        return cls.from_ntv(ntv_value, extkeys=extkeys, reindex=reindex, decode_str=decode_str)
221    
222    @classmethod 
223    def from_ntv(cls, ntv_value=None, extkeys=None, reindex=True, decode_str=False,
224                 add_type=True, lengkeys=None):
225        '''Generate an Field Object from a Ntv field object'''
226        if isinstance(ntv_value, cls):
227            return copy(ntv_value)
228        ntv = Ntv.obj(ntv_value, decode_str=decode_str)
229        #ntv = NtvList(ntv_value)
230        if ntv_value is None:
231            return cls()
232        name, typ, codec, parent, keys, coef, leng = cls.decode_ntv(ntv)
233        if parent and not extkeys:
234            return None
235        if coef:
236            keys = FieldInterface.keysfromcoef(coef, leng//coef, lengkeys)
237        elif extkeys and parent:
238            keys = cls.keysfromderkeys(extkeys, keys)
239        elif extkeys and not parent:
240            keys = extkeys
241        keys = list(range(len(codec))) if keys is None else keys
242        name = ntv.json_name(string=True) if add_type else name
243        return cls(codec=codec, name=name, keys=keys, reindex=reindex)
244
245    """@classmethod
246    def from_dict_obj(cls, bsd, typevalue=ES.def_clsName, reindex=False):
247        '''Generate an Field Object from a dict value'''
248        var = False
249        if not isinstance(bsd, dict):
250            raise FieldError('data is not a dict')
251        name = list(bsd.keys())[0]
252        bsdv = list(bsd.values())[0]
253        if not 'value' in bsdv:
254            raise FieldError('value is not present')
255        value = bsdv['value']
256        if not isinstance(value, list):
257            value = [value]
258        if 'type' in bsdv and isinstance(bsdv['type'], str):
259            typevalue = bsdv['type']
260        if 'var' in bsdv and isinstance(bsdv['var'], bool):
261            var = bsdv['var']
262        codec = [util.castval(val['codec'], typevalue) for val in value]
263        pairs = []
264        for i, rec in enumerate(value):
265            record = rec['record']
266            if not isinstance(record, list):
267                record = [record]
268            for j in record:
269                pairs.append((j, i))
270        if not pairs:
271            return (var, cls())
272        keys = list(list(zip(*sorted(pairs)))[1])
273
274        idx = cls(name=name, codec=codec, keys=keys, typevalue=None)
275        return (var, idx)"""
276
277    @classmethod
278    def merging(cls, listidx, name=None):
279        '''Create a new Field with values are tuples of listidx Field values
280
281        *Parameters*
282
283        - **listidx** : list of Field to be merged.
284        - **name** : string (default : None) - Name of the new Field
285
286        *Returns* : new Field'''
287        if not name:
288            name = str(list({idx.name for idx in listidx}))
289        values = util.transpose([idx.values for idx in listidx])
290        return cls.ntv({name: values})
291
292
293# %% abstract
294    @staticmethod    
295    @abstractmethod
296    def l_to_i(lis):
297        ''' converting a list of external values to a list of internal values'''
298        pass
299
300    @staticmethod
301    @abstractmethod
302    def s_to_i(val):
303        '''converting an external value to an internal value'''
304        pass 
305
306    @staticmethod
307    @abstractmethod
308    def n_to_i(ntv):
309        ''' converting a NTV value to an internal value'''
310        pass 
311
312    @staticmethod
313    @abstractmethod
314    def l_to_e(lis):
315        ''' converting a list of internal values to a list of external values'''
316        pass 
317
318    @staticmethod
319    @abstractmethod
320    def s_to_e(val):
321        '''converting an internal value to an external value'''
322        pass 
323
324    @staticmethod
325    @abstractmethod
326    def i_to_n(val):
327        ''' converting an internal value to a NTV value'''
328        pass 
329
330    @staticmethod
331    @abstractmethod
332    def i_to_name(val):
333        ''' return the name of the internal value'''
334        pass 
335# %% special
336
337    def __repr__(self):
338        '''return classname and number of value'''
339        return self.__class__.__name__ + '[' + str(len(self)) + ']'
340
341    def __eq__(self, other):
342        ''' equal if class and values are equal'''
343        return self.__class__ .__name__ == other.__class__.__name__ and self.values == other.values
344
345    def __len__(self):
346        ''' len of values'''
347        return len(self._keys)
348
349    def __contains__(self, item):
350        ''' item of values'''
351        return item in self.values
352
353    def __getitem__(self, ind):
354        ''' return value item (value conversion)'''
355        if isinstance(ind, tuple):
356            return [copy(self.values[i]) for i in ind]
357        #return self.values[ind]
358        return copy(self.values[ind])
359
360    def __setitem__(self, ind, value):
361        ''' modify values item'''
362        if ind < 0 or ind >= len(self):
363            raise FieldError("out of bounds")
364        self.setvalue(ind, value, extern=True)
365
366    def __delitem__(self, ind):
367        '''remove a record (value and key).'''
368        self._keys.pop(ind)
369        self.reindex()
370
371    def __hash__(self):
372        '''return hash(values)'''
373        return hash(tuple(self.values))
374
375    def _hashe(self):
376        '''return hash(values)'''
377        return hash(tuple(self.values))
378
379    def _hashi(self):
380        '''return hash(codec) + hash(keys)'''
381        return hash(tuple(self._codec)) + hash(tuple(self._keys))
382
383    def __add__(self, other):
384        ''' Add other's values to self's values in a new Field'''
385        newiindex = self.__copy__()
386        newiindex.__iadd__(other)
387        return newiindex
388
389    def __iadd__(self, other):
390        ''' Add other's values to self's values'''
391        return self.add(other, solve=False)
392
393    def add(self, other, solve=True):
394        ''' Add other's values to self's values
395
396        *Parameters*
397
398        - **other** : Field object to add to self object
399        - **solve** : Boolean (default True) - If True, replace None other's codec value
400        with self codec value.
401
402        *Returns* : self '''
403        if solve:
404            solved = copy(other)
405            for i in range(len(solved._codec)):
406                if not util.isNotNull(solved._codec[i]) and i in range(len(self._codec)):
407                    solved._codec[i] = self._codec[i]
408            values = self.values + solved.values
409        else:
410            values = self.values + other.values
411        codec = util.tocodec(values)
412        if set(codec) != set(self._codec):
413            self._codec = codec
414        self._keys = util.tokeys(values, self._codec)
415        return self
416
417    def __copy__(self):
418        ''' Copy all the data '''
419        return self.__class__(self)
420
421# %% property
422    @property
423    def cod(self):
424        '''return codec conversion to json value '''
425        return self.l_to_e(self._codec)
426        #return [codec.to_obj() for codec in self._codec]
427        #return [codec.ntv_value for codec in self._codec]
428        #return self.to_ntv(codecval=True).ntv_value
429        #return self.to_obj(modecodec='optimize', codecval=True, encoded=False, listunic=True)
430
431    @property
432    def codec(self):
433        '''return codec  '''
434        return self._codec
435
436    @property
437    def infos(self):
438        '''return dict with lencodec, typecodec, ratecodec, mincodec, maxcodec'''
439        maxi = len(self)
440        mini = len(set(self._codec))
441        xlen = len(self._codec)
442        rate = 0.0
443        if maxi == 0:
444            typecodec = 'null'
445        elif xlen == 1:
446            typecodec = 'unique'
447        elif mini == maxi:
448            typecodec = 'complete'
449        elif xlen == maxi:
450            typecodec = 'full'
451        else:
452            rate = (maxi - xlen) / (maxi - mini)
453            if xlen == mini:
454                typecodec = 'default'
455            else:
456                typecodec = 'mixed'
457        return {'lencodec': xlen, 'mincodec': mini, 'maxcodec': maxi,
458                'typecodec': typecodec, 'ratecodec': rate}
459
460    @property
461    def keys(self):
462        '''return keys  '''
463        return self._keys
464
465    @property
466    def values(self):
467        '''return values (see data model)'''
468        return [self._codec[key] for key in self._keys]
469
470    @property
471    def val(self):
472        '''return values conversion to string '''
473        return [self.s_to_e(self._codec[key]) for key in self._keys]
474        #return [self._codec[key].to_obj() for key in self._keys]
475        #return self.to_obj(modecodec='full', codecval=True, encoded=False)