python.observation.field_structure

  1# -*- coding: utf-8 -*-
  2"""
  3Created on Sun Oct  2 22:24:59 2022
  4
  5@author: philippe@loco-labs.io
  6
  7The `python.observation.field_structure` module contains the `FieldStructure` class
  8(`python.observation.field.Field` methods).
  9"""
 10from collections import defaultdict, Counter
 11
 12from observation.util import util
 13from observation.esconstante import ES
 14from observation.field_interface import FieldError
 15
 16class FieldStructure:
 17    '''this class includes Field methods :
 18
 19    *add - update methods*
 20
 21    - `FieldStructure.append`
 22    - `FieldStructure.setcodecvalue`
 23    - `FieldStructure.setcodeclist`
 24    - `FieldStructure.setname`
 25    - `FieldStructure.setkeys`
 26    - `FieldStructure.setlistvalue`
 27    - `FieldStructure.setvalue`
 28
 29    *transform methods*
 30
 31    - `FieldStructure.coupling`
 32    - `FieldStructure.extendkeys`
 33    - `FieldStructure.full`
 34    - `FieldStructure.reindex`
 35    - `FieldStructure.reorder`
 36    - `FieldStructure.sort`
 37    - `FieldStructure.tocoupled`
 38    - `FieldStructure.tostdcodec`
 39
 40    *getters methods*
 41
 42    - `FieldStructure.couplinginfos`
 43    - `FieldStructure.derkeys`
 44    - `FieldStructure.getduplicates`
 45    - `FieldStructure.iscrossed`
 46    - `FieldStructure.iscoupled`
 47    - `FieldStructure.isderived`
 48    - `FieldStructure.islinked`
 49    - `FieldStructure.isvalue`
 50    - `FieldStructure.iskeysfromderkeys`
 51    - `FieldStructure.keysfromderkeys`
 52    - `FieldStructure.keytoval`
 53    - `FieldStructure.loc`
 54    - `FieldStructure.recordfromkeys`
 55    - `FieldStructure.recordfromvalue`
 56    - `FieldStructure.valtokey`  '''
 57
 58    def append(self, value, unique=True):
 59        '''add a new value
 60
 61        *Parameters*
 62
 63        - **value** : new object value
 64        - **unique** :  boolean (default True) - If False, duplication codec if value is present
 65
 66        *Returns* : key of value '''
 67        #value = Ntv.obj(value)
 68        value = self.s_to_i(value)
 69        if value in self._codec and unique:
 70            key = self._codec.index(value)
 71        else:
 72            key = len(self._codec)
 73            self._codec.append(value)
 74        self._keys.append(key)
 75        return key
 76
 77    def coupling(self, idx, derived=True, duplicate=True, reindex=False):
 78        '''
 79        Transform indexes in coupled or derived indexes (codec extension).
 80        If derived option is True, self._codec is extended and idx codec not,
 81        else, both are coupled and both codec are extended.
 82
 83        *Parameters*
 84
 85        - **idx** : single Field or list of Field to be coupled or derived.
 86        - **derived** : boolean (default : True) - if True result is derived,
 87        if False coupled
 88        - **duplicate** : boolean (default: True) - if True, return duplicate records 
 89        (only for self index)
 90        - **reindex** : boolean (default : False). If True self.index is reindexed 
 91        with default codec. But if not derived, idx indexes MUST to be reindexed.
 92
 93        *Returns* : tuple with duplicate records (errors) if 'duplicate', None else'''
 94        if not isinstance(idx, list):
 95            index = [idx]
 96        else:
 97            index = idx
 98        idxzip = self.__class__(list(zip(*([self._keys] + [ix._keys for ix in index]))),
 99                                reindex=True)
100        self.tocoupled(idxzip)
101        if not derived:
102            for ind in index:
103                ind.tocoupled(idxzip)
104        if duplicate:
105            return self.getduplicates(reindex)
106        if reindex:
107            self.reindex()
108        return
109
110    def couplinginfos(self, other, default=False):
111        '''return a dict with the coupling info between other (distance, ratecpl,
112        rateder, dist, disttomin, disttomax, distmin, distmax, diff, typecoupl)
113
114        *Parameters*
115
116        - **other** : other index to compare
117        - **default** : comparison with default codec
118
119        *Returns* : dict'''
120        if default:
121            return util.couplinginfos(self.values, other.values)
122        if min(len(self), len(other)) == 0:
123            return {'dist': 0, 'rateder': 0, 'disttomin': 0, 'disttomax': 0,
124                    'distmin': 0, 'distmax': 0, 'diff': 0, 'typecoupl': 'null',
125                    'distance': 0, 'ratecpl': 0}
126        xs = len(self._codec)         # xs
127        xo = len(other._codec)        # xo
128        dmin = max(xs, xo)          # dmin
129        dmax = xs * xo              # dmax
130        diff = abs(xs - xo)         # diff
131        if min(xs, xo) == 1:
132            ratecpl = 0
133            if dmax - dmin + diff != 0:
134                ratecpl = diff / (dmax - dmin + diff)  # 
135            if xs == 1:
136                typec = 'derived'
137            else:
138                typec = 'derive'
139            return {'dist': dmin, 'rateder': 0, 'disttomin': 0, 'disttomax': 0,
140                    'distmin': dmin, 'distmax': dmax, 'diff': diff,
141                    'typecoupl': typec, 'distance': diff, 'ratecpl': ratecpl}
142        xso = len(util.tocodec([tuple((v1, v2))     # xab
143                  for v1, v2 in zip(self._keys, other._keys)]))
144        dic = {'dist': xso, 'distmin': dmin, 'distmax': dmax, 'diff': diff,
145               'rateder': (xso - dmin) / (dmax - dmin),            # rateDer
146               'disttomin': xso - dmin,  
147               'disttomax': dmax - xso,        
148               'distance': xso - dmin + diff,
149               'ratecpl': (xso - dmin + diff) / (dmax - dmin + diff)}  #rateCpl
150        if dic['rateder'] == 0 and dic['diff'] == 0:
151            dic['typecoupl'] = 'coupled'
152        elif dic['rateder'] == 0 and xs < xo:
153            dic['typecoupl'] = 'derived'
154        elif dic['rateder'] == 0 and xs > xo:
155            dic['typecoupl'] = 'derive'
156        elif dic['rateder'] == 1:
157            dic['typecoupl'] = 'crossed'
158        elif xs < xo:
159            dic['typecoupl'] = 'linked'
160        else:
161            dic['typecoupl'] = 'link'
162        return dic
163
164    def derkeys(self, parent):
165        '''return keys derived from parent keys
166
167        *Parameters*
168
169        - **parent** : Field - parent
170
171        *Returns* : list of keys'''
172        derkey = [ES.nullparent] * len(parent._codec)
173        for i in range(len(self)):
174            derkey[parent._keys[i]] = self._keys[i]
175        if min(derkey) < 0:
176            raise FieldError("parent is not a derive Field")
177        return derkey
178
179    def extendkeys(self, keys):
180        '''add keys to the Field
181
182        *Parameters*
183
184        - **keys** : list of int (value lower or equal than actual keys)
185
186        *Returns* : None '''
187        if min(keys) < 0 or max(keys) > len(self._codec) - 1:
188            raise FieldError('keys not consistent with codec')
189        self._keys += keys
190
191    @staticmethod
192    def full(listidx):
193        '''tranform a list of indexes in crossed indexes (value extension).
194
195        *Parameters*
196
197        - **listidx** : list of Field to transform
198
199        *Returns* : tuple of records added '''
200        idx1 = listidx[0]
201        for idx in listidx:
202            if len(idx) != len(idx):
203                return None
204        leninit = len(idx1)
205        keysadd = util.idxfull(listidx)
206        for idx, keys in zip(listidx, keysadd):
207            idx._keys += keys
208        return tuple(range(leninit, len(idx1)))
209
210    def getduplicates(self, reindex=False):
211        ''' calculate items with duplicate codec
212
213        *Parameters*
214
215        - **reindex** : boolean (default : False). If True index is reindexed with default codec
216        
217        *Returns* : tuple of items with duplicate codec'''
218        count = Counter(self._codec)
219        defcodec = list(count - Counter(list(count)))
220        dkeys = defaultdict(list)
221        for key, ind in zip(self._keys, range(len(self))):
222            dkeys[key].append(ind)
223        dcodec = defaultdict(list)
224        for key, ind in zip(self._codec, range(len(self._codec))):
225            dcodec[key].append(ind)
226        duplicates = []
227        for item in defcodec:
228            for codecitem in dcodec[item]:
229                duplicates += dkeys[codecitem]
230        if reindex:
231            self.reindex()
232        return tuple(duplicates)
233
234    def iscrossed(self, other):
235        '''return True if self is crossed to other'''
236        return self.couplinginfos(other)['rateder'] == 1.0
237
238    def iscoupled(self, other):
239        '''return True if self is coupled to other'''
240        info = self.couplinginfos(other)
241        return info['diff'] == 0 and info['rateder'] == 0
242
243    def isderived(self, other):
244        '''return True if self is derived from other'''
245        info = self.couplinginfos(other)
246        return info['diff'] != 0 and info['rateder'] == 0.0
247
248    def iskeysfromderkeys(self, other):
249        '''return True if self._keys is relative from other._keys'''
250        leng = len(other._codec)
251        if leng % len(self._codec) != 0:
252            return False
253        keys = [(i*len(self._codec))//leng for i in range(leng)]
254        return self.__class__.keysfromderkeys(other._keys, keys) == self._keys
255
256    def islinked(self, other):
257        '''return True if self is linked to other'''
258        rate = self.couplinginfos(other)['rateder']
259        return 0.0 < rate < 1.0
260
261    def isvalue(self, value, extern=True):
262        ''' return True if value is in index values
263
264        *Parameters*
265
266        - **value** : value to check
267        - **extern** : if True, compare value to external representation of self.value,
268        else, internal'''
269        if extern:
270            return value in self.val
271        return value in self.values
272
273    def keytoval(self, key, extern=True):
274        ''' return the value of a key
275
276        *Parameters*
277
278        - **key** : key to convert into values
279        - **extern** : if True, return string representation else, internal value
280
281        *Returns*
282
283        - **int** : first key finded (None else)'''
284        if key < 0 or key >= len(self._codec):
285            return None
286        if extern:
287            return self.cod[key]
288        return self._codec[key]
289
290    @staticmethod
291    def keysfromderkeys(parentkeys, derkeys):
292        '''return keys from parent keys and derkeys
293
294        *Parameters*
295
296        - **parentkeys** : list of keys from parent
297        - **derkeys** : list of derived keys
298
299        *Returns* : list of keys'''
300        #return [derkeys[parentkeys[i]] for i in range(len(parentkeys))]
301        return [derkeys[pkey] for pkey in parentkeys]
302
303    def loc(self, value, extern=True):
304        '''return a list of record number with value
305
306        *Parameters*
307
308        - **value** : value to check
309        - **extern** : if True, compare value to external representation of self.value,
310        else, internal
311
312        *Returns*
313
314        - **list of int** : list of record number finded (None else)'''
315        return self.recordfromvalue(value, extern=extern)
316
317    def recordfromvalue(self, value, extern=True):
318        '''return a list of record number with value
319
320        *Parameters*
321
322        - **value** : value to check
323        - **extern** : if True, compare value to external representation of self.value,
324        else, internal
325
326        *Returns*
327
328        - **list of int** : list of record number finded (None else)'''
329
330        if extern:
331            value = self.s_to_i(value)
332        if not value in self._codec:
333            return None
334        listkeys = [cod for cod, val in zip(
335            range(len(self._codec)), self._codec) if val == value]
336        return self.recordfromkeys(listkeys)
337
338    def recordfromkeys(self, listkeys):
339        '''return a list of record number with key in listkeys
340
341        *Parameters*
342
343        - **listkeys** : list of keys to check
344
345        *Returns*
346
347        - **list of int** : list of record number finded (None else)'''
348
349        return [rec for rec, key in zip(range(len(self)), self._keys) if key in listkeys]
350
351    def reindex(self, codec=None):
352        '''apply a reordered codec. If None, a new default codec is apply.
353
354        *Parameters*
355
356        - **codec** : list (default None) - reordered codec to apply.
357
358        *Returns* : self'''
359
360        if not codec:
361            codec = util.tocodec(self.values)
362        self._keys = util.reindex(self._keys, self._codec, codec)
363        self._codec = codec
364        return self
365
366    def reorder(self, sort=None, inplace=True):
367        '''Change the Field order with a new order define by sort and reset the codec.
368
369        *Parameters*
370
371        - **sort** : int list (default None)- new record order to apply. If None, no change.
372        - **inplace** : boolean (default True) - if True, new order is apply to self,
373        if False a new Field is created.
374
375        *Returns*
376
377        - **Field** : self if inplace, new Field if not inplace'''
378        values = util.reorder(self.values, sort)
379        codec, keys = util.resetidx(values)
380        if inplace:
381            self._keys = keys
382            self._codec = codec
383            return None
384        return self.__class__(name=self.name, codec=codec, keys=keys)
385
386    def setcodecvalue(self, oldvalue, newvalue, extern=True,
387                      nameonly=False, valueonly=False):
388        '''update all the oldvalue by newvalue
389
390        *Parameters*
391
392        - **oldvalue** : list of values to replace
393        - **newvalue** : list of new value to apply
394        - **extern** : if True, the newvalue has external representation, else internal
395        - **nameonly** : if True, only the name of ESValue is changed
396        - **valueonly** : if True, only the value of ESValue is changed
397
398        *Returns* : int - last codec rank updated (-1 if None)'''
399        if extern:
400            newvalue = self.s_to_i(newvalue)
401            oldvalue = self.s_to_i(oldvalue)
402        rank = -1
403        for i in range(len(self._codec)):
404            if self._codec[i] == oldvalue:
405                if nameonly:
406                    self._codec[i].setName(newvalue.ntv_name)
407                elif valueonly:
408                    self._codec[i].setValue(newvalue.ntv_value)
409                else:
410                    self._codec[i] = newvalue
411                rank = i
412        return rank
413
414    def setcodeclist(self, listcodec, extern=True, nameonly=False, valueonly=False):
415        '''update codec with listcodec values
416
417        *Parameters*
418
419        - **listcodec** : list of new codec values to apply
420        - **extern** : if True, the newvalue has external representation, else internal
421        - **nameonly** : if True, only the name of ESValue is changed
422        - **valueonly** : if True, only the value of ESValue is changed
423
424        *Returns* : int - last codec rank updated (-1 if None)'''
425        if extern:
426            listcodec = self.l_to_i(listcodec)
427        self._codec = listcodec
428
429    def set_keys(self, keys):
430        ''' _keys setters '''
431        self._keys = keys
432
433    def set_codec(self, codec):
434        ''' _codec setters '''
435        self._codec = codec
436
437    def setkeys(self, keys, inplace=True):
438        '''apply new keys (replace codec with extended codec from parent keys)
439
440        *Parameters*
441
442        - **keys** : list of keys to apply
443        - **inplace** : if True, update self data, else create a new Field
444
445        *Returns* : self or new Field'''
446        codec = util.tocodec(self.values, keys)
447        if inplace:
448            self._codec = codec
449            self._keys = keys
450            return self
451        return self.__class__(codec=codec, name=self.name, keys=keys)
452
453    def setname(self, name):
454        '''update the Field name
455
456        *Parameters*
457
458        - **name** : str to set into name
459
460        *Returns* : boolean - True if update'''
461        if isinstance(name, str):
462            self.name = name
463            return True
464        return False
465
466    def setvalue(self, ind, value, extern=True, nameonly=False, valueonly=False):
467        '''update a value at the rank ind (and update codec and keys)
468
469        *Parameters*
470
471        - **ind** : rank of the value
472        - **value** : new value
473        - **extern** : if True, the value has external representation, else internal
474        - **nameonly** : if True, only the name of ESValue is changed
475        - **valueonly** : if True, only the value of ESValue is changed
476
477        *Returns* : None'''
478        if extern:
479            value = self.s_to_i(value)
480        values = self.values
481        if nameonly:
482            values[ind].setName(values.ntv_name)
483        elif valueonly:
484            values[ind].setValue(values.ntv_value)
485        else:
486            values[ind] = value
487        self._codec, self._keys = util.resetidx(values)
488
489    def setlistvalue(self, listvalue, extern=True, nameonly=False, valueonly=False):
490        '''update the values (and update codec and keys)
491
492        *Parameters*
493
494        - **listvalue** : list - list of new values
495        - **extern** : if True, the value has external representation, else internal
496        - **nameonly** : if True, only the name of ESValue is changed
497        - **valueonly** : if True, only the value of ESValue is changed
498
499        *Returns* : None'''
500        if extern:
501            listvalue = self.l_to_i(listvalue)
502        values = self.values
503        for i, value_i in enumerate(listvalue):
504            if nameonly:
505                values[i].setName(value_i.ntv_name)
506            elif valueonly:
507                values[i].setValue(value_i.ntv_value)
508            else:
509                values[i] = value_i
510        self._codec, self._keys = util.resetidx(values)
511
512    def sort(self, reverse=False, inplace=True, func=str):
513        '''Define sorted index with ordered codec.
514
515        *Parameters*
516
517        - **reverse** : boolean (defaut False) - codec is sorted with reverse order
518        - **inplace** : boolean (default True) - if True, new order is apply to self,
519        if False a new Field is created.
520        - **func**    : function (default str) - key used in the sorted function
521
522        *Return*
523
524        - **Field** : self if inplace, new Field if not inplace'''
525        if inplace:
526            self.reindex(codec=sorted(self._codec, reverse=reverse, key=func))
527            self._keys.sort()
528            return self
529        oldcodec = self._codec
530        codec = sorted(oldcodec, reverse=reverse, key=str)
531        return self.__class__(name=self.name, codec=codec,
532                              keys=sorted(util.reindex(self._keys, oldcodec, codec)))
533
534    def tocoupled(self, other, coupling=True):
535        '''
536        Transform a derived index in a coupled index (keys extension) and add
537        new values to have the same length as other.
538
539        *Parameters*
540
541        - **other** : index to be coupled.
542        - **coupling** : boolean (default True) - reindex if False
543
544        *Returns* : None'''
545        dic = util.idxlink(other._keys, self._keys)
546        if not dic:
547            raise FieldError("Field is not coupled or derived from other")
548        self._codec = [self._codec[dic[i]] for i in range(len(dic))]
549        self._keys = other._keys
550        if not coupling:
551            self.reindex()
552
553    def tostdcodec(self, inplace=False, full=True):
554        '''
555        Transform codec in full or in default codec.
556
557        *Parameters*
558
559        - **inplace** : boolean (default True) - if True, new order is apply to self,
560        - **full** : boolean (default True) - if True reindex with full codec
561
562        *Return*
563
564        - **Field** : self if inplace, new Field if not inplace'''
565        if full:
566            codec = self.values
567            keys = list(range(len(codec)))
568        else:
569            codec = util.tocodec(self.values)
570            keys = util.reindex(self._keys, self._codec, codec)
571        if inplace:
572            self._codec = codec
573            self._keys = keys
574            return self
575        return self.__class__(codec=codec, name=self.name, keys=keys)
576
577    def valtokey(self, value, extern=True):
578        '''convert a value to a key
579
580        *Parameters*
581
582        - **value** : value to convert
583        - **extern** : if True, the value has external representation, else internal
584
585        *Returns*
586
587        - **int** : first key finded (None else)'''
588        if extern:
589            value = self.s_to_i(value)
590        if value in self._codec:
591            return self._codec.index(value)
592        return None