tab-dataset.tab_dataset.cfield

  1# -*- coding: utf-8 -*-
  2"""
  3The `cfield` module is part of the `tab-dataset` package.
  4
  5It contains the classes `Cfield`, `Cutil` for Field entities.
  6
  7For more information, see the 
  8[user guide](https://loco-philippe.github.io/tab-dataset/docs/user_guide.html) 
  9or the [github repository](https://github.com/loco-philippe/tab-dataset).
 10"""
 11
 12from copy import copy
 13from collections import defaultdict, Counter
 14from itertools import product
 15
 16from json_ntv import Ntv
 17from json_ntv.ntv_util import NtvUtil
 18
 19from tab_analysis import AnaRelation, AnaField
 20
 21
 22@staticmethod
 23def root(leng):
 24    '''return the root Field'''
 25    return Cfield(Cutil.identity(leng), 'root')
 26
 27
 28def identity(*args, **kwargs):
 29    '''return the same value as args or kwargs'''
 30    if len(args) > 0:
 31        return args[0]
 32    if len(kwargs) > 0:
 33        return kwargs[list(kwargs.keys())[0]]
 34    return None
 35
 36
 37class Cutil:
 38    ''' common functions for Field and Dataset class'''
 39
 40    @staticmethod
 41    def identity(leng):
 42        '''return the root_field values'''
 43        return list(range(leng))
 44
 45    @staticmethod
 46    def canonorder(lenidx):
 47        '''return a list of crossed keys from a list of number of values'''
 48        listrange = [range(lidx) for lidx in lenidx]
 49        return Cutil.transpose(Cutil.list(list(product(*listrange))))
 50
 51    @staticmethod
 52    def default(values):
 53        '''return default codec and keys from a list of values'''
 54        codec = list(dict.fromkeys(values))
 55        dic = {codec[i]: i for i in range(len(codec))}
 56        keys = [dic[val] for val in values]
 57        return (codec, keys)
 58
 59    @staticmethod
 60    def dist(key1, key2, distr=False):
 61        '''return default coupling codec between two keys list and optionaly if
 62        the relationship is distributed'''
 63        if not key1 or not key2:
 64            return 0
 65        k1k2 = [tuple((v1, v2)) for v1, v2 in zip(key1, key2)]
 66        dist = len(Cutil.tocodec(k1k2))
 67        if not distr:
 68            return dist
 69        distrib = False
 70        if dist == (max(key1) + 1) * (max(key2) + 1):
 71            distrib = max(Counter(k1k2).values()) == len(key1) // dist
 72            # distrib = min(sum(map(lambda x: (x + i) % (max(a) + 1), a)) == sum(a) for i in range(1, max(a)+1)) 
 73        return [dist, distrib]
 74
 75    @staticmethod
 76    def encode_coef(lis):
 77        '''Generate a repetition coefficient for periodic list'''
 78        if len(lis) < 2:
 79            return 0
 80        coef = 1
 81        while coef != len(lis):
 82            if lis[coef-1] != lis[coef]:
 83                break
 84            coef += 1
 85        if (not len(lis) % (coef * (max(lis) + 1)) and
 86                lis == Cutil.keysfromcoef(coef, max(lis) + 1, len(lis))):
 87            return coef
 88        return 0
 89
 90    @staticmethod
 91    def funclist(value, func, *args, **kwargs):
 92        '''return the function func applied to the object value with parameters args and kwargs'''
 93        if func in (None, []):
 94            return value
 95        lis = []
 96        if not (isinstance(value, list) or value.__class__.__name__ in ['Cfield', 'Cdataset']):
 97            listval = [value]
 98        else:
 99            listval = value
100        for val in listval:
101            try:
102                lis.append(val.func(*args, **kwargs))
103            except:
104                try:
105                    lis.append(func(val, *args, **kwargs))
106                except:
107                    try:
108                        lis.append(listval.func(val, *args, **kwargs))
109                    except:
110                        try:
111                            lis.append(func(listval, val, *args, **kwargs))
112                        except:
113                            raise FieldError("unable to apply func")
114        if len(lis) == 1:
115            return lis[0]
116        return lis
117
118    @staticmethod
119    def idxfull(setidx):
120        '''return additional keys for each index in the setidx list to have crossed setidx'''
121        setcodec = [set(idx.keys) for idx in setidx]
122        lenfull = Cutil.mul([len(codec) for codec in setcodec])
123        if lenfull <= len(setidx[0]):
124            return []
125        complet = Counter(list(product(*setcodec)))
126        complet.subtract(
127            Counter(Cutil.tuple(Cutil.transpose([idx.keys for idx in setidx]))))
128        keysadd = Cutil.transpose(Cutil.list(list(complet.elements())))
129        if not keysadd:
130            return []
131        return keysadd
132
133    @staticmethod
134    def idxlink(ref, lis):
135        ''' return a dict for each different tuple (ref value, lis value)'''
136        return dict(set(zip(ref, lis)))
137        #lis = set(util.tuple(util.transpose([ref, l2])))
138        # if not len(lis) == len(set(ref)):
139        #    return {}
140        # return dict(lis)
141
142    @staticmethod
143    def is_not_equal(value, tovalue=None, **kwargs):
144        ''' return True if value and tovalue are not equal'''
145        return value.__class__.__name__ != tovalue.__class__.__name__ or \
146            value != tovalue
147
148    @staticmethod
149    def keysfromcoef(coef, period, leng=None):
150        ''' return a list of keys with periodic structure'''
151        if not leng:
152            leng = coef * period
153        return None if not (coef and period) else [(ind % (coef * period)) // coef
154                                                   for ind in range(leng)]
155
156    @staticmethod
157    def keysfromderkeys(parentkeys, derkeys):
158        '''return keys from parent keys and derkeys
159
160        *Parameters*
161
162        - **parentkeys** : list of keys from parent
163        - **derkeys** : list of derived keys
164
165        *Returns* : list of keys'''
166        return [derkeys[pkey] for pkey in parentkeys]
167
168    @staticmethod
169    def list(tuplelists):
170        '''transform a list of tuples in a list of lists'''
171        return list(map(list, tuplelists))
172
173    @staticmethod
174    def mul(values):
175        '''return the product of values in a list or tuple (math.prod)'''
176        mul = 1
177        for val in values:
178            mul *= val
179        return mul
180
181    @staticmethod
182    def reindex(oldkeys, oldcodec, newcodec):
183        '''new keys with new order of codec'''
184        dic = {newcodec[i]: i for i in range(len(newcodec))}
185        return [dic[oldcodec[key]] for key in oldkeys]
186
187    @staticmethod
188    def reorder(values, sort=None):
189        '''return a new values list following the order define by sort'''
190        if not sort:
191            return values
192        return [values[ind] for ind in sort]
193
194    @staticmethod
195    def resetidx(values):
196        '''return codec and keys from a list of values'''
197        codec = Cutil.tocodec(values)
198        return (codec, Cutil.tokeys(values, codec))
199
200    @staticmethod
201    def tocodec(values, keys=None):
202        '''extract a list of unique values'''
203        if not keys:
204            # return list(set(values))
205            return list(dict.fromkeys(values))
206        #ind, codec = zip(*sorted(set(zip(keys, values))))
207        return list(list(zip(*sorted(set(zip(keys, values)))))[1])
208
209    @staticmethod
210    def tokeys(values, codec=None):
211        ''' return a list of keys from a list of values'''
212        if not codec:
213            codec = Cutil.tocodec(values)
214        dic = {codec[i]: i for i in range(len(codec))}  # !!!!long
215        keys = [dic[val] for val in values]    # hyper long
216        return keys
217
218    @staticmethod
219    def transpose(idxlist):
220        '''exchange row/column in a list of list'''
221        # if not isinstance(idxlist, list):
222        #    raise FieldError('index not transposable')
223        # if not idxlist:
224        #    return []
225        return list(map(list, zip(*idxlist)))
226        # return [list(elmt) for elmt in zip(*idxlist)]
227        #size = min([len(ix) for ix in idxlist])
228        # return [[ix[ind] for ix in idxlist] for ind in range(size)]
229
230    @staticmethod
231    def tuple(idx):
232        '''transform a list of list in a list of tuple'''
233        return list(map(tuple, idx))
234        # return [val if not isinstance(val, list) else tuple(val) for val in idx]
235
236    @staticmethod
237    def tupled(lis):
238        '''transform a list of list in a tuple of tuple'''
239        #return tuple(val if not isinstance(val, list) else Sfield._tupled(val) for val in lis)
240        return tuple(map(Cutil.tupled, lis)) if isinstance(lis, list) else lis
241
242    @staticmethod
243    def listed(lis):
244        '''transform a tuple of tuple in a list of list'''
245        #return [val if not isinstance(val, tuple) else Cutil.listed(val) for val in lis]
246        return list(map(Cutil.listed, lis)) if isinstance(lis, tuple) else lis
247
248class Cfield:
249    # %% intro
250    '''
251    A `Cfield` is a representation of an Field list .
252
253    *Attributes (for dynamic attributes see @property methods)* :
254
255    - **name** : name of the Field
256    - **_codec** : list of values for each key
257    - **_keys** : list of code values
258
259    The methods defined in this class are :
260
261    *constructor (@classmethod)*
262
263    - `Cfield.bol`
264    - `Cfield.from_ntv`
265    - `Cfield.ntv`
266    - `Cfield.like`
267
268    *conversion static methods*
269
270    - `Cfield.ntv_to_val` (@classmethod)
271    - `Cfield.n_to_i` (@staticmethod)
272
273    *dynamic value (getters @property)*
274
275    - `Cfield.hashf`
276    - `Cfield.to_analysis`
277    - `Cfield.values`
278    - `Cfield.codec`
279    - `Cfield.infos`
280    - `Cfield.keys`
281
282    *add - update methods*
283
284    - `Cfield.add`
285    - `Cfield.append`
286    - `Cfield.setcodecvalue`
287    - `Cfield.setcodeclist`
288    - `Cfield.setname`
289    - `Cfield.set_keys`
290    - `Cfield.set_codec`
291    - `Cfield.setkeys`
292    - `Cfield.setlistvalue`
293    - `Cfield.setvalue`
294
295    *transform methods*
296
297    - `Cfield.coupling`
298    - `Cfield.extendkeys`
299    - `Cfield.full`
300    - `Cfield.reindex`
301    - `Cfield.reorder`
302    - `Cfield.sort`
303    - `Cfield.tocoupled`
304    - `Cfield.tostdcodec`
305
306    *getters methods*
307
308    - `Cfield.couplinginfos`
309    - `Cfield.derkeys`
310    - `Cfield.getduplicates`
311    - `Cfield.iscrossed`
312    - `Cfield.iscoupled`
313    - `Cfield.isderived`
314    - `Cfield.islinked`
315    - `Cfield.isvalue`
316    - `Cfield.iskeysfromderkeys`
317    - `Cfield.keytoval`
318    - `Cfield.loc`
319    - `Cfield.recordfromkeys`
320    - `Cfield.recordfromvalue`
321    - `Cfield.valtokey`
322    '''
323
324    def __init__(self, codec=None, name=None, keys=None, default=False, reindex=False):
325        '''Two modes:
326            - a single attributes : Cfield object to copy
327            - multiple attributes : set codec, name and keys attributes'''
328        if not codec and not keys:
329            self._codec = []
330            self._keys = []
331        elif isinstance(codec, Cfield):
332            self._keys = codec._keys
333            self._codec = codec._codec
334            self.name = codec.name
335            return
336        elif not default:
337            self._keys = keys if keys else Cutil.identity(len(codec))
338            self._codec = codec if codec else Cutil.identity(len(keys))
339        else:
340            self._codec, self._keys = Cutil.default(codec)
341        self.name = name if name else 'field'
342        if reindex:
343            self.reindex()
344        return
345
346    def __repr__(self):
347        '''return classname and number of value'''
348        return self.__class__.__name__ + '[' + str(len(self)) + ']'
349
350    def __str__(self):
351        '''return json string format'''
352        return str({self.name: self.values})
353
354    def __eq__(self, other):
355        ''' equal if class and values are equal'''
356        return self.__class__ .__name__ == other.__class__.__name__ and \
357            self.values == other.values
358
359    def __len__(self):
360        ''' len of values'''
361        return len(self._keys)
362
363    def __contains__(self, item):
364        ''' item of values'''
365        return item in self.values
366
367    def __getitem__(self, ind):
368        ''' return value item (value conversion)'''
369        if isinstance(ind, tuple):
370            return [copy(self.values[i]) for i in ind]
371        # return self.values[ind]
372        return copy(self.values[ind])
373
374    def __setitem__(self, ind, item):
375        ''' modify values item'''
376        if isinstance(ind, slice):
377            start, stop, step = ind.start or 0, ind.stop or len(self), ind.step or 1
378            idxt = list(iter(range(start, stop, step)))
379            if len(idxt) != len(item):
380                raise FieldError("item length not consistent")
381            self.setlistvalue(item, idxt)
382        elif ind < 0 or ind >= len(self):
383            raise FieldError("out of bounds")
384        else: 
385            self.setvalue(ind, item)
386
387    def __delitem__(self, ind):
388        '''remove a record (value and key).'''
389        self._keys.pop(ind)
390        self.reindex()
391
392    def __hash__(self):
393        '''return hash(values)'''
394        return hash(tuple(self.values))
395
396    def _hashe(self):
397        '''return hash(values)'''
398        return hash(tuple(self.values))
399
400    def __add__(self, other):
401        ''' Add other's values to self's values in a new Field'''
402        newiindex = self.__copy__()
403        newiindex.__iadd__(other)
404        return newiindex
405
406    def __iadd__(self, other):
407        ''' Add other's values to self's values'''
408        return self.add(other, solve=False)
409
410    def __copy__(self):
411        ''' Copy all the data '''
412        return self.__class__(self)
413
414    # %% property
415    @property
416    def hashf(self):
417        '''return hash(codec infos and keys)'''
418        return hash(tuple((len(self.codec), len(set(self.codec)), len(self),
419                           self.name, tuple(self._keys))))
420
421    @property
422    def to_analysis(self):
423        '''return data for AnaField module'''
424        return {'maxcodec': len(self), 'lencodec': len(self.codec), 'id': self.name,
425                'mincodec': len(set(self.codec)), 'hashf': self.hashf}
426
427    @property
428    def codec(self):
429        '''return codec  '''
430        return self._codec
431
432    @property
433    def infos(self):
434        '''return dict with lencodec, typecodec, ratecodec, mincodec, maxcodec'''
435        return AnaField(self.to_analysis).to_dict(full=True)
436
437    @property
438    def keys(self):
439        '''return keys  '''
440        return self._keys
441
442    @property
443    def values(self):
444        '''return values (see data model)'''
445        return [self._codec[key] for key in self._keys]
446
447    # %% class methods
448    @classmethod
449    def from_ntv(cls, ntv_value=None, extkeys=None, reindex=True, decode_str=False,
450                 add_type=True, lengkeys=None):
451        '''Generate an Field Object from a Ntv field object'''
452        if isinstance(ntv_value, cls):
453            return copy(ntv_value)
454        if ntv_value is None:
455            return cls()
456        ntv = Ntv.obj(ntv_value, decode_str=decode_str)
457        #ntv = NtvList(ntv_value)
458        name, typ, codec, parent, keys, coef, leng = NtvUtil.decode_ntv_tab(
459            ntv, cls.ntv_to_val)
460        if parent and not extkeys:
461            return None
462        if coef:
463            keys = Cutil.keysfromcoef(coef, leng//coef, lengkeys)
464        elif extkeys and parent:
465            keys = Cutil.keysfromderkeys(extkeys, keys)
466        elif extkeys and not parent:
467            keys = extkeys
468        keys = list(range(len(codec))) if keys is None else keys
469        name = ntv.json_name(string=True) if add_type else name
470        return cls(codec=codec, name=name, keys=keys, reindex=reindex)
471
472    @classmethod
473    def bol(cls, leng, notdef=None, name=None, default=True):
474        '''
475        Field constructor (boolean value).
476
477        *Parameters*
478
479        - **leng** : integer - length of the Field
480        - **notdef** : list (default None) - list of records without default value
481        - **default** : boolean (default True) - default value
482        - **name** : string (default None) - name of Field'''
483        values = [default] * leng
484        if notdef:
485            for item in notdef:
486                values[item] = not default
487        return cls.ntv({name: values})
488
489    @classmethod
490    def like(cls, codec, parent, name=None, reindex=False):
491        '''Generate an Field Object from specific codec and keys from another field.
492
493        *Parameters*
494
495        - **codec** : list of objects
496        - **name** : string (default None) - name of index (see data model)
497        - **parent** : Field, parent of the new Field
498        - **reindex** : boolean (default True) - if True, default codec is apply
499
500        *Returns* : Field '''
501        if isinstance(codec, Cfield):
502            return copy(codec)
503        return cls(codec=codec, name=name, keys=parent.keys, reindex=reindex)
504
505    @classmethod
506    def ntv(cls, ntv_value=None, extkeys=None, reindex=True, decode_str=False):
507        '''Generate an Field Object from a Ntv field object'''
508        return cls.from_ntv(ntv_value, extkeys=extkeys, reindex=reindex, decode_str=decode_str)
509
510    @classmethod
511    def ntv_to_val(cls, ntv):
512        '''conversion in decode_ntv_val method'''
513        return cls.n_to_i(ntv.val)
514
515    # %% static methods
516    @staticmethod
517    def n_to_i(ntv_lis):
518        ''' converting a NtvList value to an internal value'''
519        if isinstance(ntv_lis, list) and len(ntv_lis) == 0:
520            return []
521        if isinstance(ntv_lis, list) and ntv_lis[0].__class__.__name__ in ('NtvSingle', 'NtvList'):
522            return [Cfield.n_to_i(ntv.to_obj()) for ntv in ntv_lis]
523        return ntv_lis
524
525    # %% instance methods
526    def add(self, other, solve=True):
527        ''' Add other's values to self's values
528
529        *Parameters*
530
531        - **other** : Field object to add to self object
532        - **solve** : Boolean (default True) - If True, replace None other's codec value
533        with self codec value.
534
535        *Returns* : self '''
536        if solve:
537            solved = copy(other)
538            for i in range(len(solved.codec)):
539                if solved.codec[i] is None and i in range(len(self.codec)):
540                    solved._codec[i] = self.codec[i]
541            values = self.values + solved.values
542        else:
543            values = self.values + other.values
544        codec = Cutil.tocodec(values)
545        if set(codec) != set(self._codec):
546            self._codec = codec
547        self._keys = Cutil.tokeys(values, self._codec)
548        return self
549
550    def append(self, value, unique=True):
551        '''add a new value
552
553        *Parameters*
554
555        - **value** : new object value
556        - **unique** :  boolean (default True) - If False, duplication codec if value is present
557
558        *Returns* : key of value '''
559        #value = Ntv.obj(value)
560        #value = self.s_to_i(value)
561        if value in self._codec and unique:
562            key = self._codec.index(value)
563        else:
564            key = len(self._codec)
565            self._codec.append(value)
566        self._keys.append(key)
567        return key
568
569    def coupling(self, idx, derived=True, duplicate=True, reindex=False):
570        '''
571        Transform indexes in coupled or derived indexes (codec extension).
572        If derived option is True, self._codec is extended and idx codec not,
573        else, both are coupled and both codec are extended.
574
575        *Parameters*
576
577        - **idx** : single Field or list of Field to be coupled or derived.
578        - **derived** : boolean (default : True) - if True result is derived,
579        if False coupled
580        - **duplicate** : boolean (default: True) - if True, return duplicate records
581        (only for self index)
582        - **reindex** : boolean (default : False). If True self.index is reindexed
583        with default codec. But if not derived, idx indexes MUST to be reindexed.
584
585        *Returns* : tuple with duplicate records (errors) if 'duplicate', None else'''
586        duplic = tuple()
587        if not isinstance(idx, list):
588            index = [idx]
589        else:
590            index = idx
591        idxzip = self.__class__(list(zip(*([self.keys] + [ix.keys for ix in index]))),
592                                reindex=True)
593        self.tocoupled(idxzip)        
594        if not derived:
595            for ind in index:
596                ind.tocoupled(idxzip)
597                duplic += ind.getduplicates(reindex)
598        if duplicate and not duplic:
599            return self.getduplicates(reindex)
600        if duplicate and duplic:
601            return tuple(sorted(list(set(duplic + self.getduplicates(reindex))))) 
602        if reindex:
603            self.reindex()
604        return None
605
606    def couplinginfos(self, other):
607        '''return a dict with the coupling info between other (distance, ratecpl,
608        rateder, dist, disttomin, disttomax, distmin, distmax, diff, typecoupl)
609
610        *Parameters*
611
612        - **other** : other index to compare
613
614        *Returns* : dict'''
615        if min(len(self), len(other)) == 0:
616            null = Cfield()
617            return AnaRelation([AnaField(null.to_analysis), AnaField(null.to_analysis)],
618                               Cutil.dist(null.keys, null.keys, True)
619                               ).to_dict(distances=True, misc=True)
620        return AnaRelation([AnaField(self.to_analysis), AnaField(other.to_analysis)],
621                           Cutil.dist(self.keys, other.keys, True)
622                           ).to_dict(distances=True, misc=True)
623
624    def derkeys(self, parent):
625        '''return keys derived from parent keys
626
627        *Parameters*
628
629        - **parent** : Field - parent
630
631        *Returns* : list of keys'''
632        derkey = [-1] * len(parent.codec)
633        for i in range(len(self)):
634            derkey[parent.keys[i]] = self.keys[i]
635        if min(derkey) < 0:
636            raise FieldError("parent is not a derive Field")
637        return derkey
638
639    def extendkeys(self, keys):
640        '''add keys to the Field
641
642        *Parameters*
643
644        - **keys** : list of int (value lower or equal than actual keys)
645
646        *Returns* : None '''
647        if min(keys) < 0 or max(keys) > len(self._codec) - 1:
648            raise FieldError('keys not consistent with codec')
649        self._keys += keys
650
651    @staticmethod
652    def full(listidx):
653        '''tranform a list of indexes in crossed indexes (value extension).
654
655        *Parameters*
656
657        - **listidx** : list of Field to transform
658
659        *Returns* : tuple of records added '''
660        idx1 = listidx[0]
661        for idx in listidx:
662            if len(idx) != len(idx):
663                return None
664        leninit = len(idx1)
665        keysadd = Cutil.idxfull(listidx)
666        for idx, keys in zip(listidx, keysadd):
667            idx._keys += keys
668        return tuple(range(leninit, len(idx1)))
669
670    def getduplicates(self, reindex=False):
671        ''' calculate items with duplicate codec
672
673        *Parameters*
674
675        - **reindex** : boolean (default : False). If True index is reindexed with default codec
676
677        *Returns* : tuple of items with duplicate codec'''
678        count = Counter(self._codec)
679        defcodec = list(count - Counter(list(count)))
680        dkeys = defaultdict(list)
681        for key, ind in zip(self._keys, range(len(self))):
682            dkeys[key].append(ind)
683        dcodec = defaultdict(list)
684        for key, ind in zip(self._codec, range(len(self._codec))):
685            dcodec[key].append(ind)
686        duplicates = []
687        for item in defcodec:
688            for codecitem in dcodec[item]:
689                duplicates += dkeys[codecitem]
690        if reindex:
691            self.reindex()
692        return tuple(duplicates)
693
694    def iscrossed(self, other):
695        '''return True if self is crossed to other'''
696        return self.couplinginfos(other)['rateder'] == 1.0
697
698    def iscoupled(self, other):
699        '''return True if self is coupled to other'''
700        info = self.couplinginfos(other)
701        return info['diff'] == 0 and info['rateder'] == 0.0
702
703    def isderived(self, other, only=False):
704        '''return True if self is derived from other'''
705        info = self.couplinginfos(other)
706        return not (info['diff'] == 0 and only) and info['rateder'] == 0.0
707
708    def iskeysfromderkeys(self, other):
709        '''return True if self._keys is relative from other._keys'''
710        leng = len(other.codec)
711        if leng % len(self._codec) != 0:
712            return False
713        keys = [(i*len(self._codec))//leng for i in range(leng)]
714        return Cutil.keysfromderkeys(other.keys, keys) == self.keys
715
716    def islinked(self, other):
717        '''return True if self is linked to other'''
718        rate = self.couplinginfos(other)['rateder']
719        return 0.0 < rate < 1.0
720
721    def isvalue(self, value):
722        ''' return True if value is in index values
723
724        *Parameters*
725
726        - **value** : value to check'''
727        return value in self.values
728
729    def keytoval(self, key):
730        ''' return the value of a key
731
732        *Parameters*
733
734        - **key** : key to convert into values
735        - **extern** : if True, return string representation else, internal value
736
737        *Returns*
738
739        - **int** : first key finded (None else)'''
740        if key < 0 or key >= len(self._codec):
741            return None
742        return self._codec[key]
743
744    def loc(self, value):
745        '''return a list of record number with value
746
747        *Parameters*
748
749        - **value** : value to check
750
751        *Returns*
752
753        - **list of int** : list of record number finded (None else)'''
754        return self.recordfromvalue(value)
755
756    def recordfromvalue(self, value):
757        '''return a list of record number with value
758
759        *Parameters*
760
761        - **value** : value to check
762        - **extern** : if True, compare value to external representation of self.value,
763        else, internal
764
765        *Returns*
766
767        - **list of int** : list of record number finded (None else)'''
768
769        if not value in self._codec:
770            return None
771        listkeys = [cod for cod, val in zip(
772            range(len(self._codec)), self._codec) if val == value]
773        return self.recordfromkeys(listkeys)
774
775    def recordfromkeys(self, listkeys):
776        '''return a list of record number with key in listkeys
777
778        *Parameters*
779
780        - **listkeys** : list of keys to check
781
782        *Returns*
783
784        - **list of int** : list of record number finded (None else)'''
785
786        return [rec for rec, key in zip(range(len(self)), self._keys) if key in listkeys]
787
788    def reindex(self, codec=None):
789        '''apply a reordered codec. If None, a new default codec is apply.
790
791        *Parameters*
792
793        - **codec** : list (default None) - reordered codec to apply.
794
795        *Returns* : self'''
796
797        if not codec:
798            codec = Cutil.tocodec(self.values)
799        self._keys = Cutil.reindex(self._keys, self._codec, codec)
800        self._codec = codec
801        return self
802
803    def reorder(self, sort=None, inplace=True):
804        '''Change the Field order with a new order define by sort and reset the codec.
805
806        *Parameters*
807
808        - **sort** : int list (default None)- new record order to apply. If None, no change.
809        - **inplace** : boolean (default True) - if True, new order is apply to self,
810        if False a new Field is created.
811
812        *Returns*
813
814        - **Field** : self if inplace, new Field if not inplace'''
815        values = Cutil.reorder(self.values, sort)
816        codec, keys = Cutil.resetidx(values)
817        if inplace:
818            self._keys = keys
819            self._codec = codec
820            return None
821        return self.__class__(name=self.name, codec=codec, keys=keys)
822
823    def setcodecvalue(self, oldvalue, newvalue):
824        '''update all the oldvalue by newvalue
825
826        *Parameters*
827
828        - **oldvalue** : list of values to replace
829        - **newvalue** : list of new value to apply
830
831        *Returns* : int - last codec rank updated (-1 if None)'''
832
833        rank = -1
834        for i in range(len(self._codec)):
835            if self._codec[i] == oldvalue:
836                self._codec[i] = newvalue
837                rank = i
838        return rank
839
840    def setcodeclist(self, listcodec):
841        '''update codec with listcodec values
842
843        *Parameters*
844
845        - **listcodec** : list of new codec values to apply
846
847        *Returns* : int - last codec rank updated (-1 if None)'''
848        self._codec = listcodec
849
850    def set_keys(self, keys):
851        ''' _keys setters '''
852        self._keys = keys
853
854    def set_codec(self, codec):
855        ''' _codec setters '''
856        self._codec = codec
857
858    def setkeys(self, keys, inplace=True):
859        '''apply new keys (replace codec with extended codec from parent keys)
860
861        *Parameters*
862
863        - **keys** : list of keys to apply
864        - **inplace** : if True, update self data, else create a new Field
865
866        *Returns* : self or new Field'''
867        codec = Cutil.tocodec(self.values, keys)
868        if inplace:
869            self._codec = codec
870            self._keys = keys
871            return self
872        return self.__class__(codec=codec, name=self.name, keys=keys)
873
874    def setname(self, name):
875        '''update the Field name
876
877        *Parameters*
878
879        - **name** : str to set into name
880
881        *Returns* : boolean - True if update'''
882        if isinstance(name, str):
883            self.name = name
884            return True
885        return False
886
887    def setvalue(self, ind, value):
888        '''update a value at the rank ind (and update codec and keys)
889
890        *Parameters*
891
892        - **ind** : rank of the value
893        - **value** : new value
894
895        *Returns* : None'''
896        values = self.values
897        values[ind] = value
898        self._codec, self._keys = Cutil.resetidx(values)
899
900    def setlistvalue(self, listvalue, listind=None):
901        '''update the values (and update codec and keys)
902
903        *Parameters*
904
905        - **listvalue** : list - list of new values
906        - **listind** : list(default None) - list of index
907
908        *Returns* : None'''
909        values = self.values
910        listind = listind if listind else range(len(self))
911        for i, value_i in zip(listind, listvalue):
912            values[i] = value_i
913        self._codec, self._keys = Cutil.resetidx(values)
914
915    def sort(self, reverse=False, inplace=True, func=str):
916        '''Define sorted index with ordered codec.
917
918        *Parameters*
919
920        - **reverse** : boolean (defaut False) - codec is sorted with reverse order
921        - **inplace** : boolean (default True) - if True, new order is apply to self,
922        if False a new Field is created.
923        - **func**    : function (default str) - key used in the sorted function
924
925        *Return*
926
927        - **Field** : self if inplace, new Field if not inplace'''
928        if inplace:
929            self.reindex(codec=sorted(self._codec, reverse=reverse, key=func))
930            self._keys.sort()
931            return self
932        oldcodec = self._codec
933        codec = sorted(oldcodec, reverse=reverse, key=str)
934        return self.__class__(name=self.name, codec=codec,
935                              keys=sorted(Cutil.reindex(self._keys, oldcodec, codec)))
936
937    def tocoupled(self, other, coupling=True):
938        '''
939        Transform a derived index in a coupled index (keys extension) and add
940        new values to have the same length as other.
941
942        *Parameters*
943
944        - **other** : index to be coupled.
945        - **coupling** : boolean (default True) - reindex if False
946
947        *Returns* : None'''
948        dic = Cutil.idxlink(other.keys, self._keys)
949        if not dic:
950            raise FieldError("Field is not coupled or derived from other")
951        self._codec = [self._codec[dic[i]] for i in range(len(dic))]
952        self._keys = other.keys
953        if not coupling:
954            self.reindex()
955
956    def tostdcodec(self, inplace=False, full=True):
957        '''
958        Transform codec in full or in default codec.
959
960        *Parameters*
961
962        - **inplace** : boolean (default True) - if True, new order is apply to self,
963        - **full** : boolean (default True) - if True reindex with full codec
964
965        *Return*
966
967        - **Field** : self if inplace, new Field if not inplace'''
968        if full:
969            codec = self.values
970            keys = list(range(len(codec)))
971        else:
972            codec = Cutil.tocodec(self.values)
973            keys = Cutil.reindex(self._keys, self._codec, codec)
974        if inplace:
975            self._codec = codec
976            self._keys = keys
977            return self
978        return self.__class__(codec=codec, name=self.name, keys=keys)
979
980    def valtokey(self, value):
981        '''convert a value to a key
982
983        *Parameters*
984
985        - **value** : value to convert
986
987        *Returns*
988
989        - **int** : first key finded (None else)'''
990        if value in self._codec:
991            return self._codec.index(value)
992        return None
993
994
995class FieldError(Exception):
996    ''' Field Exception'''
997    # pass