python.observation.dataset_structure

  1# -*- coding: utf-8 -*-
  2"""
  3Created on Sun Oct  2 22:24:59 2022
  4
  5@author: philippe@loco-labs.io
  6
  7The `python.observation.dataset_structure` module contains the `DatasetStructure` class
  8(`python.observation.dataset.Dataset` methods).
  9"""
 10
 11# %% declarations
 12from copy import copy
 13
 14from observation.esconstante import ES
 15from observation.field import Field
 16from observation.util import util
 17from observation.dataset_interface import DatasetError
 18from observation.fields import Sfield
 19
 20
 21class DatasetStructure:
 22    '''this class includes Dataset methods :
 23
 24    *selecting - infos methods*
 25
 26    - `DatasetStructure.couplingmatrix`
 27    - `DatasetStructure.idxrecord`
 28    - `DatasetStructure.indexinfos`
 29    - `DatasetStructure.indicator`
 30    - `DatasetStructure.iscanonorder`
 31    - `DatasetStructure.isinrecord`
 32    - `DatasetStructure.keytoval`
 33    - `DatasetStructure.loc`
 34    - `DatasetStructure.nindex`
 35    - `DatasetStructure.record`
 36    - `DatasetStructure.recidx`
 37    - `DatasetStructure.recvar`
 38    - `DatasetStructure.tree`
 39    - `DatasetStructure.valtokey`
 40
 41    *add - update methods*
 42
 43    - `DatasetStructure.add`
 44    - `DatasetStructure.addindex`
 45    - `DatasetStructure.append`
 46    - `DatasetStructure.delindex`
 47    - `DatasetStructure.delrecord`
 48    - `DatasetStructure.orindex`
 49    - `DatasetStructure.renameindex`
 50    - `DatasetStructure.setvar`
 51    - `DatasetStructure.setname`
 52    - `DatasetStructure.updateindex`
 53
 54    *structure management - methods*
 55
 56    - `DatasetStructure.applyfilter`
 57    - `DatasetStructure.coupling`
 58    - `DatasetStructure.full`
 59    - `DatasetStructure.getduplicates`
 60    - `DatasetStructure.mix`
 61    - `DatasetStructure.merging`
 62    - `DatasetStructure.reindex`
 63    - `DatasetStructure.reorder`
 64    - `DatasetStructure.setfilter`
 65    - `DatasetStructure.sort`
 66    - `DatasetStructure.swapindex`
 67    - `DatasetStructure.setcanonorder`
 68    - `DatasetStructure.tostdcodec`
 69    '''
 70    # %% methods
 71
 72    def add(self, other, name=False, solve=True):
 73        ''' Add other's values to self's values for each index
 74
 75        *Parameters*
 76
 77        - **other** : Dataset object to add to self object
 78        - **name** : Boolean (default False) - Add values with same index name (True) or
 79        same index row (False)
 80        - **solve** : Boolean (default True) - If True, replace None other's codec value
 81        with self codec value.
 82
 83        *Returns* : self '''
 84        if self.lenindex != other.lenindex:
 85            raise DatasetError('length are not identical')
 86        if name and sorted(self.lname) != sorted(other.lname):
 87            raise DatasetError('name are not identical')
 88        for i in range(self.lenindex):
 89            if name:
 90                self.lindex[i].add(other.lindex[other.lname.index(self.lname[i])],
 91                                   solve=solve)
 92            else:
 93                self.lindex[i].add(other.lindex[i], solve=solve)
 94        return self
 95
 96    def addindex(self, index, first=False, merge=False, update=False):
 97        '''add a new index.
 98
 99        *Parameters*
100
101        - **index** : Field - index to add (can be index Ntv representation)
102        - **first** : If True insert index at the first row, else at the end
103        - **merge** : create a new index if merge is False
104        - **update** : if True, update actual values if index name is present (and merge is True)
105
106        *Returns* : none '''
107        idx = self.field.ntv(index)
108        idxname = self.lname
109        if len(idx) != len(self) and len(self) > 0:
110            raise DatasetError('sizes are different')
111        if not idx.name in idxname:
112            if first:
113                self.lindex.insert(0, idx)
114            else:
115                self.lindex.append(idx)
116        elif not merge:  # si idx.name in idxname
117            while idx.name in idxname:
118                idx.name += '(2)'
119            if first:
120                self.lindex.insert(0, idx)
121            else:
122                self.lindex.append(idx)
123        elif update:  # si merge et si idx.name in idxname
124            self.lindex[idxname.index(idx.name)].setlistvalue(idx.values)
125
126    def append(self, record, unique=False):
127        '''add a new record.
128
129        *Parameters*
130
131        - **record** :  list of new index values to add to Dataset
132        - **unique** :  boolean (default False) - Append isn't done if unique
133        is True and record present
134        
135        *Returns* : list - key record'''
136        if self.lenindex != len(record):
137            raise DatasetError('len(record) not consistent')
138        record = self.field.l_to_i(record)
139        if self.isinrecord(self.idxrecord(record), False) and unique:
140            return None
141        return [self.lindex[i].append(record[i]) for i in range(self.lenindex)]
142
143    def applyfilter(self, reverse=False, filtname=ES.filter, delfilter=True, inplace=True):
144        '''delete records with defined filter value.
145        Filter is deleted after record filtering.
146
147        *Parameters*
148
149        - **reverse** :  boolean (default False) - delete record with filter's 
150        value is reverse
151        - **filtname** : string (default ES.filter) - Name of the filter Field added
152        - **delfilter** :  boolean (default True) - If True, delete filter's Field
153        - **inplace** : boolean (default True) - if True, filter is apply to self,
154
155        *Returns* : self or new Dataset'''
156        if not filtname in self.lname:
157            return None
158        if inplace:
159            ilis = self
160        else:
161            ilis = copy(self)
162        ifilt = ilis.lname.index(filtname)       
163        if self.field.__name__ == 'Sfield':
164            ilis.sort([ifilt], reverse= not reverse, func=None)
165        else:
166            ilis.sort([ifilt], reverse=reverse, func=None)
167        lisind = ilis.lindex[ifilt].recordfromvalue(reverse)
168        if lisind:
169            minind = min(lisind)
170            for idx in ilis.lindex:
171                del idx.keys[minind:]
172        if inplace:
173            self.delindex(filtname)
174        else:
175            ilis.delindex(filtname)
176            if delfilter:
177                self.delindex(filtname)
178        ilis.reindex()
179        return ilis
180
181    def couplingmatrix(self, default=False, filename=None, att='ratecpl'):
182        '''return a matrix with coupling infos between each idx.
183        One info can be stored in a file (csv format).
184
185        *Parameters*
186
187        - **default** : comparison with default codec
188        - **filename** : string (default None) - name of the file to write the matrix
189        - **att** : string - name of the info to store in the file
190
191        *Returns* : array of array of dict'''
192        return self.analysis.getmatrix()
193
194    def coupling(self, derived=True, param='rateder', level=0.1):
195        '''Transform idx with low rate in coupled or derived indexes (codec extension).
196
197        *Parameters*
198
199        - **param** : string (default 'rateder') - coupling measurement 
200        ('rateder', 'diffdistparent', 'ratecpl', 'distance')
201        - **level** : float (default 0.1) - param threshold to apply coupling.
202        - **derived** : boolean (default : True). If True, indexes are derived, 
203        else coupled.
204
205        *Returns* : None'''
206        infos = self.indexinfos()
207        parent = {'rateder': 'distparent', 'diffdistparent': 'distparent',
208                  'ratecpl': 'minparent', 'distance': 'minparent'}
209        child = [None] * len(infos)
210        for idx in range(len(infos)):
211            iparent = infos[idx][parent[param]]
212            if iparent != -1:
213                if child[iparent] is None:
214                    child[iparent] = []
215                child[iparent].append(idx)
216        for idx in range(len(infos)):
217            self._couplingidx(idx, child, derived, param,
218                              parent[param], level, infos)
219
220    def _couplingidx(self, idx, child, derived, param, parentparam, level, infos):
221        ''' Field coupling (included childrens of the Field)'''
222        inf = infos[idx]
223        if inf['cat'] in ('coupled', 'unique') or inf[parentparam] == -1\
224                or inf[param] >= level or (derived and inf['cat'] == 'derived'):
225            return
226        if child[idx]:
227            for childidx in child[idx]:
228                self._couplingidx(childidx, child, derived,
229                                  param, parentparam, level, infos)
230        self.lindex[inf[parentparam]].coupling(self.lindex[idx], derived=derived,
231                                               duplicate=False)
232        return
233
234    def delrecord(self, record, extern=True):
235        '''remove a record.
236
237        *Parameters*
238
239        - **record** :  list - index values to remove to Dataset
240        - **extern** : if True, compare record values to external representation 
241        of self.value, else, internal
242
243        *Returns* : row deleted'''
244        self.reindex()
245        reckeys = self.valtokey(record, extern=extern)
246        if None in reckeys:
247            return None
248        row = self.tiindex.index(reckeys)
249        for idx in self:
250            del idx[row]
251        return row
252
253    def delindex(self, delname=None, savename=None):
254        '''remove an Field or a list of Field.
255
256        *Parameters*
257
258        - **delname** : string or list of string - name of index to remove
259        - **savename** : string or list of string - name of index to keep
260
261        *Returns* : none '''
262        if not delname and not savename :
263            return
264        if isinstance(delname, str):
265            delname = [delname]
266        if isinstance(savename, str):
267            savename = [savename]
268        if delname and savename:
269            delname = [name for name in delname if not name in savename]
270        if not delname:
271            delname = [name for name in self.lname if not name in savename]
272        for idxname in delname:
273            if idxname in self.lname:
274                self.lindex.pop(self.lname.index(idxname))
275
276    def _fullindex(self, ind, keysadd, indexname, varname, leng, fillvalue, fillextern):
277        if not varname:
278            varname = []
279        idx = self.lindex[ind]
280        lenadd = len(keysadd[0])
281        if len(idx) == leng:
282            return
283        inf = self.indexinfos()
284        if inf[ind]['cat'] == 'unique':
285            idx.set_keys(idx.keys + [0] * lenadd)
286        elif self.lname[ind] in indexname:
287            idx.set_keys(idx.keys + keysadd[indexname.index(self.lname[ind])])
288        elif inf[ind]['parent'] == -1 or self.lname[ind] in varname:
289            fillval = fillvalue
290            if fillextern:
291                fillval = self.field.s_to_i(fillvalue)
292            idx.set_keys(idx.keys + [len(idx.codec)] * len(keysadd[0]))
293            idx.set_codec(idx.codec + [fillval])
294        else:
295            parent = inf[ind]['parent']
296            if len(self.lindex[parent]) != leng:
297                self._fullindex(parent, keysadd, indexname, varname, leng,
298                                fillvalue, fillextern)
299            if inf[ind]['cat'] == 'coupled':
300                idx.tocoupled(self.lindex[parent], coupling=True)
301            else:
302                idx.tocoupled(self.lindex[parent], coupling=False)
303
304    def full(self, reindex=False, idxname=None, varname=None, fillvalue='-',
305             fillextern=True, inplace=True, complete=True):
306        '''tranform a list of indexes in crossed indexes (value extension).
307
308        *Parameters*
309
310        - **idxname** : list of string - name of indexes to transform
311        - **varname** : string - name of indexes to use
312        - **reindex** : boolean (default False) - if True, set default codec 
313        before transformation
314        - **fillvalue** : object value used for var extension
315        - **fillextern** : boolean(default True) - if True, fillvalue is converted 
316        to internal value
317        - **inplace** : boolean (default True) - if True, filter is apply to self,
318        - **complete** : boolean (default True) - if True, Field are ordered 
319        in canonical order
320
321        *Returns* : self or new Dataset'''
322        ilis = self if inplace else copy(self)
323        if not idxname:
324            idxname = ilis.primaryname
325        if reindex:
326            ilis.reindex()
327        keysadd = util.idxfull([ilis.nindex(name) for name in idxname])
328        if keysadd and len(keysadd) != 0:
329            lenadd = len(keysadd[0])
330            for ind in range(ilis.lenindex):
331                ilis._fullindex(ind, keysadd, idxname, varname, len(ilis) + lenadd,
332                                fillvalue, fillextern)
333        '''if not keysadd or len(keysadd) == 0:
334            return ilis
335        lenadd = len(keysadd[0])
336        for ind in range(ilis.lenindex):
337            ilis._fullindex(ind, keysadd, idxname, varname, len(ilis) + lenadd,
338                            fillvalue, fillextern)   '''     
339        if complete:
340            ilis.setcanonorder()
341        return ilis
342
343    def getduplicates(self, indexname=None, resindex=None, indexview=None):
344        '''check duplicate cod in a list of indexes. Result is add in a new 
345        index or returned.
346
347        *Parameters*
348
349        - **indexname** : list of string (default none) - name of indexes to check 
350        (if None, all Field)
351        - **resindex** : string (default None) - Add a new index named resindex 
352        with check result (False if duplicate)
353        - **indexview** : list of str (default None) - list of fields to return
354
355        *Returns* : list of int - list of rows with duplicate cod '''
356        if not indexname:
357            indexname = self.lname
358        duplicates = []
359        for name in indexname:
360            duplicates += self.nindex(name).getduplicates()
361        if resindex and isinstance(resindex, str):
362            newidx = self.field([True] * len(self), name=resindex)
363            for item in duplicates:
364                newidx[item] = False
365            self.addindex(newidx)
366        dupl = tuple(set(duplicates))
367        if not indexview:
368            return dupl
369        return [tuple(self.record(ind, indexview)) for ind in dupl]
370
371    def iscanonorder(self):
372        '''return True if primary indexes have canonical ordered keys'''
373        primary = self.primary
374        canonorder = util.canonorder(
375            [len(self.lidx[idx].codec) for idx in primary])
376        return canonorder == [self.lidx[idx].keys for idx in primary]
377
378    def isinrecord(self, record, extern=True):
379        '''Check if record is present in self.
380
381        *Parameters*
382
383        - **record** : list - value for each Field
384        - **extern** : if True, compare record values to external representation
385        of self.value, else, internal
386
387        *Returns boolean* : True if found'''
388        if extern:
389            return record in util.transpose(self.extidxext)
390        return record in util.transpose(self.extidx)
391
392    def idxrecord(self, record):
393        '''return rec array (without variable) from complete record (with variable)'''
394        return [record[self.lidxrow[i]] for i in range(len(self.lidxrow))]
395
396    def indexinfos(self, keys=None):
397        '''return a dict with infos of each index :
398            - num, name, cat, diffdistparent, child, parent, distparent, 
399            crossed, pparent, rateder (struct info)
400            - lencodec, mincodec, maxcodec, typecodec, ratecodec (base info)
401
402        *Parameters*
403
404        - **keys** : string, list or tuple (default None) - list of attributes 
405        to returned.
406        if 'all' or None, all attributes are returned.
407        if 'struct', only structural attributes are returned.
408
409        *Returns* : dict'''
410        return self.analysis.getinfos(keys)
411
412    def indicator(self, fullsize=None, size=None):
413        '''generate size indicators: ol (object lightness), ul (unicity level), 
414        gain (sizegain)
415
416        *Parameters*
417
418        - **fullsize** : int (default none) - size with full codec
419        - **size** : int (default none) - size with existing codec
420        - **indexinfos** : list (default None) - indexinfos data
421
422        *Returns* : dict'''
423        if not fullsize:
424            fullsize = len(self.to_obj(encoded=True, modecodec='full'))
425        if not size:
426            size = len(self.to_obj(encoded=True))
427        nval = len(self) * (self.lenindex + 1)
428        sval = fullsize / nval
429        ncod = sum(self.indexlen) + self.lenindex
430        if nval != ncod:
431            scod = (size - ncod * sval) / (nval - ncod)
432            olight = scod / sval
433        else:
434            olight = None
435        return {'total values': nval, 'mean size': round(sval, 3),
436                'unique values': ncod, 'mean coding size': round(scod, 3),
437                'unicity level': round(ncod / nval, 3),
438                'optimize level': round(size / fullsize, 3),
439                'object lightness': round(olight, 3),
440                'maxgain': round((nval - ncod) / nval, 3),
441                'gain': round((fullsize - size) / fullsize, 3)}
442
443    def keytoval(self, listkey, extern=True):
444        '''
445        convert a keys list (key for each index) to a values list (value for each index).
446
447        *Parameters*
448
449        - **listkey** : key for each index
450        - **extern** : boolean (default True) - if True, compare rec to val else to values
451
452        *Returns*
453
454        - **list** : value for each index'''
455        return [idx.keytoval(key, extern=extern) for idx, key in zip(self.lindex, listkey)]
456
457    def loc(self, rec, extern=True, row=False):
458        '''
459        Return record or row corresponding to a list of idx values.
460
461        *Parameters*
462
463        - **rec** : list - value for each idx
464        - **extern** : boolean (default True) - if True, compare rec to val,
465        else to values
466        - **row** : Boolean (default False) - if True, return list of row,
467        else list of records
468
469        *Returns*
470
471        - **object** : variable value or None if not found'''
472        locrow = None
473        try:
474            if len(rec) == self.lenindex:
475                locrow = list(set.intersection(*[set(self.lindex[i].loc(rec[i], extern))
476                                               for i in range(self.lenindex)]))
477            elif len(rec) == self.lenidx:
478                locrow = list(set.intersection(*[set(self.lidx[i].loc(rec[i], extern))
479                                               for i in range(self.lenidx)]))
480        except:
481            pass
482        if locrow is None:
483            return None
484        if row:
485            return locrow
486        return [self.record(locr, extern=extern) for locr in locrow]
487
488    def mix(self, other, fillvalue=None):
489        '''add other Field not included in self and add other's values'''
490        sname = set(self.lname)
491        oname = set(other.lname)
492        newself = copy(self)
493        copother = copy(other)
494        for nam in oname - sname:
495            newself.addindex({nam: [fillvalue] * len(newself)})
496        for nam in sname - oname:
497            copother.addindex({nam: [fillvalue] * len(copother)})
498        return newself.add(copother, name=True, solve=False)
499
500    def merging(self, listname=None):
501        ''' add a new Field build with Field define in listname.
502        Values of the new Field are set of values in listname Field'''
503        self.addindex(Field.merging([self.nindex(name) for name in listname]))
504
505    def nindex(self, name):
506        ''' index with name equal to attribute name'''
507        if name in self.lname:
508            return self.lindex[self.lname.index(name)]
509        return None
510
511    def orindex(self, other, first=False, merge=False, update=False):
512        ''' Add other's index to self's index (with same length)
513
514        *Parameters*
515
516        - **other** : self class - object to add
517        - **first** : Boolean (default False) - If True insert indexes
518        at the first row, else at the end
519        - **merge** : Boolean (default False) - create a new index 
520        if merge is False
521        - **update** : Boolean (default False) - if True, update actual 
522        values if index name is present (and merge is True)
523
524        *Returns* : none '''
525        if len(self) != 0 and len(self) != len(other) and len(other) != 0:
526            raise DatasetError("the sizes are not equal")
527        otherc = copy(other)
528        for idx in otherc.lindex:
529            self.addindex(idx, first=first, merge=merge, update=update)
530        return self
531
532    def record(self, row, indexname=None, extern=True):
533        '''return the record at the row
534
535        *Parameters*
536
537        - **row** : int - row of the record
538        - **extern** : boolean (default True) - if True, return val record else
539        value record
540        - **indexname** : list of str (default None) - list of fields to return
541        *Returns*
542
543        - **list** : val record or value record'''
544        if indexname is None:
545            indexname = self.lname
546        if extern:
547            record = [idx.val[row] for idx in self.lindex]
548            #record = [idx.values[row].to_obj() for idx in self.lindex]
549            #record = [idx.valrow(row) for idx in self.lindex]
550        else:
551            record = [idx.values[row] for idx in self.lindex]
552        return [record[self.lname.index(name)] for name in indexname]
553
554    def recidx(self, row, extern=True):
555        '''return the list of idx val or values at the row
556
557        *Parameters*
558
559        - **row** : int - row of the record
560        - **extern** : boolean (default True) - if True, return val rec else value rec
561
562        *Returns*
563
564        - **list** : val or value for idx'''
565        if extern:
566            return [idx.values[row].to_obj() for idx in self.lidx]
567            #return [idx.valrow(row) for idx in self.lidx]
568        return [idx.values[row] for idx in self.lidx]
569
570    def recvar(self, row, extern=True):
571        '''return the list of var val or values at the row
572
573        *Parameters*
574
575        - **row** : int - row of the record
576        - **extern** : boolean (default True) - if True, return val rec else value rec
577
578        *Returns*
579
580        - **list** : val or value for var'''
581        if extern:
582            return [idx.values[row].to_obj() for idx in self.lvar]
583            #return [idx.valrow(row) for idx in self.lvar]
584        return [idx.values[row] for idx in self.lvar]
585
586    def reindex(self):
587        '''Calculate a new default codec for each index (Return self)'''
588        for idx in self.lindex:
589            idx.reindex()
590        return self
591
592    def renameindex(self, oldname, newname):
593        '''replace an index name 'oldname' by a new one 'newname'. '''
594        for i in range(self.lenindex):
595            if self.lname[i] == oldname:
596                self.lindex[i].setname(newname)
597        for i in range(len(self.lvarname)):
598            if self.lvarname[i] == oldname:
599                self.lvarname[i] = newname
600
601    def reorder(self, recorder=None):
602        '''Reorder records in the order define by 'recorder' '''
603        if recorder is None or set(recorder) != set(range(len(self))):
604            return None
605        for idx in self.lindex:
606            idx.set_keys([idx.keys[i] for i in recorder])
607        return None
608
609    def setcanonorder(self, reindex=False):
610        '''Set the canonical index order : primary - secondary/unique - variable.
611        Set the canonical keys order : ordered keys in the first columns.
612
613        *Parameters*
614        - **reindex** : boolean (default False) - if True, set default codec after
615        transformation
616
617        *Return* : self'''
618        order = self.primaryname
619        order += self.secondaryname
620        order += self.lvarname
621        order += self.lunicname
622        self.swapindex(order)
623        self.sort(reindex=reindex)
624        self.analysis.actualize()
625        return self
626
627    def setfilter(self, filt=None, first=False, filtname=ES.filter, unique=False):
628        '''Add a filter index with boolean values
629
630        - **filt** : list of boolean - values of the filter idx to add
631        - **first** : boolean (default False) - If True insert index at the first row,
632        else at the end
633        - **filtname** : string (default ES.filter) - Name of the filter Field added
634
635        *Returns* : self'''
636        if not filt:
637            filt = [True] * len(self)
638        idx = self.field(filt, name=filtname)
639        idx.reindex()
640        if not idx.cod in ([True, False], [False, True], [True], [False]):
641            raise DatasetError('filt is not consistent')
642        if unique:
643            for name in self.lname:
644                if name[:len(ES.filter)] == ES.filter:
645                    self.delindex(ES.filter)
646        self.addindex(idx, first=first)
647        return self
648
649    def setname(self, listname=None):
650        '''Update Field name by the name in listname'''
651        for i in range(min(self.lenindex, len(listname))):
652            self.lindex[i].name = listname[i]
653        self.analysis.actualize()
654
655    def sort(self, order=None, reverse=False, func=str, reindex=True):
656        '''Sort data following the index order and apply the ascending or descending
657        sort function to values.
658
659        *Parameters*
660
661        - **order** : list (default None)- new order of index to apply. If None or [],
662        the sort function is applied to the existing order of indexes.
663        - **reverse** : boolean (default False)- ascending if True, descending if False
664        - **func**    : function (default str) - parameter key used in the sorted function
665        - **reindex** : boolean (default True) - if True, apply a new codec order (key = func)
666
667        *Returns* : self'''
668        if not order:
669            order = list(range(self.lenindex))
670        orderfull = order + list(set(range(self.lenindex)) - set(order))
671        if reindex:
672            for i in order:
673                self.lindex[i].reindex(codec=sorted(
674                    self.lindex[i].codec, key=func))
675        newidx = util.transpose(sorted(util.transpose(
676            [self.lindex[orderfull[i]].keys for i in range(self.lenindex)]),
677            reverse=reverse))
678        for i in range(self.lenindex):
679            self.lindex[orderfull[i]].set_keys(newidx[i])
680        return self
681
682    def swapindex(self, order):
683        '''
684        Change the order of the index .
685
686        *Parameters*
687
688        - **order** : list of int or list of name - new order of index to apply.
689
690        *Returns* : self '''
691        if self.lenindex != len(order):
692            raise DatasetError('length of order and Dataset different')
693        if not order or isinstance(order[0], int):
694            self.lindex = [self.lindex[ind] for ind in order]
695        elif isinstance(order[0], str):
696            self.lindex = [self.nindex(name) for name in order]
697        return self
698
699    def tostdcodec(self, inplace=False, full=True):
700        '''Transform all codec in full or default codec.
701
702        *Parameters*
703
704        - **inplace** : boolean  (default False) - if True apply transformation
705        to self, else to a new Dataset
706        - **full** : boolean (default True)- full codec if True, default if False
707
708
709        *Return Dataset* : self or new Dataset'''
710        lindex = [idx.tostdcodec(inplace=False, full=full)
711                  for idx in self.lindex]
712        if inplace:
713            self.lindex = lindex
714            return self
715        return self.__class__(lindex, self.lvarname)
716
717    def tree(self, mode='derived', width=5, lname=20, string=True):
718        '''return a string with a tree of derived Field.
719
720         *Parameters*
721
722        - **lname** : integer (default 20) - length of the names        
723        - **width** : integer (default 5) - length of the lines        
724        - **mode** : string (default 'derived') - kind of tree :
725            'derived' : derived tree
726            'distance': min distance tree
727            'diff': min dist rate tree
728        '''
729        return self.analysis.tree(width=width, lname=lname, mode=mode, string=string)
730
731    def updateindex(self, listvalue, index, extern=True):
732        '''update values of an index.
733
734        *Parameters*
735
736        - **listvalue** : list - index values to replace
737        - **index** : integer - index row to update
738        - **extern** : if True, the listvalue has external representation, else internal
739
740        *Returns* : none '''
741        self.lindex[index].setlistvalue(listvalue, extern=extern)
742
743    def valtokey(self, rec, extern=True):
744        '''convert a record list (value or val for each idx) to a key list 
745        (key for each index).
746
747        *Parameters*
748
749        - **rec** : list of value or val for each index
750        - **extern** : if True, the rec value has external representation, else internal
751
752        *Returns*
753
754        - **list of int** : record key for each index'''
755        return [idx.valtokey(val, extern=extern) for idx, val in zip(self.lindex, rec)]