ES.ilist

   1# -*- coding: utf-8 -*-
   2"""
   3Created on Thu May 26 20:30:00 2022
   4
   5@author: philippe@loco-labs.io
   6
   7The `ES.ilist` module contains the `Ilist` class.
   8
   9Documentation is available in other pages :
  10
  11- The Json Standard for Ilist is define 
  12[here](https://github.com/loco-philippe/Environmental-Sensing/tree/main/documentation/IlistJSON-Standard.pdf)
  13- The concept of 'indexed list' is describe in 
  14[this page](https://github.com/loco-philippe/Environmental-Sensing/wiki/Indexed-list).
  15- The non-regression test are at 
  16[this page](https://github.com/loco-philippe/Environmental-Sensing/blob/main/python/Tests/test_ilist.py)
  17- The [examples](https://github.com/loco-philippe/Environmental-Sensing/tree/main/python/Examples/Ilist)
  18 are :
  19    - [creation](https://github.com/loco-philippe/Environmental-Sensing/blob/main/python/Examples/Ilist/Ilist_creation.ipynb)
  20    - [variable](https://github.com/loco-philippe/Environmental-Sensing/blob/main/python/Examples/Ilist/Ilist_variable.ipynb)
  21    - [update](https://github.com/loco-philippe/Environmental-Sensing/blob/main/python/Examples/Ilist/Ilist_update.ipynb)
  22    - [structure](https://github.com/loco-philippe/Environmental-Sensing/blob/main/python/Examples/Ilist/Ilist_structure.ipynb)
  23    - [structure-analysis](https://github.com/loco-philippe/Environmental-Sensing/blob/main/python/Examples/Ilist/Ilist_structure-analysis.ipynb)
  24
  25---
  26"""
  27#%% declarations
  28from collections import Counter
  29from copy import copy
  30import datetime, cbor2
  31import json
  32import csv
  33import math
  34from tabulate import tabulate
  35from ESconstante import ES
  36from ESValue import ESValue
  37from iindex import Iindex
  38from util import util, IindexEncoder, CborDecoder
  39import xarray
  40import numpy as np
  41import matplotlib.pyplot as plt
  42
  43class Ilist:
  44#%% intro
  45    '''
  46    An `Ilist` is a representation of an indexed list.
  47
  48    *Attributes (for @property see methods)* :
  49
  50    - **lindex** : list of Iindex
  51    - **lvarname** : variable name (list of string)
  52
  53    The methods defined in this class are :
  54
  55    *constructor (@classmethod))*
  56
  57    - `Ilist.Idic`
  58    - `Ilist.Iext`
  59    - `Ilist.Iobj`
  60    - `Ilist.from_csv`
  61    - `Ilist.from_obj`
  62    - `Ilist.from_file`
  63
  64    *dynamic value (getters @property)*
  65
  66    - `Ilist.extidx`
  67    - `Ilist.extidxext`
  68    - `Ilist.idxname`
  69    - `Ilist.idxref`
  70    - `Ilist.idxlen`
  71    - `Ilist.iidx`
  72    - `Ilist.keys`
  73    - `Ilist.lenindex`
  74    - `Ilist.lenidx`
  75    - `Ilist.lidx`
  76    - `Ilist.lidxrow`
  77    - `Ilist.lvar`
  78    - `Ilist.lvarrow`
  79    - `Ilist.lname`
  80    - `Ilist.lunicname`
  81    - `Ilist.lunicrow`
  82    - `Ilist.setidx`
  83    - `Ilist.tiidx`
  84    - `Ilist.textidx`
  85    - `Ilist.textidxext`
  86
  87    *global value (getters @property)*
  88
  89    - `Ilist.complete`
  90    - `Ilist.consistent`
  91    - `Ilist.dimension`
  92    - `Ilist.lencomplete`
  93    - `Ilist.primary`
  94    - `Ilist.zip`
  95
  96    *selecting - infos methods*
  97
  98    - `Ilist.couplingmatrix`
  99    - `Ilist.idxrecord`
 100    - `Ilist.indexinfos`
 101    - `Ilist.indicator`
 102    - `Ilist.iscanonorder`
 103    - `Ilist.isinrecord`
 104    - `Ilist.keytoval`
 105    - `Ilist.loc`
 106    - `Ilist.nindex`
 107    - `Ilist.record`
 108    - `Ilist.recidx`
 109    - `Ilist.recvar`
 110    - `Ilist.valtokey`
 111
 112    *add - update methods*
 113
 114    - `Ilist.add`
 115    - `Ilist.addindex`
 116    - `Ilist.append`
 117    - `Ilist.delindex`
 118    - `Ilist.delrecord`
 119    - `Ilist.renameindex`
 120    - `Ilist.setvar`
 121    - `Ilist.setname`
 122    - `Ilist.updateindex`    
 123    
 124    *structure management - methods*
 125
 126    - `Ilist.applyfilter`
 127    - `Ilist.coupling`
 128    - `Ilist.full`
 129    - `Ilist.getduplicates`
 130    - `Ilist.merge`
 131    - `Ilist.reindex`
 132    - `Ilist.reorder`
 133    - `Ilist.setfilter`
 134    - `Ilist.sort`
 135    - `Ilist.swapindex`
 136    - `Ilist.setcanonorder`
 137    - `Ilist.tostdcodec`
 138    
 139    *exports methods*
 140
 141    - `Ilist.json`
 142    - `Ilist.plot`
 143    - `Ilist.to_obj`
 144    - `Ilist.to_csv`
 145    - `Ilist.to_file`
 146    - `Ilist.to_xarray`
 147    - `Ilist.to_dataFrame`
 148    - `Ilist.view`
 149    - `Ilist.vlist`
 150    - `Ilist.voxel`
 151    '''
 152    @classmethod
 153    def Idic(cls, idxdic=None, typevalue=ES.def_clsName, fullcodec=False, var=None):
 154        '''
 155        Ilist constructor (external dictionnary).
 156
 157        *Parameters*
 158
 159        - **idxdic** : {name : values}  (see data model)
 160        - **typevalue** : str (default ES.def_clsName) - default value class (None or NamedValue)
 161        - **fullcodec** : boolean (default False) - full codec if True
 162        - **var** :  int (default None) - row of the variable'''
 163        if not idxdic: return cls.Iext(idxval=None, idxname=None, typevalue=typevalue, 
 164                                          fullcodec=fullcodec, var=var)
 165        if isinstance(idxdic, Ilist): return idxdic
 166        if not isinstance(idxdic, dict): raise IlistError("idxdic not dict")
 167        return cls.Iext(list(idxdic.values()), list(idxdic.keys()), typevalue, fullcodec, var)
 168
 169    @classmethod
 170    def Iext(cls, idxval=None, idxname=None, typevalue=ES.def_clsName, 
 171             fullcodec=False, var=None):
 172        '''
 173        Ilist constructor (external index).
 174
 175        *Parameters*
 176
 177        - **idxval** : list of Iindex or list of values (see data model)
 178        - **idxname** : list of string (default None) - list of Iindex name (see data model)
 179        - **typevalue** : str (default ES.def_clsName) - default value class (None or NamedValue)
 180        - **fullcodec** : boolean (default False) - full codec if True
 181        - **var** :  int (default None) - row of the variable'''
 182        #print('debut iext')
 183        #t0 = time()
 184        if idxname is None: idxname = []
 185        if idxval  is None: idxval  = []
 186        if not isinstance(idxval, list): return None
 187        #if len(idxval) == 0: return cls()
 188        val = []
 189        for idx in idxval:
 190            if not isinstance(idx, list): val.append([idx])
 191            else: val.append(idx)
 192        return cls(listidx=val, name=idxname, var=var, typevalue=typevalue, 
 193                   context=False)
 194        '''#if not isinstance(idxval[0], list): val = [[idx] for idx in idxval]
 195        #else:                               val = idxval
 196        name = ['i' + str(i) for i in range(len(val))]
 197        for i in range(len(idxname)): 
 198            if isinstance(idxname[i], str): name[i] = idxname[i]
 199        #print('fin init iext', time()-t0)
 200        lidx = [Iindex.Iext(idx, name, typevalue, fullcodec) 
 201                    for idx, name in zip(val, name)]
 202        #print('fin lidx iext', time()-t0)
 203        return cls(lidx, var=var)'''
 204
 205    @classmethod
 206    def from_csv(cls, filename='ilist.csv', var=None, header=True, nrow=None,
 207                 optcsv = {'quoting': csv.QUOTE_NONNUMERIC}, dtype=ES.def_dtype):
 208        '''
 209        Ilist constructor (from a csv file). Each column represents index values.
 210
 211        *Parameters*
 212
 213        - **filename** : string (default 'ilist.csv'), name of the file to read
 214        - **var** : integer (default None). column row for variable data
 215        - **header** : boolean (default True). If True, the first raw is dedicated to names
 216        - **nrow** : integer (default None). Number of row. If None, all the row else nrow
 217        - **dtype** : list of string (default None) - data type for each column (default str)
 218        - **optcsv** : dict (default : quoting) - see csv.reader options'''
 219        if not optcsv: optcsv = {}
 220        if not nrow: nrow = -1
 221        with open(filename, newline='') as f:
 222            reader = csv.reader(f, **optcsv)
 223            irow = 0
 224            for row in reader:
 225                if   irow == nrow: break 
 226                elif irow == 0:
 227                    if dtype and not isinstance(dtype, list): dtype = [dtype] * len(row)
 228                    idxval  = [[] for i in range(len(row))]
 229                    idxname = None
 230                if irow == 0 and header:  idxname = row
 231                else:
 232                    if not dtype: 
 233                        for i in range(len(row)) : idxval[i].append(row[i])
 234                    else:
 235                        for i in range(len(row)) : idxval[i].append(util.cast(row[i], dtype[i]))
 236                irow += 1
 237                
 238        return cls.Iext(idxval, idxname, typevalue=None, var=var)
 239            
 240    @classmethod
 241    def from_file(cls, file, forcestring=False) :
 242        '''
 243        Generate Object from file storage.
 244
 245         *Parameters*
 246
 247        - **file** : string - file name (with path)
 248        - **forcestring** : boolean (default False) - if True, forces the UTF-8 data format, else the format is calculated
 249
 250        *Returns* : new Object'''
 251        with open(file, 'rb') as f: btype = f.read(1)
 252        if btype==bytes('[', 'UTF-8') or forcestring:
 253            with open(file, 'r', newline='') as f: bjson = f.read()
 254        else:
 255            with open(file, 'rb') as f: bjson = f.read()
 256        return cls.from_obj(bjson)
 257
 258    @classmethod
 259    def Iobj(cls, bs=None, reindex=True, context=True):
 260        '''
 261        Generate a new Object from a bytes, string or list value
 262
 263        *Parameters*
 264
 265        - **bs** : bytes, string or list data to convert
 266        - **reindex** : boolean (default True) - if True, default codec for each Iindex
 267        - **context** : boolean (default True) - if False, only codec and keys are included'''
 268        return cls.from_obj(bs, reindex=reindex, context=context)
 269
 270    @classmethod
 271    def from_obj(cls, bs=None, reindex=True, context=True):
 272        '''
 273        Generate an Ilist Object from a bytes, string or list value
 274
 275        *Parameters*
 276
 277        - **bs** : bytes, string or list data to convert
 278        - **reindex** : boolean (default True) - if True, default codec for each Iindex
 279        - **context** : boolean (default True) - if False, only codec and keys are included'''
 280        if not bs: bs = []
 281        if   isinstance(bs, bytes): lis = cbor2.loads(bs)
 282        elif isinstance(bs, str)  : lis = json.loads(bs, object_hook=CborDecoder().codecbor)
 283        elif isinstance(bs, list) : lis = bs
 284        else: raise IlistError("the type of parameter is not available")
 285        return cls(lis, reindex=reindex, context=context)
 286
 287    def __init__(self, listidx=None, name=None, length=None, var=None, reindex=True, 
 288                 typevalue=ES.def_clsName, context=True):
 289        '''
 290        Ilist constructor.
 291
 292        *Parameters*
 293
 294        - **listidx** :  list (default None) - list of compatible Iindex data 
 295        - **name** :  list (default None) - list of name for the Iindex data
 296        - **var** :  int (default None) - row of the variable
 297        - **length** :  int (default None)  - len of each Iindex
 298        - **reindex** : boolean (default True) - if True, default codec for each Iindex
 299        - **typevalue** : str (default ES.def_clsName) - default value class (None or NamedValue)
 300        - **context** : boolean (default True) - if False, only codec and keys are included'''
 301        #print('debut')
 302        #t0 = time()
 303
 304        #init self.lidx
 305        self.name = self.__class__.__name__
 306        if not isinstance(name, list): name = [name]
 307        if listidx.__class__.__name__ in ['Ilist','Obs']: 
 308            self.lindex = [copy(idx) for idx in listidx.lindex]
 309            self.lvarname = copy(listidx.lvarname)
 310            return
 311        if not listidx: 
 312            self.lindex = []
 313            self.lvarname = []
 314            return
 315        if not isinstance(listidx, list) or not isinstance(listidx[0], (list, Iindex)): 
 316            #listidx = [listidx]
 317            listidx = [[idx] for idx in listidx]
 318        typeval = [typevalue for i in range(len(listidx))]
 319        for i in range(len(name)): 
 320            typeval[i] = util.typename(name[i], typeval[i])          
 321        if len(listidx) == 1:
 322            code, idx = Iindex.from_obj(listidx[0], typevalue=typeval[0], context=context)
 323            if len(name) > 0 and name[0]: idx.name = name[0]
 324            if idx.name is None or idx.name == ES.defaultindex: idx.name = 'i0'
 325            self.lindex = [idx]
 326            self.lvarname = [idx.name]
 327            return            
 328        #print('init', time()-t0)
 329        
 330        #init
 331        if       isinstance(var, list): idxvar = var
 332        elif not isinstance(var, int) or var < 0: idxvar = []
 333        else: idxvar = [var]
 334        codind = [Iindex.from_obj(idx, typevalue=typ, context=context) 
 335                  for idx, typ in zip(listidx, typeval)]
 336        for ii, (code, idx) in zip(range(len(codind)), codind):
 337            if len(name) > ii and name[ii]: idx.name = name[ii]
 338            if idx.name is None or idx.name == ES.defaultindex: idx.name = 'i'+str(ii)
 339            if code == ES.variable and not idxvar: idxvar = [ii]
 340        self.lindex = list(range(len(codind)))    
 341        lcodind = [codind[i] for i in range(len(codind)) if i not in idxvar]
 342        lidx    = [i         for i in range(len(codind)) if i not in idxvar]
 343        #print('fin init', time()-t0)
 344        
 345        #init length
 346        if not length:  length  = -1
 347        #leng = [len(iidx) for code, iidx in codind if code < 0 and len(iidx) != 1]
 348        leng = [len(iidx) for code, iidx in codind if code < 0 and len(iidx) > 0]
 349        leng2 = [l for l in leng if l > 1]
 350
 351        if not leng: length = 0
 352        elif not leng2: length = 1 
 353        elif max(leng2) == min(leng2) and length < 0: length = max(leng2)
 354        if idxvar: length = len(codind[idxvar[0]][1])
 355        if length == 0 :
 356            self.lvarname = [codind[i][1].name for i in idxvar]
 357            self.lindex = [iidx for code, iidx in codind]
 358            return
 359        flat = True
 360        if leng2: flat = length == max(leng2) == min(leng2)
 361        if not flat:
 362            keysset = util.canonorder([len(iidx) for code, iidx in lcodind 
 363                         if code < 0 and len(iidx) != 1])
 364            if length >= 0 and length != len(keysset[0]): 
 365                raise IlistError('length of Iindex and Ilist inconsistent')
 366            else: length = len(keysset[0])
 367        #print('fin leng', time()-t0)
 368        
 369        #init primary               
 370        primary = [(rang, iidx) for rang, (code, iidx) in zip(range(len(lcodind)), lcodind)
 371                   if code < 0 and len(iidx) != 1]
 372        for ip, (rang, iidx) in zip(range(len(primary)), primary):
 373            if not flat: iidx.keys = keysset[ip]
 374            self.lindex[lidx[rang]] = iidx
 375        #print('fin primary', time()-t0)
 376        
 377        #init secondary               
 378        for ii, (code, iidx) in zip(range(len(lcodind)), lcodind):
 379            if iidx.name is None or iidx.name == ES.defaultindex: iidx.name = 'i'+str(ii)
 380            if len(iidx.codec) == 1: 
 381                iidx.keys = [0] * length
 382                self.lindex[lidx[ii]] = iidx
 383            elif code >=0 and isinstance(self.lindex[lidx[ii]], int): 
 384                self._addiidx(lidx[ii], code, iidx, codind, length)
 385            elif code < 0 and isinstance(self.lindex[lidx[ii]], int): 
 386                raise IlistError('Ilist not canonical')
 387        #print('fin secondary', time()-t0)
 388        
 389        #init variable
 390        for i in idxvar: self.lindex[i] = codind[i][1]
 391        self.lvarname = [codind[i][1].name for i in idxvar]
 392        if reindex: self.reindex()
 393        return None
 394                
 395    def _addiidx(self, rang, code, iidx, codind, length):
 396        '''creation derived or coupled Iindex and update lindex'''
 397        if isinstance(self.lindex[code], int): 
 398            self._addiidx(code, codind[code][0], codind[code][1], codind, length)
 399        if iidx.keys == list(range(len(iidx.codec))):
 400            #if len(iidx.codec) == length: #coupled format
 401            if len(iidx.codec) == len(self.lindex[code].codec): #coupled format
 402                self.lindex[rang] = Iindex(iidx.codec, iidx.name, self.lindex[code].keys)
 403            else:  #derived format without keys
 404                parent = copy(self.lindex[code])
 405                parent.reindex()
 406                leng = len(parent.codec)    
 407                keys = [(i*len(iidx.codec))//leng for i in range(leng)]
 408                self.lindex[rang] = Iindex(iidx.codec, iidx.name, 
 409                                      Iindex.keysfromderkeys(parent.keys, keys))
 410        else:
 411            self.lindex[rang] = Iindex(iidx.codec, iidx.name, 
 412                                      Iindex.keysfromderkeys(self.lindex[code].keys, 
 413                                                            codind[rang][1].keys))
 414
 415#%% special
 416    def __str__(self):
 417        '''return string format for var and lidx'''
 418        if self.lvar: stri = str(self.lvar[0]) + '\n'
 419        else: stri = ''
 420        for idx in self.lidx: stri += str(idx)
 421        return stri
 422
 423    def __repr__(self):
 424        '''return classname, number of value and number of indexes'''
 425        return self.__class__.__name__ + '[' + str(len(self)) + ', ' + str(self.lenindex) + ']'
 426
 427    def __len__(self):
 428        ''' len of values'''
 429        if not self.lindex: return 0
 430        return len(self.lindex[0])
 431
 432    def __contains__(self, item):
 433        ''' list of lindex values'''
 434        return item in self.lindex
 435
 436    def __getitem__(self, ind):
 437        ''' return value record (value conversion)'''
 438        res = [idx[ind] for idx in self.lindex]
 439        if len(res) == 1: return res[0]
 440        return res
 441
 442    def __setitem__(self, ind, item):
 443        ''' modify the Iindex values for each Iindex at the row ind'''
 444        if not isinstance(item, list): item = [item]
 445        for val, idx in zip(item, self.lindex): idx[ind] = val
 446            
 447    def __delitem__(self, ind):
 448        ''' remove all Iindex item at the row ind'''
 449        for idx in self.lindex: del(idx[ind])
 450        
 451    def __hash__(self): 
 452        '''return sum of all hash(Iindex)'''
 453        return sum([hash(idx) for idx in self.lindex])
 454
 455    def __eq__(self, other):
 456        ''' equal if all Iindex and var are equal'''
 457        return self.__class__.__name__ == other.__class__.__name__ \
 458            and self.lvarname == other.lvarname \
 459            and set([idx in self.lindex for idx in other.lindex]) in ({True}, set())
 460    
 461    def __add__(self, other):
 462        ''' Add other's values to self's values in a new Ilist'''
 463        newil = copy(self)
 464        newil.__iadd__(other)
 465        return newil
 466
 467    def __iadd__(self, other):
 468        ''' Add other's values to self's values'''
 469        return self.add(other, name=True, solve=False)     
 470    
 471    def __or__(self, other):
 472        ''' Add other's index to self's index in a new Ilist'''
 473        newil = copy(self)
 474        newil.__ior__(other)
 475        return newil
 476
 477    def __ior__(self, other):
 478        ''' Add other's index to self's index'''
 479        if len(self) != 0 and len(self) != len(other) and len(other) != 0:
 480            raise IlistError("the sizes are not equal")
 481        otherc = copy(other)
 482        for idx in otherc.lindex: self.addindex(idx)
 483        if not self.lvarname: self.lvarname = other.lvarname
 484        return self
 485
 486    def __copy__(self):
 487        ''' Copy all the data '''
 488        #return Ilist([copy(idx) for idx in self.lindex], var=self.lvarrow)
 489        return Ilist(self)
 490
 491#%% property
 492    @property
 493    def complete(self):
 494        '''return a boolean (True if Ilist is complete and consistent)'''
 495        return self.lencomplete == len(self) and self.consistent
 496
 497    @property
 498    def consistent(self):
 499        ''' True if all the record are different'''
 500        return max(Counter(zip(*self.iidx)).values()) == 1
 501
 502    @property
 503    def dimension(self):
 504        ''' integer : number of primary Iindex'''
 505        return len(self.primary)
 506
 507    @property
 508    def extidx(self):
 509        '''idx values (see data model)'''
 510        return [idx.values for idx in self.lidx]
 511
 512    @property
 513    def extidxext(self):
 514        '''idx val (see data model)'''
 515        return [idx.val for idx in self.lidx]
 516
 517    @property    
 518    def idxname(self):
 519        ''' list of idx name'''
 520        return [idx.name for idx in self.lidx]
 521
 522    @property
 523    def idxref(self):
 524        ''' list of idx parent row (idx row if linked)'''
 525        return [inf['parent'] if inf['typecoupl'] != 'linked' else 
 526                inf['num'] for inf in self.indexinfos()]  
 527    
 528    @property    
 529    def idxlen(self):
 530        ''' list of idx codec length'''
 531        return [len(idx.codec) for idx in self.lidx]
 532
 533    @property    
 534    def indexlen(self):
 535        ''' list of index codec length'''
 536        return [len(idx.codec) for idx in self.lindex]
 537
 538    @property 
 539    def iidx(self):
 540        ''' list of keys for each idx'''
 541        return [idx.keys for idx in self.lidx]
 542    
 543    @property 
 544    def keys(self):
 545        ''' list of keys for each index'''
 546        return [idx.keys for idx in self.lindex]
 547
 548    @property
 549    def lencomplete(self):
 550        '''number of values if complete (prod(idxlen primary))'''
 551        return util.mul([self.idxlen[i] for i in self.primary])
 552    
 553    @property    
 554    def lenindex(self):
 555        ''' number of indexes'''
 556        return len(self.lindex)
 557
 558    @property    
 559    def lenidx(self):
 560        ''' number of idx'''
 561        return len(self.lidx)
 562
 563    @property
 564    def lidx(self):
 565        '''list of idx'''
 566        return [self.lindex[i] for i in self.lidxrow]
 567
 568    @property
 569    def lvar(self):
 570        '''list of var'''
 571        return [self.lindex[i] for i in self.lvarrow]
 572
 573    @property
 574    def lunicrow(self):
 575        '''list of unic idx row'''
 576        return [self.lname.index(name) for name in self.lunicname]
 577
 578    @property
 579    def lvarrow(self):
 580        '''list of var row'''
 581        return [self.lname.index(name) for name in self.lvarname]
 582
 583    @property
 584    def lidxrow(self):
 585        '''list of idx row'''
 586        return [i for i in range(self.lenindex) if i not in self.lvarrow]
 587        #return [self.lname.index(name) for name not in self.idxvar]
 588    
 589    @property    
 590    def lunicname(self):
 591        ''' list of unique index name'''
 592        return [idx.name for idx in self.lindex if len(idx.codec) == 1]
 593
 594    @property    
 595    def lname(self):
 596        ''' list of index name'''
 597        return [idx.name for idx in self.lindex]
 598
 599    @property    
 600    def primary(self):
 601        ''' list of primary idx'''
 602        idxinfos = self.indexinfos()
 603        return [idxinfos.index(idx) for idx in idxinfos if idx['cat'] == 'primary']
 604
 605    @property
 606    def setidx(self): 
 607        '''list of codec for each idx'''
 608        return [idx.codec for idx in self.lidx]
 609    
 610    @property 
 611    def tiidx(self):
 612        ''' list of keys for each record'''
 613        return util.list(list(zip(*self.iidx)))
 614
 615    @property
 616    def textidx(self):
 617        '''list of values for each rec'''
 618        return util.transpose(self.extidx)
 619
 620    @property
 621    def textidxext(self):
 622        '''list of val for each rec'''
 623        return util.transpose(self.extidxext)
 624
 625    @property
 626    def typevalue(self):
 627        '''return typevalue calculated from Iindex name'''
 628        return [util.typename(name)for name in self.lname]  
 629
 630
 631    @property
 632    def zip(self):
 633        '''return a zip format for textidx : tuple(tuple(rec))'''
 634        textidx = self.textidx
 635        return tuple(tuple(idx) for idx in textidx)
 636    
 637    #%% methods
 638    def add(self, other, name=False, solve=True):
 639        ''' Add other's values to self's values for each index 
 640
 641        *Parameters*
 642
 643        - **other** : Ilist object to add to self object
 644        - **name** : Boolean (default False) - Add values with same index name (True) or 
 645        same index row (False)
 646
 647        *Returns* : self '''      
 648        if self.lenindex != other.lenindex: raise IlistError('length are not identical')
 649        if name and sorted(self.lname) != sorted(other.lname): raise IlistError('name are not identical')
 650        for i in range(self.lenindex): 
 651            if name: self.lindex[i].add(other.lindex[other.lname.index(self.lname[i])], 
 652                                        solve=solve)
 653            else: self.lindex[i].add(other.lindex[i], solve=solve)
 654        if not self.lvarname: self.lvarname = other.lvarname
 655        return self        
 656
 657    def addindex(self, index, first=False, merge=False, update=False):
 658        '''add a new index.
 659
 660        *Parameters*
 661
 662        - **index** : Iindex - index to add (can be index representation)
 663        - **first** : If True insert index at the first row, else at the end
 664        - **merge** : create a new index if merge is False
 665        - **update** : if True, update actual values if index name is present (and merge is True)
 666
 667        *Returns* : none '''      
 668        idx = Iindex.Iobj(index)
 669        idxname = self.lname
 670        if len(idx) != len(self) and len(self) > 0: 
 671            raise IlistError('sizes are different')
 672        if not idx.name in idxname: 
 673            if first: self.lindex.insert(0, idx)
 674            else: self.lindex.append(idx)
 675        elif idx.name in idxname and not merge:
 676            while idx.name in idxname: idx.name += '(2)'
 677            if first: self.lindex.insert(0, idx)
 678            else: self.lindex.append(idx)
 679        elif update:
 680            self.lindex[idxname.index(idx.name)].setlistvalue(idx.values)
 681            
 682    def append(self, record, unique=False, typevalue=ES.def_clsName):
 683        '''add a new record.
 684
 685        *Parameters*
 686
 687        - **record** :  list of new index values to add to Ilist
 688        - **unique** :  boolean (default False) - Append isn't done if unique is True and record present
 689        - **typevalue** : list of string (default ES.def_clsName) - typevalue to convert record or
 690         string if typevalue is not define in indexes
 691
 692        *Returns* : list - key record'''
 693        if self.lenindex != len(record): raise('len(record) not consistent')
 694        if not isinstance(typevalue, list): typevalue = [typevalue] * len(record)
 695        typevalue = [util.typename(self.lname[i], typevalue[i]) for i in range(self.lenindex)]
 696        record = [util.castval(val, typ) for val, typ in zip(record, typevalue)]
 697        '''for i in range(self.lenindex):
 698            if not typevalue[i] and self.typevalue[i]: typevalue[i] = ES.valname[self.typevalue[i]]
 699        #if dtype and not isinstance(dtype, list): dtype = [dtype] * len(record)
 700        #if dtype: record = [util.cast(value, typ) for value, typ in zip(record, dtype)]
 701        record = [util.cast(value, typ) for value, typ in zip(record, typevalue)]'''
 702        if self.isinrecord(self.idxrecord(record), False) and unique: return None
 703        return [self.lindex[i].append(record[i]) for i in range(self.lenindex)]
 704
 705    def applyfilter(self, reverse=False, filtname=ES.filter, delfilter=True, inplace=True):
 706        '''delete records with defined filter value.
 707        Filter is deleted after record filtering.
 708
 709        *Parameters*
 710
 711        - **reverse** :  boolean (default False) - delete record with filter's value is reverse
 712        - **filtname** : string (default ES.filter) - Name of the filter Iindex added 
 713        - **delfilter** :  boolean (default True) - If True, delete filter's Iindex
 714        - **inplace** : boolean (default True) - if True, filter is apply to self,
 715
 716        *Returns* : self or new Ilist'''
 717        if inplace: il = self
 718        else: il = copy(self)
 719        if not filtname in il.lname: return False
 720        ifilt = il.lname.index(filtname)        
 721        il.sort([ifilt], reverse=not reverse, func=None)
 722        minind = min(il.lindex[ifilt].recordfromvalue(reverse))
 723        for idx in il.lindex: del(idx.keys[minind:])
 724        if delfilter: self.delindex(filtname)
 725        il.reindex()
 726        return il
 727    
 728    def couplingmatrix(self, default=False, file=None, att='rate'):
 729        '''return a matrix with coupling infos between each idx.
 730        One info can be stored in a file (csv format).
 731
 732        *Parameters*
 733
 734        - **default** : comparison with default codec 
 735        - **file** : string (default None) - name of the file to write the matrix
 736        - **att** : string - name of the info to store in the file
 737
 738        *Returns* : array of array of dict'''
 739        lenidx = self.lenidx
 740        mat = [[None for i in range(lenidx)] for i in range(lenidx)]
 741        for i in range(lenidx):
 742            for j  in range(i, lenidx): 
 743                mat[i][j] = self.lidx[i].couplinginfos(self.lidx[j], default=default)
 744            for j  in range(i): 
 745                mat[i][j] = copy(mat[j][i])
 746                if   mat[i][j]['typecoupl'] == 'derived': mat[i][j]['typecoupl'] = 'derive'
 747                elif mat[i][j]['typecoupl'] == 'derive' : mat[i][j]['typecoupl'] = 'derived'                
 748                elif mat[i][j]['typecoupl'] == 'linked' : mat[i][j]['typecoupl'] = 'link'
 749                elif mat[i][j]['typecoupl'] == 'link'   : mat[i][j]['typecoupl'] = 'linked'
 750        if file:
 751            with open(file, 'w', newline='') as f:
 752                writer = csv.writer(f)
 753                writer.writerow(self.idxname)
 754                for i in range(lenidx): 
 755                    writer.writerow([mat[i, j][att] for j in range(lenidx)])
 756                writer.writerow(self.idxlen)
 757        return mat
 758
 759    def coupling(self, mat=None, derived=True, rate=0.1):  
 760        '''Transform idx with low rate in coupled or derived indexes (codec extension).
 761
 762        *Parameters*
 763
 764        - **mat** : array of array (default None) - coupling matrix 
 765        - **rate** : integer (default 0.1) - threshold to apply coupling.
 766        - **derived** : boolean (default : True).If True, indexes are derived, else coupled.
 767
 768        *Returns* : list - coupling infos for each idx'''
 769        infos = self.indexinfos(mat=mat)  
 770        coupl = True
 771        while coupl:
 772            coupl = False
 773            for i in range(len(infos)):
 774                if infos[i]['typecoupl'] != 'coupled' and (infos[i]['typecoupl'] 
 775                    not in ('derived', 'unique') or not derived) and infos[i]['linkrate'] < rate: 
 776                    self.lidx[infos[i]['parent']].coupling(self.lidx[i], derived=derived)
 777                    coupl = True
 778            infos = self.indexinfos()  
 779        return infos
 780        
 781    def delrecord(self, record, extern=True):
 782        '''remove a record.
 783
 784        *Parameters*
 785
 786        - **record** :  list - index values to remove to Ilist
 787        - **extern** : if True, compare record values to external representation of self.value, 
 788        else, internal
 789        
 790        *Returns* : row deleted'''
 791        self.reindex()
 792        reckeys = self.valtokey(record, extern=extern)
 793        if None in reckeys: return None
 794        row = self.tiidx.index(reckeys)
 795        for idx in self: del(idx[row])
 796        return row
 797        
 798    def delindex(self, indexname):
 799        '''remove an index.
 800
 801        *Parameters*
 802
 803        - **indexname** : string - name of index to remove
 804
 805        *Returns* : none '''      
 806        self.lindex.pop(self.lname.index(indexname))
 807
 808    def full(self, reindex=False, indexname=None, fillvalue='-', fillextern=True, 
 809             inplace=True, complete=True):
 810        '''tranform a list of indexes in crossed indexes (value extension).
 811
 812        *Parameters*
 813
 814        - **indexname** : list of string - name of indexes to transform
 815        - **reindex** : boolean (default False) - if True, set default codec before transformation
 816        - **fillvalue** : object value used for var extension
 817        - **fillextern** : boolean(default True) - if True, fillvalue is converted to typevalue
 818        - **inplace** : boolean (default True) - if True, filter is apply to self,
 819        - **complete** : boolean (default True) - if True, Iindex are ordered in canonical order
 820
 821        *Returns* : self or new Ilist'''
 822        if inplace: il = self
 823        else: il = copy(self)
 824        if not indexname: primary = il.primary
 825        else: primary = [il.idxname.index(name) for name in indexname]
 826        secondary = [idx for idx in range(il.lenidx) if idx not in primary]
 827        if reindex: il.reindex()
 828        keysadd = util.idxfull([il.lidx[i] for i in primary])
 829        if not keysadd or len(keysadd) == 0: return il
 830        leninit = len(il)
 831        lenadd  = len(keysadd[0])
 832        inf = il.indexinfos()
 833        for i,j in zip(primary, range(len(primary))):
 834            if      inf[i]['cat'] == 'unique': il.lidx[i].keys += [0] * lenadd
 835            else:   il.lidx[i].keys += keysadd[j]
 836        for i in secondary:
 837            if      inf[i]['cat'] == 'unique': il.lidx[i].keys += [0] * lenadd
 838            else:   il.lidx[i].tocoupled(il.lidx[inf[i]['parent']], coupling=False)  
 839        for i in range(il.lenidx):
 840            if len(il.lidx[i].keys) != leninit + lenadd:
 841                raise IlistError('primary indexes have to be present')
 842        if il.lvarname:
 843            il.lvar[0].keys += [len(il.lvar[0].codec)] * len(keysadd[0])
 844            if fillextern: il.lvar[0].codec.append(util.castval(fillvalue, 
 845                             util.typename(il.lvarname[0], ES.def_clsName)))
 846            else: il.lvar[0].codec.append(fillvalue)
 847            #il.lvar[0].codec.append(util.cast(fillvalue, ES.def_dtype))
 848        if complete : il.setcanonorder()
 849        return il
 850
 851    def getduplicates(self, indexname=None, resindex=None):
 852        '''check duplicate cod in a list of indexes. Result is add in a new index or returned.
 853
 854        *Parameters*
 855
 856        - **indexname** : list of string - name of indexes to check
 857        - **resindex** : string (default None) - Add a new index with check result
 858
 859        *Returns* : list of int - list of rows with duplicate cod '''   
 860        if not indexname: primary = self.primary
 861        else: primary = [self.idxname.index(name) for name in indexname]
 862        duplicates = []
 863        for idx in primary: duplicates += self.lidx[idx].getduplicates()
 864        if resindex and isinstance(resindex, str):
 865            newidx = Iindex([True] * len(self), name=resindex)
 866            for item in duplicates: newidx[item] = False
 867            self.addindex(newidx)
 868        return tuple(set(duplicates))
 869            
 870    def iscanonorder(self):
 871        '''return True if primary indexes have canonical ordered keys'''
 872        primary = self.primary
 873        canonorder = util.canonorder([len(self.lidx[idx].codec) for idx in primary])
 874        return canonorder == [self.lidx[idx].keys for idx in primary]
 875
 876    def isinrecord(self, record, extern=True):
 877        '''Check if record is present in self.
 878
 879        *Parameters*
 880
 881        - **record** : list - value for each Iindex
 882        - **extern** : if True, compare record values to external representation of self.value, 
 883        else, internal
 884
 885        *Returns boolean* : True if found'''
 886        if extern: return record in self.textidxext
 887        return record in self.textidx
 888    
 889    def idxrecord(self, record):
 890        '''return rec array (without variable) from complete record (with variable)'''
 891        return [record[self.lidxrow[i]] for i in range(len(self.lidxrow))]
 892    
 893    def indexinfos(self, keys=None, mat=None, default=False, base=False):
 894        '''return an array with infos of each index :
 895            - num, name, cat, typecoupl, diff, parent, pname, pparent, linkrate
 896            - lencodec, min, max, typecodec, rate, disttomin, disttomax (base info)
 897
 898        *Parameters*
 899
 900        - **keys** : list (default none) - list of information to return (reduct dict), all if None
 901        - **default** : build infos with default codec if new coupling matrix is calculated
 902        - **mat** : array of array (default None) - coupling matrix 
 903        - **base** : boolean (default False) - if True, add Iindex infos
 904
 905        *Returns* : array'''
 906        infos = [{} for i in range(self.lenidx)]
 907        if not mat: mat = self.couplingmatrix(default=default)
 908        for i in range(self.lenidx):
 909            infos[i]['num']  = i
 910            infos[i]['name'] = self.idxname[i]
 911            minrate = 1.00
 912            mindiff = len(self)
 913            disttomin = None 
 914            minparent = i
 915            infos[i]['typecoupl'] = 'null'
 916            for j in range(self.lenidx):
 917                if mat[i][j]['typecoupl'] == 'derived': 
 918                    minrate = 0.00
 919                    if mat[i][j]['diff'] < mindiff:
 920                        mindiff = mat[i][j]['diff'] 
 921                        minparent = j 
 922                elif mat[i][j]['typecoupl'] == 'linked' and minrate > 0.0:
 923                    if not disttomin or mat[i][j]['disttomin'] < disttomin:
 924                        disttomin = mat[i][j]['disttomin']
 925                        minrate = mat[i][j]['rate']
 926                        minparent = j
 927                if j < i:
 928                    if mat[i][j]['typecoupl'] == 'coupled':
 929                        minrate = 0.00
 930                        minparent = j
 931                        break
 932                    elif mat[i][j]['typecoupl'] == 'crossed' and minrate > 0.0:
 933                        if not disttomin or mat[i][j]['disttomin'] < disttomin:
 934                            disttomin = mat[i][j]['disttomin']
 935                            minrate = mat[i][j]['rate']
 936                            minparent = j
 937            if self.lidx[i].infos['typecodec'] == 'unique':
 938                infos[i]['cat']           = 'unique'
 939                infos[i]['typecoupl']     = 'unique'
 940                infos[i]['parent']        = i    
 941            elif minrate == 0.0: 
 942                infos[i]['cat']           = 'secondary'
 943                infos[i]['parent']        = minparent
 944            else: 
 945                infos[i]['cat']           = 'primary'
 946                infos[i]['parent']        = minparent                         
 947                if minparent == i: 
 948                    infos[i]['typecoupl'] = 'crossed'
 949            if minparent != i: 
 950                infos[i]['typecoupl']     = mat[i][minparent]['typecoupl']
 951            infos[i]['linkrate']          = round(minrate, 2)
 952            infos[i]['pname']             = self.idxname[infos[i]['parent']]
 953            infos[i]['pparent']           = 0
 954            if base: infos[i]            |= self.lidx[i].infos
 955        for i in range(self.lenidx): util.pparent(i, infos)
 956        if not keys: return infos
 957        return [{k:v for k,v in inf.items() if k in keys} for inf in infos]
 958
 959    def indicator(self, fullsize=None, size=None, indexinfos=None):
 960        '''generate size indicators: ol (object lightness), ul (unicity level), gain (sizegain)
 961        
 962        *Parameters*
 963
 964        - **fullsize** : int (default none) - size with fullcodec
 965        - **size** : int (default none) - size with existing codec
 966        - **indexinfos** : list (default None) - indexinfos data
 967
 968        *Returns* : dict'''        
 969        if not indexinfos: indexinfos = self.indexinfos()
 970        if not fullsize: fullsize = len(self.to_obj(indexinfos=indexinfos, encoded=True, fullcodec=True))
 971        if not size:     size     = len(self.to_obj(indexinfos=indexinfos, encoded=True))
 972        lenidx = self.lenidx
 973        nv = len(self) * (lenidx + 1)
 974        sv = fullsize / nv
 975        nc = sum(self.idxlen) + lenidx
 976        if nv != nc: 
 977            sc = (size - nc * sv) / (nv - nc)
 978            ol = sc / sv
 979        else: ol = None
 980        return {'init values': nv, 'mean size': round(sv, 3),
 981                'unique values': nc, 'mean coding size': round(sc, 3), 
 982                'unicity level': round(nc / nv, 3), 'object lightness': round(ol, 3),
 983                'gain':round((fullsize - size) / fullsize, 3)}
 984        
 985    def json(self, **kwargs):
 986        '''
 987        Return json dict, json string or Cbor binary.
 988
 989        *Parameters (kwargs)*
 990
 991        - **encoded** : boolean (default False) - choice for return format (bynary if True, dict else)
 992        - **encode_format** : string (default 'json') - choice for return format (json, bson or cbor)
 993        - **json_res_index** : default False - if True add the index to the value
 994        - **order** : default [] - list of ordered index
 995        - **codif** : dict (default {}). Numerical value for string in CBOR encoder
 996
 997        *Returns* : string or dict'''
 998        return self.to_obj(**kwargs)
 999
1000    def keytoval(self, listkey, extern=True):
1001        '''
1002        convert a keys list (key for each idx) to a values list (value for each idx).
1003
1004        *Parameters*
1005
1006        - **listkey** : key for each idx
1007        - **extern** : boolean (default True) - if True, compare rec to val else to values 
1008        
1009        *Returns*
1010
1011        - **list** : value for each index'''
1012        return [idx.keytoval(key, extern=extern) for idx, key in zip(self.lidx, listkey)] 
1013    
1014    def loc(self, rec, extern=True, row=False):
1015        '''
1016        Return variable value or row corresponding to a list of idx values.
1017
1018        *Parameters*
1019
1020        - **rec** : list - value for each idx
1021        - **extern** : boolean (default True) - if True, compare rec to val else to values 
1022        - **row** : Boolean (default False) - if True, return list of row, else list of variable values
1023        
1024        *Returns*
1025
1026        - **object** : variable value or None if not found'''
1027        locrow = list(set.intersection(*[set(self.lidx[i].loc(rec[i], extern)) 
1028                                       for i in range(self.lenidx)]))
1029        if row: return locrow
1030        else: return self.lvar[0][tuple(locrow)]
1031
1032    def merge(self, name='merge', fillvalue=math.nan, mergeidx=False, updateidx=False):
1033        '''
1034        Merge method replaces Ilist objects included in variable data into its constituents.
1035
1036        *Parameters*
1037
1038        - **name** : str (default 'merge') - name of the new Ilist object
1039        - **fillvalue** : object (default nan) - value used for the additional data 
1040        - **mergeidx** : create a new index if mergeidx is False
1041        - **updateidx** : if True (and mergeidx is True), update actual values if index name is present 
1042        
1043        *Returns*: merged Ilist '''     
1044        find = True
1045        ilm = copy(self)
1046        nameinit = ilm.idxname
1047        while find:
1048            find = False
1049            for i in range(len(ilm)):
1050                if not ilm.lvar[0].values[i].__class__.__name__ in ['Ilist', 'Obs']: continue
1051                find = True
1052                il = ilm.lvar[0].values[i].merge()
1053                ilname = il.idxname
1054                record = ilm.recidx(i, extern=False)
1055                for val, j in zip(reversed(record), reversed(range(len(record)))): # Ilist pere
1056                    nameidx = ilm.lidx[j].name
1057                    updidx = nameidx in nameinit and not updateidx
1058                    il.addindex ([nameidx, [val] * len(il)], first=True,
1059                                          merge=mergeidx, update=updidx) # ajout des index au fils
1060                for name in ilname:
1061                    fillval = util.castval(fillvalue, util.typename(name, ES.def_clsName))
1062                    ilm.addindex([name, [fillval] * len(ilm)], 
1063                                          merge=mergeidx, update=False) # ajout des index au père
1064                del(ilm[i])
1065                il.renameindex(il.lvarname[0], ilm.lvarname[0]) 
1066                ilm += il
1067                break
1068        return ilm
1069
1070    def merging(self, listname=None):
1071        ''' add a new index build with indexes define in listname'''
1072        self.addindex(Iindex.merging([self.nindex(name) for name in listname]))
1073        return None
1074    
1075    def nindex(self, name):
1076        ''' index with name equal to attribute name'''
1077        if name in self.lname: return self.lindex[self.lname.index(name)]
1078        return None
1079    
1080    def plot(self, order=None, line=True, size=5, marker='o', maxlen=20):
1081        '''
1082        This function visualize data with line or colormesh.
1083
1084        *Parameters*
1085
1086        - **line** : Boolean (default True) - Choice line or colormesh.
1087        - **order** : list (defaut None) - order of the axes (x, y, hue or col)
1088        - **size** : int (defaut 5) - plot size
1089        - **marker** : Char (default 'o') - Symbol for each point.
1090        - **maxlen** : Integer (default 20) - maximum length for string
1091
1092        *Returns*
1093
1094        - **None**  '''
1095        if not self.consistent : return
1096        xa = self.to_xarray(numeric = True, lisfunc=[util.cast], dtype='str',
1097                             npdtype='str', maxlen=maxlen)
1098        if not order: order = [0,1,2]
1099        
1100        if   len(xa.dims) == 1:
1101            xa.plot.line(x=xa.dims[0]+'_row', size=size, marker=marker)
1102        elif len(xa.dims) == 2 and line:
1103            xa.plot.line(x=xa.dims[order[0]]+ '_row',
1104                xticks=list(xa.coords[xa.dims[0]+'_row'].values),
1105                #hue=xa.dims[order[1]]+'_row', size=size, marker=marker)
1106                hue=xa.dims[order[1]], size=size, marker=marker)
1107        elif len(xa.dims) == 2 and not line:
1108            xa.plot(x=xa.dims[order[0]]+'_row', y=xa.dims[order[1]]+'_row',
1109                xticks=list(xa.coords[xa.dims[order[0]]+'_row'].values),
1110                yticks=list(xa.coords[xa.dims[order[1]]+'_row'].values),
1111                size = size)
1112        elif len(xa.dims) == 3 and line:
1113            xa.plot.line(x=xa.dims[order[0]]+ '_row', col=xa.dims[order[1]],
1114                xticks=list(xa.coords[xa.dims[order[0]]+'_row'].values),
1115                hue=xa.dims[order[2]], col_wrap=2, size=size, marker=marker)
1116        elif len(xa.dims) == 3 and not line:
1117            xa.plot(x=xa.dims[order[0]]+'_row', y=xa.dims[order[1]]+'_row', 
1118                xticks=list(xa.coords[xa.dims[order[0]]+'_row'].values),
1119                yticks=list(xa.coords[xa.dims[order[1]]+'_row'].values),
1120                col=xa.dims[order[2]], col_wrap=2, size=size)
1121        plt.show()
1122        return {xa.dims[i]: list(xa.coords[xa.dims[i]].values) for i in range(len(xa.dims))}
1123
1124    def record(self, row, extern=True):
1125        '''return the record at the row
1126           
1127        *Parameters*
1128
1129        - **row** : int - row of the record
1130        - **extern** : boolean (default True) - if True, return val record else value record
1131
1132        *Returns*
1133
1134        - **list** : val record or value record'''
1135        if extern: return [idx.valrow(row) for idx in self.lindex]
1136        #if extern: return [idx.val[row] for idx in self.lindex]
1137        return [idx.values[row] for idx in self.lindex]
1138    
1139    def recidx(self, row, extern=True):
1140        '''return the list of idx val or values at the row
1141           
1142        *Parameters*
1143
1144        - **row** : int - row of the record
1145        - **extern** : boolean (default True) - if True, return val rec else value rec
1146
1147        *Returns*
1148
1149        - **list** : val or value for idx'''
1150        #if extern: return [idx.val[row] for idx in self.lidx]
1151        if extern: return [idx.valrow(row) for idx in self.lidx]
1152        return [idx.values[row] for idx in self.lidx]
1153
1154    def recvar(self, row, extern=True):
1155        '''return the list of var val or values at the row
1156           
1157        *Parameters*
1158
1159        - **row** : int - row of the record
1160        - **extern** : boolean (default True) - if True, return val rec else value rec
1161
1162        *Returns*
1163
1164        - **list** : val or value for var'''
1165        #if extern: return [idx.val[row] for idx in self.lidx]
1166        if extern: return [idx.valrow(row) for idx in self.lvar]
1167        return [idx.values[row] for idx in self.lvar]
1168
1169    def reindex(self):
1170        '''Calculate a new default codec for each index (Return self)'''
1171        for idx in self.lindex: idx.reindex()
1172        return self       
1173
1174    def renameindex(self, oldname, newname):
1175        '''replace an index name 'oldname' by a new one 'newname'. '''
1176        for i in range(self.lenindex):
1177            if self.lname[i] == oldname: self.lindex[i].setname(newname)
1178        for i in range(len(self.lvarname)):
1179            if self.lvarname[i] == oldname: self.lvarname[i] = newname
1180
1181    def reorder(self, recorder=None):
1182        '''Reorder records in the order define by 'recorder' '''
1183        if recorder is None or set(recorder) != set(range(len(self))): return
1184        for idx in self.lindex: idx.keys = [idx.keys[i] for i in recorder]
1185        return None
1186        
1187    def setcanonorder(self):
1188        '''Set the canonical index order : primary - secondary/unique - variable.
1189        Set the canonical keys order : ordered keys in the first columns.
1190        Return self'''
1191        order = [self.lidxrow[idx] for idx in self.primary]
1192        order += [idx for idx in self.lidxrow if not idx in order]
1193        order += self.lvarrow
1194        self.swapindex(order)
1195        self.sort()
1196        return self
1197        
1198    def setfilter(self, filt=None, first=False, filtname=ES.filter):
1199        '''Add a filter index with boolean values
1200           
1201        - **filt** : list of boolean - values of the filter idx to add
1202        - **first** : boolean (default False) - If True insert index at the first row, else at the end
1203        - **filtname** : string (default ES.filter) - Name of the filter Iindex added 
1204
1205        *Returns* : none'''
1206        if not filt: filt = [True] * len(self)
1207        idx = Iindex(filt, name=filtname)
1208        idx.reindex()
1209        if not idx.cod in ([True, False], [False, True], [True], [False]):
1210            raise IlistError('filt is not consistent')
1211        if ES.filter in self.lname: self.delindex(ES.filter)
1212        self.addindex(idx, first=first)
1213                
1214    def setname(self, listname=None):
1215        '''Update Iindex name by the name in listname'''
1216        for i in range(min(self.lenindex, len(listname))):
1217            self.lindex[i].name = listname[i]
1218
1219    def setvar(self, var=None):
1220        '''Define a var index by the name or the index row'''
1221        if var is None: self.lvarname = []
1222        elif isinstance(var, int) and var >= 0 and var < self.lenindex: 
1223            self.lvarname = [self.lname[var]]
1224        elif isinstance(var, str) and var in self.lname:
1225            self.lvarname = [var]
1226        else: raise IlistError('var is not consistent with Ilist')
1227
1228    def sort(self, order=None, reverse=False, func=str):
1229        '''Sort data following the index order and apply the ascending or descending 
1230        sort function to values.
1231        
1232        *Parameters*
1233
1234        - **order** : list (default None)- new order of index to apply. If None or [], 
1235        the sort function is applied to the existing order of indexes.
1236        - **reverse** : boolean (default False)- ascending if True, descending if False
1237        - **func**    : function (default str) - parameter key used in the sorted function
1238
1239        *Returns* : self'''
1240        if not order: order = []
1241        orderfull = order + list(set(range(self.lenindex)) - set(order))
1242        for idx in [self.lindex[i] for i in order]:
1243            idx.reindex(codec=sorted(idx.codec, key=func))
1244        newidx = util.transpose(sorted(util.transpose(
1245            [self.lindex[orderfull[i]].keys for i in range(self.lenindex)]), 
1246            reverse=reverse))
1247        for i in range(self.lenindex): self.lindex[orderfull[i]].keys = newidx[i]
1248        return self
1249
1250    def swapindex(self, order):
1251        '''
1252        Change the order of the index .
1253
1254        *Parameters*
1255
1256        - **order** : list of int - new order of index to apply.
1257
1258        *Returns* : self '''
1259        if self.lenindex != len(order): raise IlistError('length of order and Ilist different')
1260        self.lindex=[self.lindex[order[i]] for i in range(len(order))]
1261        return self
1262
1263    def tostdcodec(self, inplace=False, full=True):
1264        '''Transform all codec in full or default codec.
1265        
1266        *Parameters*
1267
1268        - **inplace** : boolean  (default False) - if True apply transformation to self, else to a new Ilist
1269        - **full** : boolean (default True)- full codec if True, default if False
1270
1271
1272        *Return Ilist* : self or new Ilist'''
1273        lindex = [idx.tostdcodec(inplace=False, full=full) for idx in self.lindex]
1274        if inplace:
1275            self.lindex = lindex
1276            return self
1277        return Ilist(lindex, var=self.lvarrow[0])
1278
1279    def to_csv(self, filename, optcsv={'quoting': csv.QUOTE_NONNUMERIC}, **kwargs):
1280        '''
1281        Generate csv file to display data.
1282
1283        *Parameters*
1284
1285        - **filename** : string - file name (with path)
1286        - **optcsv** : parameter for csv.writer
1287
1288        *Parameters (kwargs)*
1289
1290        - **name=listcode** : element (default None) - eg location='ns'
1291            - listcode : string with Code for each index (j: json, n: name, s: simple).
1292            - name : name of the index 
1293        - **lenres** : Integer (default : 0) - Number of raws (all if 0)
1294        - **header** : Boolean (default : True) - If True, first line with names
1295        - **optcsv** : parameter for csv.writer
1296        - **ifunc** : function (default None) - function to apply to indexes
1297        - **other kwargs** : parameter for ifunc
1298        
1299        *Returns* : size of csv file '''
1300        size = 0
1301        if not optcsv: optcsv = {}
1302        tab = self._to_tab(**kwargs)
1303        with open(filename, 'w', newline='') as csvfile:
1304            writer = csv.writer(csvfile, **optcsv)
1305            for lign in tab : 
1306                size += writer.writerow(lign)
1307        return size
1308
1309    def to_dataFrame(self, info=False, idx=None, fillvalue='?', fillextern=True,
1310                  lisfunc=None, name=None, numeric=False, npdtype=None, **kwargs):
1311        '''
1312        Complete the Object and generate a Pandas dataFrame with the dimension define by idx.
1313
1314        *Parameters*
1315
1316        - **info** : boolean (default False) - if True, add _dict attributes to attrs Xarray
1317        - **idx** : list (default none) - list of idx to be completed. If [],
1318        self.primary is used.
1319        - **fillvalue** : object (default '?') - value used for the new extval
1320        - **fillextern** : boolean(default True) - if True, fillvalue is converted to typevalue
1321        - **lisfunc** : function (default none) - list of function to apply to indexes before export
1322        - **name** : string (default None) - DataArray name. If None, variable name
1323        - **numeric** : Boolean (default False) - Generate a numeric DataArray.Values.
1324        - **npdtype** : string (default None) - numpy dtype for the DataArray ('object' if None)
1325        - **kwargs** : parameter for lisfunc
1326
1327        *Returns* : pandas.DataFrame '''
1328        if self.consistent :
1329            return self.to_xarray(info=info, idx=idx, fillvalue=fillvalue, 
1330                                  fillextern=fillextern, lisfunc=lisfunc, name=name,
1331                                  numeric=numeric, npdtype=npdtype, **kwargs
1332                                  ).to_dataframe(name=name)
1333        return None
1334    
1335    def to_xarray(self, info=False, idx=None, fillvalue='?', fillextern=True,
1336                  lisfunc=None, name=None, numeric=False, npdtype=None, attrs=None, **kwargs):
1337        '''
1338        Complete the Object and generate a Xarray DataArray with the dimension define by idx.
1339
1340        *Parameters*
1341
1342        - **info** : boolean (default False) - if True, add _dict attributes to attrs Xarray
1343        - **idx** : list (default none) - list of idx to be completed. If [],
1344        self.primary is used.
1345        - **fillvalue** : object (default '?') - value used for the new extval
1346        - **fillextern** : boolean(default True) - if True, fillvalue is converted to typevalue
1347        - **lisfunc** : function (default none) - list of function to apply to indexes before export
1348        - **name** : string (default None) - DataArray name. If None, variable name
1349        - **numeric** : Boolean (default False) - Generate a numeric DataArray.Values.
1350        - **npdtype** : string (default None) - numpy dtype for the DataArray ('object' if None)
1351        - **attrs** : dict (default None) - attributes for the DataArray
1352        - **kwargs** : parameter for lisfunc
1353
1354        *Returns* : DataArray '''
1355        option = {'dtype': None} | kwargs 
1356        if not self.consistent : raise IlistError("Ilist not consistent")
1357        if len(self.lvarname) == 0 : raise IlistError("Variable is not defined")
1358        if isinstance(lisfunc, list) and len(lisfunc) == 1: 
1359            lisfunc = lisfunc * self.lenindex
1360        elif isinstance(lisfunc, list) and len(lisfunc) != self.lenindex : 
1361            lisfunc = [None] * self.lenindex
1362        elif not isinstance(lisfunc, list):
1363            funcvar = lisfunc            
1364            lisfunc = [None] * self.lenindex
1365            lisfunc[self.lvarrow[0]] = funcvar
1366        lisfuncname = dict(zip(self.lname, lisfunc))
1367        if idx is None or idx==[] : idx = self.primary
1368        axesname = [self.idxname[i] for i in idx[:len(self.idxname)]]
1369        ilf = self.full(indexname=axesname, fillvalue=fillvalue, 
1370                        fillextern=fillextern, inplace=False)
1371        ilf.setcanonorder()
1372        idxilf = list(range(len(idx[:len(self.idxname)])))
1373        coord = ilf._xcoord(idxilf, lisfuncname, **option)
1374        dims = [ilf.idxname[i] for i in idxilf]
1375        if numeric: 
1376            lisfunc[self.lvarrow[0]] = util.cast
1377            fillvalue= math.nan
1378            npdtype='float'
1379            option['dtype'] = 'float'
1380        data = ilf.lvar[0].to_numpy(func=lisfunc[self.lvarrow[0]], 
1381                                    npdtype=npdtype, **option
1382                                     ).reshape([ilf.idxlen[idx] for idx in idxilf])
1383        if not name: name = self.name
1384        if not isinstance(attrs, dict): attrs = {}
1385        for nam in self.lunicname: attrs[nam] = self.nindex(nam).codec[0]
1386        if info: attrs |= ilf.indexinfos()
1387        return xarray.DataArray(data, coord, dims, attrs=attrs, name=name)
1388
1389    def to_file(self, file, **kwargs) :
1390        '''Generate file to display data.
1391
1392         *Parameters (kwargs)*
1393
1394        - **file** : string - file name (with path)
1395        - **kwargs** : see 'to_obj' parameters
1396
1397        *Returns* : Integer - file lenght (bytes)  '''
1398        option = {'encode_format': 'cbor'} | kwargs | {'encoded': True}
1399        data = self.to_obj(**option)
1400        if option['encode_format'] == 'cbor':
1401            size = len(data)
1402            with open(file, 'wb') as f: f.write(data)
1403        else:
1404            size = len(bytes(data, 'UTF-8'))
1405            with open(file, 'w', newline='') as f: f.write(data)
1406        return size
1407    
1408    def to_obj(self, indexinfos=None, **kwargs):
1409        '''Return a formatted object (json string, cbor bytes or json dict). 
1410
1411        *Parameters (kwargs)*
1412
1413        - **encoded** : boolean (default False) - choice for return format (string/bytes if True, dict else)
1414        - **encode_format**  : string (default 'json')- choice for return format (json, cbor)
1415        - **codif** : dict (default ES.codeb). Numerical value for string in CBOR encoder
1416        - **fullcodec** : boolean (default False) - if True, each index is with a full codec
1417        - **defaultcodec** : boolean (default False) - if True, each index is whith a default codec
1418        - **name** : boolean (default False) - if False, default index name are not included
1419
1420        *Returns* : string, bytes or dict'''
1421        option = {'fullcodec': False, 'defaultcodec': False, 'encoded': False, 
1422                  'encode_format': 'json', 'codif': ES.codeb, 'name': False} | kwargs
1423        option2 = {'encoded': False, 'encode_format': 'json', 'codif': option['codif']}
1424        lis = []
1425        if option['fullcodec'] or option['defaultcodec']: 
1426            for idx in self.lidx: 
1427                idxname = option['name'] or idx.name != 'i' + str(self.lname.index(idx.name))
1428                lis.append(idx.tostdcodec(full=not option['defaultcodec'])
1429                           .to_obj(keys=not option['fullcodec'], name=idxname, **option2))
1430        else:
1431            if not indexinfos: indexinfos=self.indexinfos(default=False)
1432            notkeyscrd = True 
1433            if self.iscanonorder(): notkeyscrd = None
1434            for idx, inf in zip(self.lidx, indexinfos):
1435                idxname = option['name'] or idx.name != 'i' + str(self.lname.index(idx.name))
1436                if   inf['typecoupl'] == 'unique' : 
1437                    lis.append(idx.tostdcodec(full=False).to_obj(name=idxname, **option2))
1438                elif inf['typecoupl'] == 'crossed': 
1439                    lis.append(idx.to_obj(keys=notkeyscrd, name=idxname, **option2))
1440                elif inf['typecoupl'] == 'coupled': 
1441                    lis.append(idx.setkeys(self.lidx[inf['parent']].keys, inplace=False).
1442                               to_obj(parent=self.lidxrow[inf['parent']], 
1443                                      name=idxname, **option2))
1444                elif inf['typecoupl'] == 'linked' : 
1445                    lis.append(idx.to_obj(keys=True, name=idxname, **option2))
1446                elif inf['typecoupl'] == 'derived': 
1447                    if idx.iskeysfromderkeys(self.lidx[inf['parent']]):
1448                        lis.append(idx.to_obj(parent=self.lidxrow[inf['parent']], 
1449                                          name=idxname, **option2))                        
1450                    else:
1451                        keys=idx.derkeys(self.lidx[inf['parent']])
1452                        lis.append(idx.to_obj(keys=keys, parent=self.lidxrow[inf['parent']], 
1453                                          name=idxname, **option2))
1454                else: raise IlistError('Iindex type undefined')
1455        if self.lenindex > 1: parent = ES.variable
1456        else: parent = ES.nullparent
1457        for i in self.lvarrow: 
1458            idx = self.lindex[i]
1459            idxname = option['name'] or idx.name != 'i' + str(self.lname.index(idx.name))
1460            if i != self.lenindex - 1:
1461                lis.insert(i, idx.tostdcodec(full=True).
1462                           to_obj(keys=False, parent=parent, name=idxname, **option2))        
1463            else:
1464                lis.append(idx.tostdcodec(full=True).
1465                           to_obj(keys=False, parent=parent, name=idxname, **option2))        
1466        if option['encoded'] and option['encode_format'] == 'json': 
1467            return  json.dumps(lis, cls=IindexEncoder)
1468        if option['encoded'] and option['encode_format'] == 'cbor': 
1469            return cbor2.dumps(lis, datetime_as_timestamp=True, 
1470                               timezone=datetime.timezone.utc, canonical=True)
1471        return lis
1472
1473    def updateindex(self, listvalue, index, extern=True, typevalue=None):
1474        '''update values of an index.
1475
1476        *Parameters*
1477
1478        - **listvalue** : list - index values to replace
1479        - **index** : integer - index row to update
1480        - **typevalue** : str (default None) - class to apply to the new value 
1481        - **extern** : if True, the listvalue has external representation, else internal
1482
1483        *Returns* : none '''      
1484        self.lindex[index].setlistvalue(listvalue, extern=extern, typevalue=typevalue)
1485
1486    def valtokey(self, rec, extern=True):
1487        '''convert a rec list (value or val for each idx) to a key list (key for each idx).
1488
1489        *Parameters*
1490
1491        - **rec** : list of value or val for each idx
1492        - **extern** : if True, the rec value has external representation, else internal
1493
1494        *Returns*
1495
1496        - **list of int** : rec key for each idx'''
1497        return [idx.valtokey(val, extern=extern) for idx, val in zip(self.lidx, rec)]
1498
1499    def view(self, **kwargs) :
1500        '''
1501        Generate tabular list to display data.
1502
1503        *Parameters (kwargs)*
1504
1505        - **name=listcode** : element (default None) - eg location='ns'
1506            - listcode : string with Code for each index (j: json, n: name, s: simple).
1507            - name : name of the index 
1508        - **defcode** : String (default : 'j') - default list code (if 'all' is True)
1509        - **all** : Boolean (default : True) - 'defcode apply to all indexes or none
1510        - **lenres** : Integer (default : 0) - Number of raws (all if 0)
1511        - **header** : Boolean (default : True) - First line with names
1512        - **width** : Integer (default None) - Number of characters displayed for each attribute (all if None)
1513        - **ifunc** : function (default None) - function to apply to indexes
1514        - **tabulate params** : default 'tablefmt': 'simple', 'numalign': 'left', 'stralign': 'left',
1515                   'floatfmt': '.3f' - See tabulate module
1516        - **other kwargs** : parameter for ifunc
1517
1518        *Returns* : list or html table (tabulate format) '''
1519        #print(kwargs)
1520        opttab = {'defcode': 'j', 'all': True, 'lenres': 0, 'header':True}
1521        optview = {'tablefmt': 'simple', 'numalign': 'decimal', 'stralign': 'left', 'floatfmt': '.2f'}
1522        option = opttab | optview | kwargs
1523        tab = self._to_tab(**option)
1524        width = ({'width': None} | kwargs)['width']
1525        if width: tab = [[(lambda x : x[:width] if type(x)==str else x)(val) 
1526                           for val in lig] for lig in tab]
1527        return tabulate(tab, headers='firstrow', **{k: option[k] for k in optview})
1528    
1529    def vlist(self, *args, func=None, index=-1, **kwargs):
1530        '''
1531        Apply a function to an index and return the result.
1532
1533        *Parameters*
1534
1535        - **func** : function (default none) - function to apply to extval or extidx
1536        - **args, kwargs** : parameters for the function
1537        - **index** : integer - index to update (index=-1 for variable)
1538
1539        *Returns* : list of func result'''
1540        if index == -1 and self.lvar: return self.lvar[0].vlist(func, *args, **kwargs)
1541        if index == -1 and self.lenindex == 1: index = 0        
1542        return self.lindex[index].vlist(func, *args, **kwargs) 
1543
1544    def voxel(self):
1545        '''
1546        Plot not null values in a cube with voxels and return indexes values.
1547        
1548        *Returns* : **dict of indexes values**
1549        '''
1550        if not self.consistent : return
1551        if   self.lenidx  > 3: raise IlistError('number of idx > 3')
1552        elif self.lenidx == 2: self.addindex(Iindex('null', ' ', keys=[0]*len(self)))
1553        elif self.lenidx == 1: 
1554            self.addindex(Iindex('null', ' ', keys=[0]*len(self)))
1555            self.addindex(Iindex('null', '  ', keys=[0]*len(self)))
1556        xa = self.to_xarray(idx=[0,1,2], fillvalue='?', fillextern=False,
1557                             lisfunc=util.isNotEqual, tovalue='?')
1558        ax = plt.figure().add_subplot(projection='3d')
1559        ax.voxels(xa, edgecolor='k')
1560        ax.set_xticks(np.arange(self.idxlen[self.idxname.index(xa.dims[0])]))
1561        ax.set_yticks(np.arange(self.idxlen[self.idxname.index(xa.dims[1])]))
1562        ax.set_zticks(np.arange(self.idxlen[self.idxname.index(xa.dims[2])]))
1563        ax.set(xlabel=xa.dims[0][:8], 
1564               ylabel=xa.dims[1][:8],
1565               zlabel=xa.dims[2][:8])
1566        plt.show()
1567        return {xa.dims[i]: list(xa.coords[xa.dims[i]].values) 
1568                for i in range(len(xa.dims))}
1569
1570    def _to_tab(self, **kwargs):
1571        ''' data preparation (dict of dict) for view or csv export. 
1572        Representation is included if :
1573            - code is definie in the name element of the field
1574            - or code is defined in 'defcode' element and 'all' element is True
1575
1576        *Parameters (kwargs)*
1577
1578        - **name=listcode** : element (default None) - eg location='ns'
1579            - listcode : string with Code for each index (j: json, n: name, s: simple, f: ifunc).
1580            - name : name of the index 
1581        - **defcode** : String (default : 'j') - default list code (if 'all' is True)
1582        - **all** : Boolean (default : True) - 'defcode apply to all indexes or none
1583        - **lenres** : Integer (default : 0) - Number of raws (all if 0)
1584        - **ifunc** : function (default None) - function to apply to indexes
1585        - **other kwargs** : parameter for ifunc'''
1586
1587        option = {'defcode': 'j', 'all': True, 'lenres': 0, 'ifunc': None,
1588                  'header': True}  | kwargs
1589        tab = list()
1590        resList = []
1591        diccode = {'j': '', 'n': 'name-', 's': 'smpl-', 'f': 'func-'}
1592        if option['header']:
1593            for name in self.lname:
1594                if name in option:
1595                    for n, code in diccode.items():
1596                        if n in option[name]: resList.append(code + name)
1597                elif option['all']:
1598                    for n, code in diccode.items():
1599                        if n in option['defcode']: resList.append(code + name)                
1600                '''for n, code in diccode.items():
1601                    if (name in option and n in option[name]) or (
1602                                option['all'] and n in option['defcode']):
1603                        resList.append(code + name)'''
1604            tab.append(resList)
1605        lenres = option['lenres']
1606        if lenres == 0 : lenres = len(self)
1607        for i in range(min(lenres, len(self))) :
1608            resList = []
1609            for name in self.lname:
1610                if name in option:
1611                    for n, code in diccode.items():
1612                        if n in option[name]: 
1613                            val = self.nindex(name).values[i]
1614                            if n == 'j': resList.append(util.cast(val, dtype='json'))
1615                            if n == 'n': resList.append(util.cast(val, dtype='name'))
1616                            if n == 's': resList.append(util.cast(val, dtype='json', string=True))
1617                            if n == 'f': resList.append(util.funclist(val, option['ifunc'], **kwargs))
1618                elif option['all']:
1619                    for n, code in diccode.items():
1620                        if n in option['defcode']: 
1621                            val = self.nindex(name).values[i]
1622                            if n == 'j': resList.append(util.cast(val, dtype='json'))
1623                            if n == 'n': resList.append(util.cast(val, dtype='name'))
1624                            if n == 's': resList.append(util.cast(val, dtype='json', string=True))
1625                            if n == 'f': resList.append(util.funclist(val, option['ifunc'], **kwargs))
1626            tab.append(resList)
1627        return tab
1628    
1629    def _xcoord(self, axe, lisfuncname=None, **kwargs) :
1630        ''' Coords generation for Xarray'''
1631        if 'maxlen' in kwargs : maxlen=kwargs['maxlen']
1632        else: maxlen = 20
1633        info = self.indexinfos()
1634        coord = {}
1635        for i in self.lidxrow:
1636            fieldi = info[i]
1637            if fieldi['cat'] == 'unique': continue
1638            if isinstance(lisfuncname, dict) and len(lisfuncname) == self.lenindex: 
1639                funci= lisfuncname[self.lname[i]]
1640            else : funci = None
1641            iname = self.idxname[i]
1642            if i in axe :  
1643                coord[iname] = self.lidx[i].to_numpy(func=funci, codec=True, **kwargs)
1644                coord[iname+'_row'] = (iname, np.arange(len(coord[iname])))
1645                coord[iname+'_str'] = (iname, self.lidx[i].to_numpy(func=util.cast, 
1646                                       codec=True, dtype='str', maxlen=maxlen))
1647            else:
1648                self.lidx[i].setkeys(self.lidx[fieldi['pparent']].keys)
1649                coord[iname] = (self.idxname[fieldi['pparent']], 
1650                                self.lidx[i].to_numpy(func=funci, codec=True, **kwargs))
1651        return coord
1652
1653class IlistError(Exception):
1654    ''' Ilist Exception'''
1655    #pass