tab-dataset.tab_dataset.field

  1# -*- coding: utf-8 -*-
  2"""
  3The `field` module is part of the `tab-dataset` package.
  4
  5It contains the classes `Sfield` and `Nfield` for Field entities.
  6
  7For more information, see the 
  8[user guide](https://loco-philippe.github.io/tab-dataset/docs/user_guide.html) 
  9or the [github repository](https://github.com/loco-philippe/tab-dataset).
 10"""
 11from copy import copy
 12import json
 13
 14from json_ntv.ntv import Ntv, NtvSingle
 15from json_ntv.ntv_util import NtvJsonEncoder
 16
 17from tab_dataset.field_interface import FieldInterface
 18from tab_dataset.cfield import Cfield, FieldError, Cutil
 19
 20DEFAULTINDEX = '$default'
 21
 22
 23class Sfield(FieldInterface, Cfield):
 24    '''
 25    `Sfield` is a child class of `Cfield` where internal value can be different
 26    from external value (list is converted in tuple and dict in json-object).
 27
 28    Attributes are the same as Cfield class
 29
 30    The methods defined in this class are :
 31
 32    *constructor (@classmethod)*
 33
 34    - `Cfield.bol`
 35    - `Cfield.from_ntv`
 36    - `Cfield.ntv`
 37    - `Cfield.like`
 38    - `Sfield.merging`
 39
 40    *conversion (@staticmethod)*
 41
 42    - `Sfield.l_to_i`
 43    - `Sfield.s_to_i`
 44    - `Sfield.l_to_e`
 45    - `Sfield.s_to_e`
 46    - `Sfield.i_to_n`
 47    - `Sfield.n_to_i`
 48    - `Sfield.i_to_name`
 49    - `Cfield.ntv_to_val` (@classmethod)
 50    
 51    *dynamic value (getters @property)*
 52
 53    - `Sfield.val`
 54    - `Sfield.cod`
 55    - `Cfield.hashf`
 56    - `Cfield.to_analysis`
 57    - `Cfield.values`
 58    - `Cfield.codec`
 59    - `Cfield.infos`
 60    - `Cfield.keys`
 61    
 62    *add - update methods*
 63
 64    - `Sfield.append`
 65    - `Sfield.setcodecvalue`
 66    - `Sfield.setcodeclist`
 67    - `Sfield.setlistvalue`
 68    - `Sfield.setvalue`
 69    - `Cfield.add`
 70    - `Cfield.setname`
 71    - `Cfield.set_keys`
 72    - `Cfield.set_codec`
 73    - `Cfield.setkeys`
 74
 75    *transform methods*
 76
 77    - `Cfield.coupling`
 78    - `Cfield.extendkeys`
 79    - `Cfield.full`
 80    - `Cfield.reindex`
 81    - `Cfield.reorder`
 82    - `Cfield.sort`
 83    - `Cfield.tocoupled`
 84    - `Cfield.tostdcodec`
 85    
 86    *getters methods*
 87
 88    - `Sfield.isvalue`
 89    - `Sfield.keytoval`
 90    - `Sfield.loc`
 91    - `Sfield.recordfromvalue`
 92    - `Sfield.valtokey`
 93    - `Cfield.couplinginfos`
 94    - `Cfield.derkeys`
 95    - `Cfield.getduplicates`
 96    - `Cfield.iscrossed`
 97    - `Cfield.iscoupled`
 98    - `Cfield.isderived`
 99    - `Cfield.islinked`
100    - `Cfield.iskeysfromderkeys`
101    - `Cfield.recordfromkeys`
102
103    *export methods (`observation.field_interface.SfieldInterface`)*
104
105    - `Sfield.json`
106    - `Sfield.to_obj`
107    - `Sfield.to_dict_obj`
108    - `Sfield.to_numpy`
109    - `Sfield.to_pandas`
110    - `Sfield.vlist`
111    - `Sfield.v_name`
112    - `Sfield.vSimple`
113    '''
114
115    def __init__(self, codec=None, name=None, keys=None,
116                 lendefault=0, reindex=False, fast=False):
117        '''
118        Sfield constructor.
119
120        *Parameters*
121
122        - **codec** :  list (default None) - external different values of index (see data model)
123        - **keys** :  list (default None)  - key value of index (see data model)
124        - **name** : string (default None) - name of index (see data model)
125        - **lendefault** : integer (default 0) - default len if no keys is defined
126        - **reindex** : boolean (default True) - if True, default codec is apply
127        - **fast**: boolean (default False) - if True, codec is created without conversion'''
128        if isinstance(codec, Sfield):
129            Cfield.__init__(self, copy(codec._codec),
130                            copy(codec.name), copy(codec._keys))
131            return
132        if codec is None:
133            codec = []
134        if not isinstance(codec, list):
135            codec = [codec]
136        leng = lendefault
137        if codec and len(codec) > 0 and not leng:
138            leng = len(codec)
139        if not keys is None:
140            leng = len(keys)
141        if not name:
142            name = DEFAULTINDEX
143        if not (keys is None or isinstance(keys, list)):
144            raise FieldError("keys not list")
145        if keys is None and leng == 0:
146            keys = []
147        elif keys is None:
148            keys = [(i*len(codec))//leng for i in range(leng)]
149        if codec == []:
150            keysset = Cutil.tocodec(keys)
151            codec = self.l_to_i(keysset, fast=True)
152        codec = self.l_to_i(codec, fast=fast)
153        Cfield.__init__(self, codec, name, keys, reindex=reindex)
154
155    def __setitem__(self, ind, item):
156        ''' modify values item'''
157        if isinstance(ind, slice):
158            start, stop, step = ind.start or 0, ind.stop or len(self), ind.step or 1
159            idxt = list(iter(range(start, stop, step)))
160            if len(idxt) != len(item):
161                raise FieldError("item length not consistent")
162            self.setlistvalue(item, idxt, extern=True)
163        elif ind < 0 or ind >= len(self):
164            raise FieldError("out of bounds")
165        else: 
166            self.setvalue(ind, item, extern=True)
167
168    def __str__(self):
169        '''return json string format'''
170        return str({self.name: self.l_to_e(self.values)})
171        # return '    ' + self.to_obj(encoded=True, modecodec='full', untyped=False) + '\n'
172
173    @classmethod
174    def merging(cls, listidx, name=None):
175        '''Create a new Field with values are tuples of listidx Field values
176
177        *Parameters*
178
179        - **listidx** : list of Field to be merged.
180        - **name** : string (default : None) - Name of the new Field
181
182        *Returns* : new Field'''
183        if not name:
184            name = str(list({idx.name for idx in listidx}))
185        values = Cutil.transpose([idx.values for idx in listidx])
186        return cls.ntv({name: values})
187
188    @staticmethod
189    def l_to_i(lis, fast=False):
190        ''' converting a list of external values to a list of internal values'''
191        if fast:
192            return lis
193        return [Sfield.s_to_i(val, fast) for val in lis]
194
195    @staticmethod
196    def s_to_i(val, fast=False):
197        '''converting an external value to an internal value'''
198        if fast:
199            return val
200        if val is None or isinstance(val, bool):
201            return json.dumps(val)
202        if isinstance(val, list):
203            return Cutil.tupled(val)
204        if isinstance(val, dict):
205            return json.dumps(val, cls=NtvJsonEncoder)
206        return val
207
208    @staticmethod
209    def n_to_i(ntv_lis, fast=False):
210        ''' converting a NtvList value to an internal value'''
211        if isinstance(ntv_lis, list) and len(ntv_lis) == 0:
212            return []
213        if isinstance(ntv_lis, list) and ntv_lis[0].__class__.__name__ in ('NtvSingle', 'NtvList'):
214            # return [Sfield.n_to_i(ntv.val, fast) for ntv in ntv_lis]
215            return [Sfield.n_to_i(ntv.to_obj(), fast) for ntv in ntv_lis]
216        return Sfield.s_to_i(ntv_lis, fast)
217
218    @staticmethod
219    def l_to_e(lis, fast=False):
220        ''' converting a list of internal values to a list of external values'''
221        if fast:
222            return lis
223        return [Sfield.s_to_e(val) for val in lis]
224
225    @staticmethod
226    def s_to_e(val, fast=False):
227        '''converting an internal value to an external value'''
228        if fast:
229            return val
230        if val in ('null', 'false', 'true'):
231            return json.loads(val)
232        # if val is None or isinstance(val, bool):
233        #    return json.dumps(val)
234        if isinstance(val, tuple):
235            return Cutil.listed(val)
236        if isinstance(val, str) and len(val) > 0 and val[0] == '{':
237            return json.loads(val)
238        return val
239
240    @staticmethod
241    def i_to_n(val):
242        ''' converting an internal value to a NTV value'''
243        return Ntv.obj(Sfield.s_to_e(val))
244
245    @staticmethod
246    def l_to_n(lis):
247        ''' converting a list of internal values to a list of NTV value'''
248        #return Ntv.obj(Sfield.l_to_e(lis))
249        return [Sfield.i_to_n(val) for val in lis]
250
251    @staticmethod
252    def i_to_name(val):
253        ''' return the name of the internal value'''
254        return ''
255
256    @property
257    def cod(self):
258        '''return codec conversion to json value '''
259        return self.l_to_e(self._codec)
260
261    @property
262    def val(self):
263        '''return values conversion to string '''
264        return [self.s_to_e(self._codec[key]) for key in self._keys]
265
266    def append(self, value, unique=True):
267        '''add a new value
268
269        *Parameters*
270
271        - **value** : new object value
272        - **unique** :  boolean (default True) - If False, duplication codec if value is present
273
274        *Returns* : key of value '''
275        return super().append(self.s_to_i(value), unique)
276
277    def isvalue(self, value, extern=True):
278        ''' return True if value is in index values
279
280        *Parameters*
281
282        - **value** : value to check
283        - **extern** : if True, compare value to external representation of self.value,
284        else, internal'''
285        if extern:
286            return value in self.val
287        return super().isvalue(value)
288
289    def keytoval(self, key, extern=True):
290        ''' return the value of a key
291
292        *Parameters*
293
294        - **key** : key to convert into values
295        - **extern** : if True, return string representation else, internal value
296
297        *Returns*
298
299        - **int** : first key finded (None else)'''
300        if extern:
301            return self.s_to_e(super().keytoval(key))
302        return super().keytoval(key)
303
304    def loc(self, value, extern=True):
305        '''return a list of record number with value
306
307        *Parameters*
308
309        - **value** : value to check
310        - **extern** : if True, compare value to external representation of self.value,
311        else, internal
312
313        *Returns*
314
315        - **list of int** : list of record number finded (None else)'''
316        return self.recordfromvalue(value, extern=extern)
317
318    def recordfromvalue(self, value, extern=True):
319        '''return a list of record number with value
320
321        *Parameters*
322
323        - **value** : value to check
324        - **extern** : if True, compare value to external representation of self.value,
325        else, internal
326
327        *Returns*
328
329        - **list of int** : list of record number finded (None else)'''
330
331        if extern:
332            return super().recordfromvalue(self.s_to_i(value))
333        return super().recordfromvalue(value)
334
335    def setcodecvalue(self, oldvalue, newvalue, extern=True):
336        '''update all the oldvalue by newvalue
337
338        *Parameters*
339
340        - **oldvalue** : list of values to replace
341        - **newvalue** : list of new value to apply
342        - **extern** : if True, the newvalue has external representation, else internal
343
344        *Returns* : int - last codec rank updated (-1 if None)'''
345        if extern:
346            # ,nameonly, valueonly)
347            return super().setcodecvalue(self.s_to_i(oldvalue), self.s_to_i(newvalue))
348        return super().setcodecvalue(oldvalue, newvalue)  # , nameonly, valueonly)
349
350    def setcodeclist(self, listcodec, extern=True):
351        '''update codec with listcodec values
352
353        *Parameters*
354
355        - **listcodec** : list of new codec values to apply
356        - **extern** : if True, the newvalue has external representation, else internal
357
358        *Returns* : int - last codec rank updated (-1 if None)'''
359        if extern:
360            super().setcodeclist(self.l_to_i(listcodec))  # , nameonly, valueonly)
361        super().setcodeclist(listcodec)  # , nameonly, valueonly)
362
363    def setvalue(self, ind, value, extern=True):
364        '''update a value at the rank ind (and update codec and keys)
365
366        *Parameters*
367
368        - **ind** : rank of the value
369        - **value** : new value
370        - **extern** : if True, the value has external representation, else internal
371
372        *Returns* : None'''
373        if extern:
374            super().setvalue(ind, self.s_to_i(value))
375        else:
376            super().setvalue(ind, value)
377
378    def setlistvalue(self, listvalue, listind=None, extern=True):
379        '''update the values (and update codec and keys)
380
381        *Parameters*
382
383        - **listvalue** : list - list of new values
384        - **listind** : list(default None) - list of index
385        - **extern** : if True, the value has external representation, else internal
386
387        *Returns* : None'''
388        if extern:
389            super().setlistvalue(self.l_to_i(listvalue), listind=listind)
390        else:
391            super().setlistvalue(listvalue, listind=listind)
392
393    def valtokey(self, value, extern=True):
394        '''convert a value to a key
395
396        *Parameters*
397
398        - **value** : value to convert
399        - **extern** : if True, the value has external representation, else internal
400
401        *Returns*
402
403        - **int** : first key finded (None else)'''
404        if extern:
405            return super().valtokey(self.s_to_i(value))
406        return super().valtokey(value)
407
408
409
410class Nfield(Sfield):
411    ''' Nfield is a child class of SField where values are NTV objects
412
413    The methods defined in this class are conversion methods:
414
415    *converting external value to internal value:*
416
417    - `Nfield.l_to_i`
418    - `Nfield.s_to_i`
419
420    *converting internal value to external value:*
421
422    - `Nfield.l_to_e`
423    - `Nfield.s_to_e`
424
425    *converting internal value / NTV value:*
426
427    - `Nfield.i_to_n`
428    - `Nfield.n_to_i`
429
430    *extract the name of the value:*
431
432    - `Nfield.i_to_name`
433    '''
434
435    def __str__(self):
436        '''return json string format'''
437        return str(self.to_ntv(modecodec='full'))
438        # return '    ' + self.to_obj(encoded=True, modecodec='full', untyped=False) + '\n'
439
440    @staticmethod
441    def l_to_i(lis, fast=False):
442        ''' converting a list of external values to a list of internal values
443
444        *Parameters*
445
446        - **fast**: boolean (default False) - list is created with a list of json values
447        without control'''
448        if fast:
449            return [NtvSingle(val, fast=True) for val in lis]
450        return [Ntv.from_obj(val) for val in lis]
451
452    @staticmethod
453    def s_to_i(val, fast=False):
454        '''converting an external value to an internal value
455
456        *Parameters*
457
458        - **fast**: boolean (default False) - list is created with a list of json values
459        without control'''
460        if fast:
461            return NtvSingle(val, fast=True)
462        return Ntv.from_obj(val)
463
464    @staticmethod
465    def n_to_i(ntv_lis, fast=False):
466        ''' converting a NTV value to an internal value'''
467        return ntv_lis
468
469    @staticmethod
470    def l_to_e(lis, fast=False):
471        ''' converting a list of internal values to a list of external values'''
472        return [ntv.to_obj() for ntv in lis]
473
474    @staticmethod
475    def s_to_e(val, fast=False):
476        '''converting an internal value to an external value'''
477        return val.to_obj()
478
479    @staticmethod
480    def i_to_n(val):
481        ''' converting an internal value to a NTV value'''
482        return val
483
484    @staticmethod
485    def l_to_n(lis):
486        ''' converting a list of internal values to a list of NTV value'''
487        #return Ntv.obj(Nfield.l_to_e(lis))
488        return [Sfield.i_to_n(val) for val in lis]
489    
490    @staticmethod
491    def i_to_name(val):
492        ''' return the name of the internal value'''
493        return val.name