tab-dataset.tab_dataset.field

  1# -*- coding: utf-8 -*-
  2"""
  3The `field` module is part of the `tab-dataset` package.
  4
  5It contains the classes `Sfield` and `Nfield` for Field entities.
  6
  7For more information, see the 
  8[user guide](https://loco-philippe.github.io/tab-dataset/docs/user_guide.html) 
  9or the [github repository](https://github.com/loco-philippe/tab-dataset).
 10"""
 11from copy import copy
 12import json
 13
 14from json_ntv import Ntv, NtvSingle, NtvJsonEncoder
 15
 16from tab_dataset.field_interface import FieldInterface
 17from tab_dataset.cfield import Cfield, FieldError, Cutil
 18
 19DEFAULTINDEX = '$default'
 20
 21
 22class Sfield(FieldInterface, Cfield):
 23    '''
 24    `Sfield` is a child class of `Cfield` where internal value can be different
 25    from external value (list is converted in tuple and dict in json-object).
 26
 27    Attributes are the same as Cfield class
 28
 29    The methods defined in this class are :
 30
 31    *constructor (@classmethod)*
 32
 33    - `Cfield.bol`
 34    - `Cfield.from_ntv`
 35    - `Cfield.ntv`
 36    - `Cfield.like`
 37    - `Sfield.merging`
 38
 39    *conversion (@staticmethod)*
 40
 41    - `Sfield.l_to_i`
 42    - `Sfield.s_to_i`
 43    - `Sfield.l_to_e`
 44    - `Sfield.s_to_e`
 45    - `Sfield.i_to_n`
 46    - `Sfield.n_to_i`
 47    - `Sfield.i_to_name`
 48    - `Cfield.ntv_to_val` (@classmethod)
 49    
 50    *dynamic value (getters @property)*
 51
 52    - `Sfield.val`
 53    - `Sfield.cod`
 54    - `Cfield.hashf`
 55    - `Cfield.to_analysis`
 56    - `Cfield.values`
 57    - `Cfield.codec`
 58    - `Cfield.infos`
 59    - `Cfield.keys`
 60    
 61    *add - update methods*
 62
 63    - `Sfield.append`
 64    - `Sfield.setcodecvalue`
 65    - `Sfield.setcodeclist`
 66    - `Sfield.setlistvalue`
 67    - `Sfield.setvalue`
 68    - `Cfield.add`
 69    - `Cfield.setname`
 70    - `Cfield.set_keys`
 71    - `Cfield.set_codec`
 72    - `Cfield.setkeys`
 73
 74    *transform methods*
 75
 76    - `Cfield.coupling`
 77    - `Cfield.extendkeys`
 78    - `Cfield.full`
 79    - `Cfield.reindex`
 80    - `Cfield.reorder`
 81    - `Cfield.sort`
 82    - `Cfield.tocoupled`
 83    - `Cfield.tostdcodec`
 84    
 85    *getters methods*
 86
 87    - `Sfield.isvalue`
 88    - `Sfield.keytoval`
 89    - `Sfield.loc`
 90    - `Sfield.recordfromvalue`
 91    - `Sfield.valtokey`
 92    - `Cfield.couplinginfos`
 93    - `Cfield.derkeys`
 94    - `Cfield.getduplicates`
 95    - `Cfield.iscrossed`
 96    - `Cfield.iscoupled`
 97    - `Cfield.isderived`
 98    - `Cfield.islinked`
 99    - `Cfield.iskeysfromderkeys`
100    - `Cfield.recordfromkeys`
101
102    *export methods (`observation.field_interface.SfieldInterface`)*
103
104    - `Sfield.json`
105    - `Sfield.to_obj`
106    - `Sfield.to_dict_obj`
107    - `Sfield.to_numpy`
108    - `Sfield.to_pandas`
109    - `Sfield.vlist`
110    - `Sfield.v_name`
111    - `Sfield.vSimple`
112    '''
113
114    def __init__(self, codec=None, name=None, keys=None,
115                 lendefault=0, reindex=False, fast=False):
116        '''
117        Sfield constructor.
118
119        *Parameters*
120
121        - **codec** :  list (default None) - external different values of index (see data model)
122        - **keys** :  list (default None)  - key value of index (see data model)
123        - **name** : string (default None) - name of index (see data model)
124        - **lendefault** : integer (default 0) - default len if no keys is defined
125        - **reindex** : boolean (default True) - if True, default codec is apply
126        - **fast**: boolean (default False) - if True, codec is created without conversion'''
127        if isinstance(codec, Sfield):
128            Cfield.__init__(self, copy(codec._codec),
129                            copy(codec.name), copy(codec._keys))
130            return
131        if codec is None:
132            codec = []
133        if not isinstance(codec, list):
134            codec = [codec]
135        leng = lendefault
136        if codec and len(codec) > 0 and not leng:
137            leng = len(codec)
138        if not keys is None:
139            leng = len(keys)
140        if not name:
141            name = DEFAULTINDEX
142        if not (keys is None or isinstance(keys, list)):
143            raise FieldError("keys not list")
144        if keys is None and leng == 0:
145            keys = []
146        elif keys is None:
147            keys = [(i*len(codec))//leng for i in range(leng)]
148        if codec == []:
149            keysset = Cutil.tocodec(keys)
150            codec = self.l_to_i(keysset, fast=True)
151        codec = self.l_to_i(codec, fast=fast)
152        Cfield.__init__(self, codec, name, keys, reindex=reindex)
153
154    def __setitem__(self, ind, item):
155        ''' modify values item'''
156        if isinstance(ind, slice):
157            start, stop, step = ind.start or 0, ind.stop or len(self), ind.step or 1
158            idxt = list(iter(range(start, stop, step)))
159            if len(idxt) != len(item):
160                raise FieldError("item length not consistent")
161            self.setlistvalue(item, idxt, extern=True)
162        elif ind < 0 or ind >= len(self):
163            raise FieldError("out of bounds")
164        else: 
165            self.setvalue(ind, item, extern=True)
166
167    def __str__(self):
168        '''return json string format'''
169        return str({self.name: self.l_to_e(self.values)})
170        # return '    ' + self.to_obj(encoded=True, modecodec='full', untyped=False) + '\n'
171
172    @classmethod
173    def merging(cls, listidx, name=None):
174        '''Create a new Field with values are tuples of listidx Field values
175
176        *Parameters*
177
178        - **listidx** : list of Field to be merged.
179        - **name** : string (default : None) - Name of the new Field
180
181        *Returns* : new Field'''
182        if not name:
183            name = str(list({idx.name for idx in listidx}))
184        values = Cutil.transpose([idx.values for idx in listidx])
185        return cls.ntv({name: values})
186
187    @staticmethod
188    def l_to_i(lis, fast=False):
189        ''' converting a list of external values to a list of internal values'''
190        if fast:
191            return lis
192        return [Sfield.s_to_i(val, fast) for val in lis]
193
194    @staticmethod
195    def s_to_i(val, fast=False):
196        '''converting an external value to an internal value'''
197        if fast:
198            return val
199        if val is None or isinstance(val, bool):
200            return json.dumps(val)
201        if isinstance(val, list):
202            return Cutil.tupled(val)
203        if isinstance(val, dict):
204            return json.dumps(val, cls=NtvJsonEncoder)
205        return val
206
207    @staticmethod
208    def n_to_i(ntv_lis, fast=False):
209        ''' converting a NtvList value to an internal value'''
210        if isinstance(ntv_lis, list) and len(ntv_lis) == 0:
211            return []
212        if isinstance(ntv_lis, list) and ntv_lis[0].__class__.__name__ in ('NtvSingle', 'NtvList'):
213            # return [Sfield.n_to_i(ntv.val, fast) for ntv in ntv_lis]
214            return [Sfield.n_to_i(ntv.to_obj(), fast) for ntv in ntv_lis]
215        return Sfield.s_to_i(ntv_lis, fast)
216
217    @staticmethod
218    def l_to_e(lis, fast=False):
219        ''' converting a list of internal values to a list of external values'''
220        if fast:
221            return lis
222        return [Sfield.s_to_e(val) for val in lis]
223
224    @staticmethod
225    def s_to_e(val, fast=False):
226        '''converting an internal value to an external value'''
227        if fast:
228            return val
229        if val in ('null', 'false', 'true'):
230            return json.loads(val)
231        # if val is None or isinstance(val, bool):
232        #    return json.dumps(val)
233        if isinstance(val, tuple):
234            return Cutil.listed(val)
235        if isinstance(val, str) and len(val) > 0 and val[0] == '{':
236            return json.loads(val)
237        return val
238
239    @staticmethod
240    def i_to_n(val):
241        ''' converting an internal value to a NTV value'''
242        return Ntv.obj(Sfield.s_to_e(val))
243
244    @staticmethod
245    def i_to_name(val):
246        ''' return the name of the internal value'''
247        return ''
248
249    @property
250    def cod(self):
251        '''return codec conversion to json value '''
252        return self.l_to_e(self._codec)
253
254    @property
255    def val(self):
256        '''return values conversion to string '''
257        return [self.s_to_e(self._codec[key]) for key in self._keys]
258
259    def append(self, value, unique=True):
260        '''add a new value
261
262        *Parameters*
263
264        - **value** : new object value
265        - **unique** :  boolean (default True) - If False, duplication codec if value is present
266
267        *Returns* : key of value '''
268        return super().append(self.s_to_i(value), unique)
269
270    def isvalue(self, value, extern=True):
271        ''' return True if value is in index values
272
273        *Parameters*
274
275        - **value** : value to check
276        - **extern** : if True, compare value to external representation of self.value,
277        else, internal'''
278        if extern:
279            return value in self.val
280        return super().isvalue(value)
281
282    def keytoval(self, key, extern=True):
283        ''' return the value of a key
284
285        *Parameters*
286
287        - **key** : key to convert into values
288        - **extern** : if True, return string representation else, internal value
289
290        *Returns*
291
292        - **int** : first key finded (None else)'''
293        if extern:
294            return self.s_to_e(super().keytoval(key))
295        return super().keytoval(key)
296
297    def loc(self, value, extern=True):
298        '''return a list of record number with value
299
300        *Parameters*
301
302        - **value** : value to check
303        - **extern** : if True, compare value to external representation of self.value,
304        else, internal
305
306        *Returns*
307
308        - **list of int** : list of record number finded (None else)'''
309        return self.recordfromvalue(value, extern=extern)
310
311    def recordfromvalue(self, value, extern=True):
312        '''return a list of record number with value
313
314        *Parameters*
315
316        - **value** : value to check
317        - **extern** : if True, compare value to external representation of self.value,
318        else, internal
319
320        *Returns*
321
322        - **list of int** : list of record number finded (None else)'''
323
324        if extern:
325            return super().recordfromvalue(self.s_to_i(value))
326        return super().recordfromvalue(value)
327
328    def setcodecvalue(self, oldvalue, newvalue, extern=True):
329        '''update all the oldvalue by newvalue
330
331        *Parameters*
332
333        - **oldvalue** : list of values to replace
334        - **newvalue** : list of new value to apply
335        - **extern** : if True, the newvalue has external representation, else internal
336
337        *Returns* : int - last codec rank updated (-1 if None)'''
338        if extern:
339            # ,nameonly, valueonly)
340            return super().setcodecvalue(self.s_to_i(oldvalue), self.s_to_i(newvalue))
341        return super().setcodecvalue(oldvalue, newvalue)  # , nameonly, valueonly)
342
343    def setcodeclist(self, listcodec, extern=True):
344        '''update codec with listcodec values
345
346        *Parameters*
347
348        - **listcodec** : list of new codec values to apply
349        - **extern** : if True, the newvalue has external representation, else internal
350
351        *Returns* : int - last codec rank updated (-1 if None)'''
352        if extern:
353            super().setcodeclist(self.l_to_i(listcodec))  # , nameonly, valueonly)
354        super().setcodeclist(listcodec)  # , nameonly, valueonly)
355
356    def setvalue(self, ind, value, extern=True):
357        '''update a value at the rank ind (and update codec and keys)
358
359        *Parameters*
360
361        - **ind** : rank of the value
362        - **value** : new value
363        - **extern** : if True, the value has external representation, else internal
364
365        *Returns* : None'''
366        if extern:
367            super().setvalue(ind, self.s_to_i(value))
368        else:
369            super().setvalue(ind, value)
370
371    def setlistvalue(self, listvalue, listind=None, extern=True):
372        '''update the values (and update codec and keys)
373
374        *Parameters*
375
376        - **listvalue** : list - list of new values
377        - **listind** : list(default None) - list of index
378        - **extern** : if True, the value has external representation, else internal
379
380        *Returns* : None'''
381        if extern:
382            super().setlistvalue(self.l_to_i(listvalue), listind=listind)
383        else:
384            super().setlistvalue(listvalue, listind=listind)
385
386    def valtokey(self, value, extern=True):
387        '''convert a value to a key
388
389        *Parameters*
390
391        - **value** : value to convert
392        - **extern** : if True, the value has external representation, else internal
393
394        *Returns*
395
396        - **int** : first key finded (None else)'''
397        if extern:
398            return super().valtokey(self.s_to_i(value))
399        return super().valtokey(value)
400
401
402
403class Nfield(Sfield):
404    ''' Nfield is a child class of SField where values are NTV objects
405
406    The methods defined in this class are conversion methods:
407
408    *converting external value to internal value:*
409
410    - `Nfield.l_to_i`
411    - `Nfield.s_to_i`
412
413    *converting internal value to external value:*
414
415    - `Nfield.l_to_e`
416    - `Nfield.s_to_e`
417
418    *converting internal value / NTV value:*
419
420    - `Nfield.i_to_n`
421    - `Nfield.n_to_i`
422
423    *extract the name of the value:*
424
425    - `Nfield.i_to_name`
426    '''
427
428    def __str__(self):
429        '''return json string format'''
430        return str(self.to_ntv(modecodec='full'))
431        # return '    ' + self.to_obj(encoded=True, modecodec='full', untyped=False) + '\n'
432
433    @staticmethod
434    def l_to_i(lis, fast=False):
435        ''' converting a list of external values to a list of internal values
436
437        *Parameters*
438
439        - **fast**: boolean (default False) - list is created with a list of json values
440        without control'''
441        if fast:
442            return [NtvSingle(val, fast=True) for val in lis]
443        return [Ntv.from_obj(val) for val in lis]
444
445    @staticmethod
446    def s_to_i(val, fast=False):
447        '''converting an external value to an internal value
448
449        *Parameters*
450
451        - **fast**: boolean (default False) - list is created with a list of json values
452        without control'''
453        if fast:
454            return NtvSingle(val, fast=True)
455        return Ntv.from_obj(val)
456
457    @staticmethod
458    def n_to_i(ntv_lis, fast=False):
459        ''' converting a NTV value to an internal value'''
460        return ntv_lis
461
462    @staticmethod
463    def l_to_e(lis, fast=False):
464        ''' converting a list of internal values to a list of external values'''
465        return [ntv.to_obj() for ntv in lis]
466
467    @staticmethod
468    def s_to_e(val, fast=False):
469        '''converting an internal value to an external value'''
470        return val.to_obj()
471
472    @staticmethod
473    def i_to_n(val):
474        ''' converting an internal value to a NTV value'''
475        return val
476
477    @staticmethod
478    def i_to_name(val):
479        ''' return the name of the internal value'''
480        return val.name