python.observation.util

  1# -*- coding: utf-8 -*-
  2"""
  3Created on Fri Jul 29 12:48:16 2022
  4
  5@author: philippe@loco-labs.io
  6"""
  7
  8from collections import Counter
  9from itertools import product
 10import datetime
 11import re
 12import numpy as np
 13import math
 14
 15from observation.timeslot import TimeInterval
 16from observation.esconstante import ES, _classval
 17from observation.esvalue_base import ESValue
 18
 19
 20def identity(*args, **kwargs):
 21    '''return the same value as args or kwargs'''
 22    if len(args) > 0:
 23        return args[0]
 24    if len(kwargs) > 0:
 25        return kwargs[list(kwargs.keys())[0]]
 26    return None
 27
 28
 29class util:
 30    ''' common functions for Field and Dataset class'''
 31# %% util
 32    c1 = re.compile('\d+\.?\d*[,\-_ ;:]')
 33    c2 = re.compile('[,\-_ ;:]\d+\.?\d*')
 34
 35    @staticmethod
 36    def canonorder(lenidx):
 37        '''return a list of crossed keys from a list of number of values'''
 38        listrange = [range(lidx) for lidx in lenidx]
 39        return util.transpose(util.list(list(product(*listrange))))
 40
 41    """@staticmethod
 42    def cast(val, dtype=None, string=True, default=None, maxlen=None):
 43        ''' convert val in the type defined by the string dtype'''
 44        typeval = val.__class__.__name__
 45        if dtype == 'name':
 46            if typeval in ES.className and val.name:
 47                name = val.name
 48            else:
 49                name = default
 50            if maxlen:
 51                return name[:maxlen]
 52            return name
 53        if dtype == 'simple':
 54            if typeval in ES.className:
 55                return val.vSimple(string=string)
 56            else:
 57                return val
 58        if dtype == 'json':
 59            if typeval in ES.className:
 60                return val.json()
 61            else:
 62                return val
 63        if dtype == 'obj':
 64            if typeval in ES.className:
 65                return val.to_obj(encoded=False)
 66            else:
 67                return val
 68        if dtype == 'str':
 69            if maxlen:
 70                return str(val)[:maxlen]
 71            else:
 72                return str(val)
 73        if not dtype:
 74            return ESValue._castsimple(val)
 75        if dtype == 'int':
 76            try:
 77                return int(val.lstrip())
 78            except:
 79                return math.nan
 80        if dtype == 'float':
 81            if typeval in ['float', 'int']:
 82                return float(val)
 83            if typeval in ES.className:
 84                return val.to_float()
 85            try:
 86                return float(val.lstrip())
 87            except:
 88                return math.nan
 89        if dtype == 'datetime':
 90            try:
 91                return TimeInterval._dattz(datetime.datetime.fromisoformat(val))
 92            except ValueError:
 93                return datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
 94        if dtype == 'coord':
 95            if util.c1.search(val) and util.c2.search(val):
 96                return [float(util.c1.search(val)[0][:-1]), float(util.c2.search(val)[0][1:])]
 97            else:
 98                return [-1, -1]
 99        if typeval in ES.ESclassName:
100            return val
101        if dtype in ES.typeName:  # tous les types environmental sensing
102            return ESValue.from_obj(val, ES.typeName[dtype])
103        raise utilError("dtype : " + dtype + " inconsistent with data")
104
105    @staticmethod
106    def castobj(lis, classvalue=None):
107        ''' convert a list of values in the ESValue defined by the string classvalue'''
108        if not lis:
109            return lis
110        return [util.castval(val, classvalue) for val in lis]
111
112    @staticmethod
113    def castval(val, classvalue=None):
114        ''' convert a value in the ESValue defined by the string classvalue'''
115        classn, name, value = ESValue._decodevalue(val)
116        if classn:
117            classvalue = classn
118        if classvalue is None:
119            dtype = None
120        else:
121            dtype = ES.valname[classvalue]
122
123        if dtype is None:
124            return util.cast(val)
125        if classvalue in ES.ESclassName:  # ESValue.cast()
126            return util.cast(ESValue.from_obj(val, classvalue), dtype)
127        if classvalue == ES.ES_clsName:   # cast auto ESValue
128            return _classval()[ESValue.valClassName(val)].from_obj(val)
129            # return _classval()[ESValue.valClassName(val)](val)
130        if classvalue in ES.className:
131            return _classval()[classvalue].obj(value)
132            # return _classval()[classvalue](value)
133        return val"""
134
135    @staticmethod
136    def couplinginfos(l1, l2):
137        '''return a dict with the coupling info between two list'''
138        if not l1 or not l2:
139            return {'dist': 0, 'rateder': 0, 'disttomin': 0, 'disttomax': 0,
140                    'distmin': 0, 'distmax': 0, 'diff': 0, 'typecoupl': 'null'}
141        ls = len(util.tocodec(l1))
142        lo = len(util.tocodec(l2))
143        x0 = max(ls, lo)
144        x1 = ls * lo
145        diff = abs(ls - lo)
146        if min(ls, lo) == 1:
147            if ls == 1:
148                typec = 'derived'
149            else:
150                typec = 'derive'
151            return {'dist': x0, 'rateder': 0, 'disttomin': 0, 'disttomax': 0,
152                    'distmin': x0, 'distmax': x1, 'diff': diff, 'typecoupl': typec}
153        x = len(util.tocodec([tuple((v1, v2)) for v1, v2 in zip(l1, l2)]))
154        dic = {'dist': x, 'rateder': (x - x0) / (x1 - x0),
155               'disttomin': x - x0,  'disttomax': x1 - x,
156               'distmin': x0, 'distmax': x1, 'diff': diff}
157        if dic['rateder'] == 0 and dic['diff'] == 0:
158            dic['typecoupl'] = 'coupled'
159        elif dic['rateder'] == 0 and ls < lo:
160            dic['typecoupl'] = 'derived'
161        elif dic['rateder'] == 0 and ls > lo:
162            dic['typecoupl'] = 'derive'
163        elif dic['rateder'] == 1:
164            dic['typecoupl'] = 'crossed'
165        elif ls < lo:
166            dic['typecoupl'] = 'linked'
167        else:
168            dic['typecoupl'] = 'link'
169        return dic
170
171    @staticmethod
172    def filter(func, lis, res, *args, **kwargs):
173        '''apply "func" to each value of "lis" and tests if equals "res".
174        Return the list of index with True result.'''
175        lisf = util.funclist(lis, func, *args, **kwargs)
176        return [i for i in range(len(lis)) if lisf[i] == res]
177
178    @staticmethod
179    def funclist(value, func, *args, **kwargs):
180        '''return the function func applied to the object value with parameters args and kwargs'''
181        if func in (None, []):
182            return value
183        lis = []
184        if not (isinstance(value, list) or value.__class__.__name__ in ['Field', 'Dataset', 'Observation']):
185            listval = [value]
186        else:
187            listval = value
188        for val in listval:
189            try:
190                lis.append(val.func(*args, **kwargs))
191            except:
192                try:
193                    lis.append(func(val, *args, **kwargs))
194                except:
195                    try:
196                        lis.append(listval.func(val, *args, **kwargs))
197                    except:
198                        try:
199                            lis.append(func(listval, val, *args, **kwargs))
200                        except:
201                            raise utilError("unable to apply func")
202        if len(lis) == 1:
203            return lis[0]
204        return lis
205
206    @staticmethod
207    def hash(listval):
208        ''' return sum of hash values in the list'''
209        # return sum([hash(i) for i in listval])
210        return hash(tuple(listval))
211
212    @staticmethod
213    def idxfull(setidx):
214        '''return additional keys for each index in the setidx list to have crossed setidx'''
215        setcodec = [set(idx.keys) for idx in setidx]
216        lenfull = util.mul([len(codec) for codec in setcodec])
217        if lenfull <= len(setidx[0]):
218            return []
219        complet = Counter(list(product(*setcodec)))
220        complet.subtract(
221            Counter(util.tuple(util.transpose([idx.keys for idx in setidx]))))
222        keysadd = util.transpose(util.list(list(complet.elements())))
223        if not keysadd:
224            return []
225        return keysadd
226
227    @staticmethod
228    def isEqual(value, tovalue=None, **kwargs):
229        ''' return True if value and tovalue are equal'''
230        return value.__class__.__name__ == tovalue.__class__.__name__ and \
231            value == tovalue
232
233    @staticmethod
234    def isNotEqual(value, tovalue=None, **kwargs):
235        ''' return True if value and tovalue are not equal'''
236        return value.__class__.__name__ != tovalue.__class__.__name__ or \
237            value != tovalue
238
239    @staticmethod
240    def isNotNull(value, nullvalue=None, **kwargs):
241        '''return boolean. True if value' is not a NullValue'''
242        if value.__class__.__name__ in ES.valname:
243            return value != value.__class__(nullvalue)
244        return not value is None
245
246    @staticmethod
247    def idxlink(ref, l2):
248        ''' return a dict for each different tuple (ref value, l2 value)'''
249        return dict(set(zip(ref, l2)))
250        #lis = set(util.tuple(util.transpose([ref, l2])))
251        # if not len(lis) == len(set(ref)):
252        #    return {}
253        # return dict(lis)
254
255    @staticmethod
256    def json(val, **option):
257        datecast = True
258        if 'datetime' in option:
259            datecast = option['datetime']
260        val = ESValue.uncastsimple(val, datecast)
261        if isinstance(val, (str, int, float, bool, list, dict, type(None), bytes, datetime.datetime)):
262            return val
263        if option['simpleval']:
264            return val.json(**option)
265        if val.__class__.__name__ in ES.ESclassName:  # ESValue
266            if not option['typevalue']:
267                return val.json(**option)
268            else:
269                return {ES.valname[val.__class__.__name__]: val.json(**option)}
270        if val.__class__.__name__ == 'Dataset':
271            return {ES.ili_valName: val.json(**option)}
272        if val.__class__.__name__ == 'Field':
273            return {ES.iin_valName: val.json(**option)}
274        if val.__class__.__name__ == 'Observation':
275            return {ES.obs_valName: val.to_obj(**option)}
276
277    @staticmethod
278    def list(tuplelists):
279        '''transform a list of tuples in a list of lists'''
280        return list(map(list, tuplelists))
281
282    @staticmethod
283    def listed(idx):
284        '''transform a tuple of tuple in a list of list'''
285        return [val if not isinstance(val, tuple) else util.listed(val) for val in idx]
286
287    @staticmethod
288    def mul(values):
289        '''return the product of values in a list or tuple (math.prod)'''
290        mul = 1
291        for val in values:
292            mul *= val
293        return mul
294
295    @staticmethod
296    def pparent(row, infos):
297        '''return field 'pparent' '''
298        if row < 0:
299            return row
300        field = infos[row]
301        # if field['pparent'] != 0:
302        if field['pparent'] != -2:
303            return field['pparent']
304        if field['cat'] == 'primary':
305            field['pparent'] = field['num']
306        elif field['cat'] == 'unique':
307            field['pparent'] = -1
308        else:
309            field['pparent'] = util.pparent(field['parent'], infos)
310        return field['pparent']
311
312    @staticmethod
313    def pparent2(row, infos):
314        '''return field 'pparent' '''
315        if row < 0:
316            return row
317        field = infos[row]
318        # if field['pparent'] != 0:
319        if field['pparent'] != -2:
320            return field['pparent']
321        if field['cat'] == 'primary':
322            field['pparent'] = field['num']
323        elif field['cat'] == 'unique':
324            field['pparent'] = -1
325        else:
326            field['pparent'] = util.pparent2(field['parent'], infos)
327        return field['pparent']
328
329    @staticmethod
330    def reindex(oldkeys, oldcodec, newcodec):
331        '''new keys with new order of codec'''
332        dic = {newcodec[i]: i for i in range(len(newcodec))}
333        return [dic[oldcodec[key]] for key in oldkeys]
334
335    @staticmethod
336    def reorder(values, sort=None):
337        '''return a new values list following the order define by sort'''
338        if not sort:
339            return values
340        return [values[ind] for ind in sort]
341
342    @staticmethod
343    def resetidx(values):
344        '''return codec and keys from a list of values'''
345        codec = util.tocodec(values)
346        return (codec, util.tokeys(values, codec))
347
348    @staticmethod
349    def str(listvalues):
350        '''return a list with values in the str format'''
351        return [str(val) for val in listvalues]
352
353    @staticmethod
354    def tokeys(values, codec=None):
355        ''' return a list of keys from a list of values'''
356        if not codec:
357            codec = util.tocodec(values)
358        dic = {codec[i]: i for i in range(len(codec))}  # !!!!long
359        keys = [dic[val] for val in values]    # hyper long
360        return keys
361
362    @staticmethod
363    def tovalues(keys, codec):
364        '''return a list of values from a list of keys and a list of codec values'''
365        return [codec[key] for key in keys]
366
367    @staticmethod
368    def tonumpy(valuelist, func=None, **kwargs):
369        '''return a Numpy Array from a list of values converted by func'''
370        if func is None:
371            func = identity
372        if func == 'index':
373            return np.array(list(range(len(valuelist))))
374        valList = util.funclist(valuelist, func, **kwargs)
375        if isinstance(valList[0], str):
376            try:
377                datetime.datetime.fromisoformat(valList[0])
378            except:
379                return np.array(valList)
380            return np.array(valList, dtype=np.datetime64)
381        if isinstance(valList[0], datetime.datetime):
382            return np.array(valList, dtype=np.datetime64)
383        return np.array(valList)
384
385    @staticmethod
386    def tocodec(values, keys=None):
387        '''extract a list of unique values'''
388        if not keys:
389            #return list(set(values))
390            return list(dict.fromkeys(values))
391        ind, codec = zip(*sorted(set(zip(keys, values))))
392        return list(codec)
393
394    @staticmethod
395    def transpose(idxlist):
396        '''exchange row/column in a list of list'''
397        if not isinstance(idxlist, list):
398            raise utilError('index not transposable')
399        if not idxlist:
400            return []
401        size = min([len(ix) for ix in idxlist])
402        return [[ix[ind] for ix in idxlist] for ind in range(size)]
403
404    @staticmethod
405    def tuple(idx):
406        '''transform a list of list in a list of tuple'''
407        return [val if not isinstance(val, list) else tuple(val) for val in idx]
408
409    @staticmethod
410    def tupled(idx):
411        '''transform a list of list in a tuple of tuple'''
412        return tuple([val if not isinstance(val, list) else util.tupled(val) for val in idx])
413
414class utilError(Exception):
415    ''' util Exception'''
416    # pass