python.observation.esobservation

  1# -*- coding: utf-8 -*-
  2"""
  3Created on Tue Aug  3 23:40:06 2021
  4
  5@author: philippe@loco-labs.io
  6
  7An `Observation` is an object representing a set of information having
  8spatial and temporal characteristics associated with measurable or observable
  9 properties.
 10
 11The `Observation` Object is built around three main bricks :
 12
 13- Dataset Object which deal with indexing,
 14- ESValue Object which integrate the specificities of environmental data,
 15- Tools dedicated to particular domains
 16([Shapely](https://shapely.readthedocs.io/en/stable/manual.html)
 17for location, TimeSlot for Datation)
 18
 19The `python.observation.esobservation` module contains the `Observation` class.
 20
 21Documentation is available in other pages :
 22
 23- The concept of 'observation' is describe in
 24[this page](https://github.com/loco-philippe/Environmental-Sensing/wiki/Observation).
 25- The concept of 'indexed list' is describe in
 26[this page](https://github.com/loco-philippe/Environmental-Sensing/wiki/Indexed-list).
 27- The non-regression test are at [this page]
 28(https://github.com/loco-philippe/Environmental-Sensing/blob/main/python/Tests/test_obs.py)
 29- The [examples]
 30(https://github.com/loco-philippe/Environmental-Sensing/tree/main/python/Examples/Observation)
 31"""
 32import datetime
 33import json
 34from copy import copy
 35import folium
 36import cbor2
 37
 38from observation.dataset import Dataset
 39from observation.util import util
 40from observation.field_interface import FieldEncoder, CborDecoder
 41from observation.esconstante import ES
 42from observation.esvalue import LocationValue, DatationValue, PropertyValue, ExternValue
 43from observation.esvalue_base import ESValue, ESValueEncoder
 44from observation.dataset_analysis import Analysis
 45
 46
 47class Observation(Dataset):
 48    """
 49    An `Observation` is derived from `observation.Dataset` object.
 50
 51    *Additional attributes (for @property see methods)* :
 52
 53    - **name** : textual description
 54    - **param** : namedValue dictionnary (external data)
 55
 56    The methods defined in this class (included inherited) are :
 57
 58    *constructor (@classmethod))*
 59
 60    - `Observation.dic`
 61    - `Observation.std`
 62    - `python.observation.dataset.Dataset.obj`
 63    - `Observation.from_obj`
 64    - `python.observation.dataset.Dataset.from_file`
 65
 66    *dynamic value (getters @property)*
 67
 68    - `Observation.bounds`
 69    - `Observation.id`
 70    - `Observation.jsonFeature`
 71    - `Observation.setLocation`
 72    - `Observation.setDatation`
 73    - `Observation.setProperty`
 74    - `Observation.setResult`
 75
 76    *dynamic value inherited (getters @property)*
 77
 78    - `python.observation.dataset.Dataset.extidx`
 79    - `python.observation.dataset.Dataset.extidxext`
 80    - `python.observation.dataset.Dataset.idxname`
 81    - `python.observation.dataset.Dataset.idxlen`
 82    - `python.observation.dataset.Dataset.iidx`
 83    - `python.observation.dataset.Dataset.keys`
 84    - `python.observation.dataset.Dataset.lenindex`
 85    - `python.observation.dataset.Dataset.lenidx`
 86    - `python.observation.dataset.Dataset.lidx`
 87    - `python.observation.dataset.Dataset.lidxrow`
 88    - `python.observation.dataset.Dataset.lvar`
 89    - `python.observation.dataset.Dataset.lvarrow`
 90    - `python.observation.dataset.Dataset.lname`
 91    - `python.observation.dataset.Dataset.lunicname`
 92    - `python.observation.dataset.Dataset.lunicrow`
 93    - `python.observation.dataset.Dataset.setidx`
 94
 95    *global value (getters @property)*
 96
 97    - `python.observation.dataset.Dataset.complete`
 98    - `python.observation.dataset.Dataset.consistent`
 99    - `python.observation.dataset.Dataset.dimension`
100    - `python.observation.dataset.Dataset.lencomplete`
101    - `python.observation.dataset.Dataset.primary`
102    - `python.observation.dataset.Dataset.zip`
103
104    *selecting - infos methods*
105
106    - `python.observation.dataset.Dataset.couplingmatrix`
107    - `python.observation.dataset.Dataset.idxrecord`
108    - `python.observation.dataset.Dataset.indexinfos`
109    - `python.observation.dataset.Dataset.indicator`
110    - `python.observation.dataset.Dataset.iscanonorder`
111    - `python.observation.dataset.Dataset.isinrecord`
112    - `python.observation.dataset.Dataset.keytoval`
113    - `python.observation.dataset.Dataset.loc`
114    - `python.observation.dataset.Dataset.nindex`
115    - `python.observation.dataset.Dataset.record`
116    - `python.observation.dataset.Dataset.recidx`
117    - `python.observation.dataset.Dataset.recvar`
118    - `python.observation.dataset.Dataset.valtokey`
119
120    *add - update methods*
121
122    - `python.observation.dataset.Dataset.add`
123    - `python.observation.dataset.Dataset.addindex`
124    - `python.observation.dataset.Dataset.append`
125    - `Observation.appendObs`
126    - `python.observation.dataset.Dataset.delindex`
127    - `python.observation.dataset.Dataset.delrecord`
128    - `python.observation.dataset.Dataset.renameindex`
129    - `python.observation.dataset.Dataset.setname`
130    - `python.observation.dataset.Dataset.updateindex`
131
132    *structure management - methods*
133
134    - `python.observation.dataset.Dataset.applyfilter`
135    - `python.observation.dataset.Dataset.coupling`
136    - `python.observation.dataset.Dataset.full`
137    - `python.observation.dataset.Dataset.getduplicates`
138    - `python.observation.dataset.Dataset.merge`
139    - `python.observation.dataset.Dataset.reindex`
140    - `python.observation.dataset.Dataset.reorder`
141    - `python.observation.dataset.Dataset.setfilter`
142    - `python.observation.dataset.Dataset.sort`
143    - `python.observation.dataset.Dataset.swapindex`
144    - `python.observation.dataset.Dataset.setcanonorder`
145    - `python.observation.dataset.Dataset.tostdcodec`
146
147    *exports methods*
148
149    - `Observation.choropleth`
150    - `python.observation.dataset.Dataset.json`
151    - `python.observation.dataset.Dataset.plot`
152    - `python.observation.dataset.Dataset.to_csv`
153    - `python.observation.dataset.Dataset.to_file`
154    - `Observation.to_obj`
155    - `Observation.to_xarray`
156    - `python.observation.dataset.Dataset.to_dataframe`
157    - `python.observation.dataset.Dataset.view`
158    - `python.observation.dataset.Dataset.vlist`
159    - `python.observation.dataset.Dataset.voxel`
160    """
161
162# %% constructor
163    def __init__(self, listidx=None, name=None, param=None, reindex=True):
164        '''Observation constructor
165
166        *Parameters*
167
168        - **listidx**  : object (default None) - list of Field data or Dataset or Observation
169        - **name**     : string (default None) - Obs name
170        - **param**    : dict (default None) - Dict with parameter data or user's data'''
171
172        if isinstance(listidx, Observation):
173            self.lindex = [copy(idx) for idx in listidx.lindex]
174            if not listidx.param is None:
175                self.param = dict(listidx.param.items())
176            else:
177                self.param = param
178            self.name = listidx.name
179            self.analysis = Analysis(self)
180            return
181
182        if isinstance(listidx, Dataset):
183            self.lindex = [copy(idx) for idx in listidx.lindex]
184            self.param = param
185            self.name = name
186            self.analysis = Analysis(self)
187            return
188
189        if not listidx:
190            Dataset.__init__(self)
191        else:
192            Dataset.__init__(self, listidx=listidx, reindex=reindex)
193        self.name = name
194        self.param = param
195        return
196
197    @classmethod
198    def dic(cls, idxdic=None, typevalue=ES.def_clsName, name=None, param=None):
199        '''
200        Observation constructor (external dictionnary).
201
202        *Parameters*
203
204        - **idxdic** : dict (default None) - dict of Field element (Field name :
205        list of Field values)
206        - **typevalue** : str (default ES.def_clsName) - default value class (None or NamedValue)
207        - **var** :  int (default None) - row of the variable
208        - **name**     : string (default None) - Observation name
209        - **param**    : dict (default None) - Dict with parameter data or user's data'''
210        listidx = Dataset.dic(idxdic, typevalue=typevalue)
211        return cls(listidx=listidx, name=name, param=param)
212
213    @classmethod
214    def std(cls, result=None, datation=None, location=None, property=None,
215            name=None, param=None, typevalue=ES.def_clsName):
216        '''
217        Generate an Observation Object with standard indexes
218
219        *Parameters*
220
221        - **datation** : compatible Field (default None) - index for DatationValue
222        - **location** : compatible Field (default None) - index for LocationValue
223        - **property** : compatible Field (default None) - index for PropertyValue
224        - **result  ** : compatible Field (default None) - index for Variable(NamedValue)
225        - **name**     : string (default None) - Observation name
226        - **param**    : dict (default None) - Dict with parameter data or user's data'''
227        idxdic = {}
228        length = 0
229        std_val = (result, datation, location, property)
230        es_val = (ES.res_classES, ES.dat_classES,
231                  ES.loc_classES, ES.prp_classES)
232        for std, esv in zip(std_val, es_val):
233            value = []
234            if not std is None and isinstance(std, list):
235                value = std
236            elif not std is None and not isinstance(std, list):
237                value = [std]
238            length = max(length, len(value))
239            idxdic[esv] = value
240        for item in idxdic.items():
241            if len(item[1]) == 1:
242                idxdic[item[0]] = item[1] * length
243        return cls.dic(idxdic=idxdic, typevalue=typevalue, name=name, param=param)
244
245    @classmethod
246    def from_obj(cls, bs=None, reindex=True, context=True):
247        '''
248        Generate an Observation Object from a bytes, string or dic value
249
250        *Parameters*
251
252        - **bs** : bytes, string or dict data to convert
253        - **reindex** : boolean (default True) - if True, default codec for each Field
254        - **context** : boolean (default True) - if False, only codec and keys are included'''
255        if not bs:
256            bs = {}
257        if isinstance(bs, bytes):
258            dic = cbor2.loads(bs)
259        elif isinstance(bs, str):
260            dic = json.loads(bs, object_hook=CborDecoder().codecbor)
261        elif isinstance(bs, dict):
262            dic = bs
263        else:
264            raise ObsError("the type of parameter is not available")
265
266        param = None
267        if ES.param in dic:
268            param = dic[ES.param]
269        if param and not isinstance(param, dict):
270            raise ObsError('param is not a dict')
271
272        name = None
273        if ES.name in dic:
274            name = dic[ES.name]
275        if name and not isinstance(name, str):
276            raise ObsError('name is not a str')
277
278        data = None
279        if ES.data in dic:
280            data = dic[ES.data]
281        if data and not isinstance(data, (list, dict)):
282            raise ObsError('data is not a list and not a dict')
283
284        return cls(listidx=Dataset.obj(data, reindex=reindex, context=context),
285                   name=name, param=param)
286
287# %% special
288    def __copy__(self):
289        ''' Copy all the data '''
290        return Observation(self)
291
292    def __str__(self):
293        '''return string format'''
294        stro = ''
295        if self.name:
296            stro = ES.name + ': ' + self.name + '\n'
297        stri = Dataset.__str__(self)
298        if not stri == '':
299            stro += ES.data + ':\n' + stri
300        if self.param:
301            stro += ES.param + ':\n    ' + json.dumps(self.param) + '\n'
302        return stro
303
304    def __hash__(self):
305        '''return sum of all hash(Field)'''
306        return hash(json.dumps(self.param)) + hash(self.name) + Dataset.__hash__(self)
307
308# %% properties
309    @property
310    def bounds(self):
311        '''
312        **list of `observation.esvalue` (@property)** : `observation.esvalue`
313        bounding box for each axis.'''
314        bound = [None, None, None]
315        if self.setDatation:
316            bound[0] = ESValue.boundingBox(self.setDatation).bounds
317        if self.setLocation:
318            bound[1] = ESValue.boundingBox(self.setLocation).bounds
319        if self.setProperty:
320            bound[2] = ESValue.boundingBox(self.setProperty).bounds
321        return bound
322
323    @property
324    def __geo_interface__(self):
325        '''**dict (@property)** : return the union of Location geometry (see shapely)'''
326        codecgeo = self.nindex('location').codec
327        if len(codecgeo) == 0:
328            return ""
329        if len(codecgeo) == 1:
330            return codecgeo[0].value.__geo_interface__
331        collec = codecgeo[0].value
332        for loc in codecgeo[1:]:
333            collec = collec.union(loc.value)
334        return collec.__geo_interface__
335
336    @property
337    def id(self):
338        '''**integer (@property)** : hash value (unique)'''
339        return hash(self)
340
341    @property
342    def jsonFeature(self):
343        '''**string (@property)** : "FeatureCollection" with Location geometry'''
344        if self.setLocation:
345            geo = self.__geo_interface__
346            if geo['type'][:5] == 'Multi':
347                typ = geo['type'][5:]
348                lis = [{"type":  typ, "coordinates": geo['coordinates'][i]}
349                       for i in range(len(geo['coordinates']))]
350            elif geo['type'] in ['Point', 'Polygon']:
351                lis = [geo]
352            elif geo['type'] == 'GeometryCollection':
353                lis = geo['geometries']
354            fea = [{"type": "Feature", "id": i, "geometry": lis[i]}
355                   for i in range(len(lis))]
356            return json.dumps({"type": "FeatureCollection", "features": fea},
357                              cls=ESValueEncoder)
358        return ''
359
360    @property
361    def setDatation(self):
362        '''**list (@property)** : list of codec values in the datation index'''
363        if self.nindex(ES.dat_classES):
364            return self.nindex(ES.dat_classES).codec
365        return None
366
367    @property
368    def setLocation(self):
369        '''**list (@property)** : list of codec values in the location index'''
370        if self.nindex(ES.loc_classES):
371            return self.nindex(ES.loc_classES).codec
372        return None
373
374    @property
375    def setProperty(self):
376        '''**list (@property)** : list of codec values in the property index'''
377        if self.nindex(ES.prp_classES):
378            return self.nindex(ES.prp_classES).codec
379        return None
380
381    @property
382    def setResult(self):
383        '''
384        **list (@property)** : list of codec values in the result index'''
385        if self.nindex(ES.res_classES):
386            return self.nindex(ES.res_classES).codec
387        return None
388
389# %% methods
390    def appendObs(self, obs, unique=False, fillvalue='-'):
391        '''
392        Add an `Observation` as a new Result `observation.esvalue` with bounding
393        box for the Index `observation.esvalue`
394
395        *Parameters*
396
397        - **obs** : Observation object
398        - **fillvalue** : object value used for default value
399
400        *Returns*
401
402        - **int** : last index in the `Observation`'''
403        lname = self.lname
404        record = [fillvalue] * len(lname)
405        if ES.dat_classES in lname:
406            record[lname.index(ES.dat_classES)] = DatationValue.Box(
407                obs.bounds[0])
408        if ES.loc_classES in lname:
409            record[lname.index(ES.loc_classES)] = LocationValue.Box(
410                obs.bounds[1])
411        if ES.prp_classES in lname:
412            record[lname.index(ES.prp_classES)] = PropertyValue.Box(
413                obs.bounds[2])
414        if ES.res_classES in lname:
415            record[lname.index(ES.res_classES)] = ExternValue(obs)
416        return self.append(record, unique=unique)
417
418    def choropleth(self, name="choropleth", line=True):
419        '''
420        Display `Observation` on a folium.Map (only with dimension=1)
421
422        - **name** : String, optionnal (default 'choropleth') - Name of the choropleth
423        - **line** : Boolean, optionnal (default True) - Line between recods if True
424
425        *Returns* : None'''
426        primary = self.primary
427        if self.dimension == 1:
428            mapf = folium.Map(
429                location=self.setLocation[0].coorInv, zoom_start=6)
430            folium.Choropleth(
431                geo_data=self.jsonFeature,
432                name=self.name,
433                data=self.to_xarray(
434                    numeric=True, coord=True).to_dataframe(name='obs'),
435                key_on="feature.id",
436                columns=[self.idxname[primary[0]] + '_row', 'obs'],
437                fill_color="OrRd",
438                fill_opacity=0.7,
439                line_opacity=0.4,
440                line_weight=2,
441                legend_name=name
442            ).add_to(mapf)
443            if line:
444                folium.PolyLine(
445                    util.funclist(self.nindex('location'),
446                                  LocationValue.vPointInv)
447                ).add_to(mapf)
448            folium.LayerControl().add_to(mapf)
449            return mapf
450        return None
451
452    def to_obj(self, **kwargs):
453        '''Return a formatted object (json string, cbor bytes or json dict).
454
455        *Parameters (kwargs)*
456
457        - **encoded** : boolean (default False) - choice for return format
458        (string/bytes if True, dict else)
459        - **encode_format**  : string (default 'json')- choice for return format (json, cbor)
460        - **codif** : dict (default ES.codeb). Numerical value for string in CBOR encoder
461        - **modecodec** : string (default 'optimize') - if 'full', each index is with a full codec
462        if 'default' each index has keys, if 'optimize' keys are optimized,
463        if 'dict' dict format is used, if 'nokeys' keys are absent, if 'ndjson' 
464        a list of ndjson elements is created
465        - **name** : boolean (default False) - if False, default index name are not included
466        - **fullvar** : boolean (default True) - if True and modecodec='optimize,
467        variable index is with a full codec
468        - **geojson** : boolean (default False) - geojson for LocationValue if True
469
470        - **json_param**     : Boolean - include Obs Param
471        - **json_info**      : Boolean - include all infos
472        - **json_info_detail**: Boolean - include the other infos
473
474        *Returns* : string, bytes or dict'''
475        option = {'modecodec': 'optimize', 'encoded': False,
476                  'encode_format': 'json', 'codif': ES.codeb, 'name': False,
477                  'json_param': False, 'json_info': False, 'json_info_detail': False,
478                  'geojson': False, 'fullvar': True} | kwargs
479        if option['modecodec'] == 'ndjson':
480            lisobs = {ES.id: self.id}
481            if self.param:
482                lisobs[ES.param] = self.param
483            if self.name:
484                lisobs[ES.name] = self.name
485            return [lisobs] + Dataset.to_obj(self, modecodec='ndjson', id=self.id)
486        option2 = option | {'encoded': False, 'encode_format': 'json'}
487        dic = {ES.type: ES.obs_classES}
488
489        if self.name:
490            dic[ES.obs_name] = self.name
491        if self.param:
492            dic[ES.obs_param] = self.param
493        dic[ES.obs_data] = Dataset.to_obj(self, **option2)
494        if option["json_param"] and self.param:
495            dic[ES.obs_param] = self.param
496        dic |= self._info(**option)
497        if option['codif'] and option['encode_format'] != 'cbor':
498            js2 = {}
499            for key, val in dic.items():
500                if key in option['codif']:
501                    js2[option['codif'][key]] = val
502                else:
503                    js2[key] = val
504        else:
505            js2 = dic
506
507        if option['encoded'] and option['encode_format'] == 'json':
508            return json.dumps(js2, cls=FieldEncoder)
509        if option['encoded'] and option['encode_format'] == 'cbor':
510            return cbor2.dumps(js2, datetime_as_timestamp=True,
511                               timezone=datetime.timezone.utc, canonical=True)
512        return dic
513
514    def to_xarray(self, info=False, idxname=None, varname=None, fillvalue='?',
515                  fillextern=True, lisfunc=None, numeric=False, npdtype=None,
516                  **kwargs):
517        '''
518        Complete the Observation and generate a Xarray DataArray with the dimension define by idx.
519
520        *Parameters*
521
522        - **info** : boolean (default False) - if True, add _dict attributes to attrs Xarray
523        - **idxname** : list (default none) - list of idx to be completed. If None,
524        self.primary is used.
525        - **varname** : string (default none) - Name of the variable to use. If None,
526        first lvarname is used.
527        - **fillvalue** : object (default '?') - value used for the new extval
528        - **fillextern** : boolean(default True) - if True, fillvalue is converted to typevalue
529        - **lisfunc** : function (default none) - list of function to apply to indexes before export
530        - **numeric** : Boolean (default False) - Generate a numeric DataArray.Values.
531        - **npdtype** : string (default None) - numpy dtype for the DataArray ('object' if None)
532        - **kwargs** : parameter for lisfunc
533
534        *Returns* : DataArray '''
535        return Dataset.to_xarray(self, info=info, idxname=idxname, varname=varname,
536                               fillvalue=fillvalue, fillextern=fillextern,
537                               lisfunc=lisfunc, name=self.name, numeric=numeric,
538                               npdtype=npdtype, attrs=self.param, **kwargs)
539# %% internal
540
541    def _info(self, **kwargs):
542        ''' Create json dict with info datas
543
544        *Parameters*
545
546        - **json_info** : boolean (default False) - if True, add main information
547        about Observation and Field
548        - **json_info_detail** : boolean (default False) - if True, add complemantary
549        information about Field
550        '''
551        option = {"json_info": False, "json_info_detail": False} | kwargs
552        dcobs = {}
553        dcindex = {}
554        if not option['json_info']:
555            return dcobs
556        dcobs[ES.name] = self.name
557        dcobs[ES.id] = self.id
558        dcobs[ES.length] = len(self)
559        dcobs[ES.lenindex] = self.lenindex
560        dcobs[ES.complete] = self.complete
561        dcobs[ES.dimension] = self.dimension
562        if option['json_info_detail']:
563            infos = self.indexinfos()
564        for ind, idx in enumerate(self.lidx):
565            dcidx = {}
566            dcidx[ES.num] = ind
567            dcidx[ES.typevalue] = idx.typevalue
568            dcidx[ES.lencodec] = len(idx.codec)
569            dcidx[ES.box] = Observation._info_box(idx, **option)
570            if option['json_info_detail']:
571                dcidx |= infos[ind]
572            dcindex[idx.name] = dcidx
573        return {ES.information: {ES.observation: dcobs, ES.index: dcindex}}
574
575    @staticmethod
576    def _info_box(idx, **option):
577        ''' return box informations's'''
578        if idx.typevalue == ES.dat_clsName:
579            return DatationValue.boundingBox(idx.codec)
580        if idx.typevalue == ES.loc_clsName and not option["geojson"]:
581            return LocationValue.boundingBox(idx.codec)
582        if idx.typevalue == ES.loc_clsName and option["geojson"]:
583            return LocationValue.Box(LocationValue.boundingBox(idx.codec)).__geo_interface__
584        if idx.typevalue == ES.prp_clsName:
585            return PropertyValue.boundingBox(idx.codec)
586        return None
587
588
589class ObsError(Exception):
590    '''Observation exception'''