ntv-numpy.ntv_numpy.xdataset

  1# -*- coding: utf-8 -*-
  2"""
  3@author: Philippe@loco-labs.io
  4
  5The `xdataset` module is part of the `ntv-numpy.ntv_numpy` package ([specification document](
  6https://loco-philippe.github.io/ES/JSON%20semantic%20format%20(JSON-NTV).htm)).
  7
  8It contains the classes `Xdataset`, `XdatasetInterface`, `XdatasetCategory` for
  9the multidimensional dataset.
 10
 11For more information, see the
 12[user guide](https://loco-philippe.github.io/ntv-numpy/docs/user_guide.html)
 13 or the [github repository](https://github.com/loco-philippe/ntv-numpy).
 14"""
 15from abc import ABC, abstractmethod
 16import json
 17import pprint
 18from json_ntv import Ntv
 19from ntv_numpy.ndarray import Nutil
 20from ntv_numpy.xndarray import Xndarray
 21from ntv_numpy.xconnector import XarrayConnec, ScippConnec, AstropyNDDataConnec
 22from ntv_numpy.xconnector import PandasConnec
 23
 24
 25class XdatasetCategory(ABC):
 26    ''' category of Xndarray (dynamic tuple of full_name) - see Xdataset docstring'''
 27
 28    xnd: list = NotImplemented
 29    names: list = NotImplemented
 30
 31    @abstractmethod
 32    def dims(self, var, json_name=False):
 33        '''method defined in Xdataset class'''
 34
 35    @property
 36    def data_arrays(self):
 37        '''return a tuple of data_arrays Xndarray full_name'''
 38        return tuple(sorted(nda for nda in self.namedarrays
 39                            if nda not in self.dimensions + self.uniques))
 40
 41    @property
 42    def dimensions(self):
 43        '''return a tuple of dimensions Xndarray full_name'''
 44        dimable = []
 45        for var in self.variables:
 46            dimable += self.dims(var)
 47        return tuple(sorted(set(nda for nda in dimable if nda in self.namedarrays)))
 48
 49    @property
 50    def shape(self):
 51        '''return an array with the length of dimensions'''
 52        return [len(self[dim]) for dim in self.dimensions]
 53
 54    @property
 55    def coordinates(self):
 56        '''return a tuple of coordinates Xndarray full_name'''
 57        dims = set(self.dimensions)
 58        if not dims:
 59            return ()
 60        return tuple(sorted(set(xnda.name for xnda in self.xnd
 61                                if xnda.xtype == 'variable' and set(xnda.links) != dims)))
 62
 63    @property
 64    def data_vars(self):
 65        '''return a tuple of data_vars Xndarray full_name'''
 66        dims = set(self.dimensions)
 67        if not dims:
 68            return self.variables
 69        return tuple(sorted(xnda.name for xnda in self.xnd
 70                            if xnda.xtype == 'variable' and set(xnda.links) == dims))
 71
 72    @property
 73    def namedarrays(self):
 74        '''return a tuple of namedarray Xndarray full_name'''
 75        return tuple(sorted(xnda.name for xnda in self.xnd if xnda.xtype == 'namedarray'))
 76
 77    @property
 78    def variables(self):
 79        '''return a tuple of variables Xndarray full_name'''
 80        return tuple(sorted(xnda.name for xnda in self.xnd if xnda.xtype == 'variable'))
 81
 82    @property
 83    def undef_vars(self):
 84        '''return a tuple of variables Xndarray full_name with inconsistent shape'''
 85        return tuple(sorted(var for var in self.variables if self[var].shape !=
 86                            [len(self[dim]) for dim in self.dims(var)]))
 87
 88    @property
 89    def undef_links(self):
 90        '''return a tuple of variables Xndarray full_name with inconsistent links'''
 91        return tuple(sorted(link for var in self.variables for link in self[var].links
 92                            if link not in self.names))
 93
 94    @property
 95    def masks(self):
 96        '''return a tuple of additional Xndarray full_name with boolean ntv_type'''
 97        return tuple(sorted(xnda.full_name for xnda in self.xnd
 98                            if xnda.xtype == 'additional' and xnda.ntv_type == 'boolean'))
 99
100    @property
101    def data_add(self):
102        '''return a tuple of additional Xndarray full_name with not boolean ntv_type'''
103        return tuple(sorted(xnda.full_name for xnda in self.xnd
104                            if xnda.xtype == 'additional' and xnda.ntv_type != 'boolean'))
105
106    @property
107    def metadata(self):
108        '''return a tuple of metadata Xndarray full_name'''
109        return tuple(sorted(xnda.full_name for xnda in self.xnd if xnda.xtype == 'meta'))
110
111    @property
112    def uniques(self):
113        '''return a tuple of unique Xndarray full_name'''
114        return tuple(full_name for full_name in self.namedarrays if len(self[full_name]) == 1)
115
116    @property
117    def additionals(self):
118        '''return a tuple of additionals Xndarray full_name'''
119        return tuple(sorted(xnda.full_name for xnda in self.xnd if xnda.xtype == 'additional'))
120
121    def group(self, grp):
122        '''return a tuple of Xndarray full_name with the same name'''
123        if isinstance(grp, str):
124            return tuple(sorted(xnda.full_name for xnda in self.xnd
125                                if grp in (xnda.name, xnda.full_name)))
126        return tuple(sorted(nam for gr_nam in grp for nam in self.group(gr_nam)))
127
128    def add_group(self, add_name):
129        '''return a tuple of Xndarray full_name with the same add_name'''
130        return tuple(sorted(xnda.full_name for xnda in self.xnd if xnda.add_name == add_name))
131
132
133class XdatasetInterface(ABC):
134    ''' Xdataset interface - see Xdataset docstring'''
135
136    name: str = NotImplemented
137    xnd: list = NotImplemented
138
139    @staticmethod
140    def read_json(jsn, **kwargs):
141        ''' convert json data into a Xdataset.
142
143        *Parameters*
144
145        - **convert** : boolean (default True) - If True, convert json data with
146        non Numpy ntv_type into Xndarray with python type
147        '''
148        option = {'convert': True} | kwargs
149        jso = json.loads(jsn) if isinstance(jsn, str) else jsn
150        value, name = Ntv.decode_json(jso)[:2]
151
152        xnd = [Xndarray.read_json({key: val}, **option)
153               for key, val in value.items()]
154        return Xdataset(xnd, name)
155
156    def to_json(self, **kwargs):
157        ''' convert a Xdataset into json-value.
158
159        *Parameters*
160
161        - **encoded** : Boolean (default False) - json value if False else json text
162        - **header** : Boolean (default True) - including 'xdataset' type
163        - **notype** : list of Boolean (default list of None) - including data type if False
164        - **novalue** : Boolean (default False) - including value if False
165        - **noshape** : Boolean (default True) - if True, without shape if dim < 1
166        - **format** : list of string (default list of 'full') - representation
167        format of the ndarray,
168        '''
169        notype = kwargs['notype'] if ('notype' in kwargs and isinstance(kwargs['notype'], list) and
170                                      len(kwargs['notype']) == len(self)) else [False] * len(self)
171        forma = kwargs['format'] if ('format' in kwargs and isinstance(kwargs['format'], list) and
172                                     len(kwargs['format']) == len(self)) else ['full'] * len(self)
173        noshape = kwargs.get('noshape', True)
174        dic_xnd = {}
175        for xna, notyp, forma in zip(self.xnd, notype, forma):
176            dic_xnd |= xna.to_json(notype=notyp, novalue=kwargs.get('novalue', False),
177                                   noshape=noshape, format=forma, header=False)
178        return Nutil.json_ntv(self.name, 'xdataset', dic_xnd,
179                              header=kwargs.get('header', True),
180                              encoded=kwargs.get('encoded', False))
181
182    def to_xarray(self, **kwargs):
183        '''return a xr.DataArray or a xr.Dataset from a Xdataset
184
185        *Parameters*
186
187        - **dataset** : Boolean (default True) - if False and a single data_var,
188        return a xr.DataArray
189        - **info** : Boolean (default True) - if True, add json representation
190        of 'relative' Xndarrays and 'data_arrays' Xndarrays in attrs
191        '''
192        return XarrayConnec.xexport(self, **kwargs)
193
194    @staticmethod
195    def from_xarray(xar, **kwargs):
196        '''return a Xdataset from a DataArray or a Dataset'''
197        return XarrayConnec.ximport(xar, Xdataset, **kwargs)
198
199    def to_scipp(self, **kwargs):
200        '''return a sc.DataArray or a sc.Dataset from a Xdataset
201
202        *Parameters*
203
204        - **dataset** : Boolean (default True) - if False and a single data_var,
205        return a DataArray
206        - **info** : Boolean (default True) - if True return a DataGroup with
207        metadata and data_arrays
208        - **ntv_type** : Boolean (default True) - if True add ntv-type to the name
209        '''
210        return ScippConnec.xexport(self, **kwargs)
211
212    @staticmethod
213    def from_scipp(sci, **kwargs):
214        '''return a Xdataset from a scipp object DataArray, Dataset or DataGroup'''
215        return ScippConnec.ximport(sci, Xdataset, **kwargs)
216
217    def to_nddata(self, **kwargs):
218        '''return a NDData from a Xdataset'''
219        return AstropyNDDataConnec.xexport(self, **kwargs)
220
221    @staticmethod
222    def from_nddata(ndd, **kwargs):
223        '''return a Xdataset from a NDData'''
224        return AstropyNDDataConnec.ximport(ndd, Xdataset, **kwargs)
225
226    def to_dataframe(self, **kwargs):
227        '''return a pd.DataFrame from a Xdataset
228
229        *Parameters*
230
231        - **ntv_type**: Boolean (default True) - if False use full_name else json_name
232        - **info**: Boolean (default True) - if True add xdt.info in DataFrame.attrs
233        - **dims**: list of string (default None) - order of dimensions full_name to apply
234        '''
235        return PandasConnec.xexport(self, **kwargs)
236
237    @staticmethod
238    def from_dataframe(dfr, **kwargs):
239        '''return a Xdataset from a pd.DataFrame
240
241        *Parameters*
242
243        - dims: list of string (default None) - order of dimensions to apply
244        '''
245        return PandasConnec.ximport(dfr, Xdataset, **kwargs)
246
247
248class Xdataset(XdatasetCategory, XdatasetInterface):
249    ''' Representation of a multidimensional Dataset
250
251    *Attributes :*
252    - **name** :  String - name of the Xdataset
253    - **xnd**:   list of Xndarray
254
255    *dynamic values (@property)*
256    - `xtype`
257    - `validity`
258    - `dic_xnd`
259    - `partition`
260    - `length`
261    - `info`
262
263    *methods*
264    - `parent`
265    - `dims`
266    - `shape_dims`
267    - `to_canonical`
268    - `to_ndarray`
269    - `to_darray`
270
271    *XdatasetCategory (@property)*
272    - `names`
273    - `data_arrays`
274    - `dimensions`
275    - `coordinates`
276    - `data_vars`
277    - `namedarrays`
278    - `variables`
279    - `undef_vars`
280    - `undef_links`
281    - `masks`
282    - `data_add`
283    - `meta`
284    - `metadata`
285    - `uniques`
286    - `additionals`
287    - `group`
288    - `add_group`
289
290    *XdatasetInterface methods *
291    - `read_json` (static)
292    - `to_json`
293    - `from_xarray` (static)
294    - `to_xarray`
295    - `from_scipp` (static)
296    - `to_scipp`
297    - `from_nddata` (static)
298    - `to_nddata`
299    - `from_dataframe` (static)
300    - `to_dataframe`
301    '''
302
303    def __init__(self, xnd=None, name=None):
304        '''Xdataset constructor
305
306            *Parameters*
307
308            - **xnd** : Xdataset/Xndarray/list of Xndarray (default None),
309            - **name** : String (default None) - name of the Xdataset
310        '''
311        self.name = name
312        match xnd:
313            case list():
314                self.xnd = xnd
315            case xdat if isinstance(xdat, Xdataset):
316                self.name = xdat.name
317                self.xnd = xdat.xnd
318            case xnda if isinstance(xnda, Xndarray):
319                self.xnd = [xnda]
320            case _:
321                self.xnd = []
322
323    def __repr__(self):
324        '''return classname and number of value'''
325        return (self.__class__.__name__ + '[' + str(len(self)) + ']\n' +
326                pprint.pformat(self.to_json(novalue=True, header=False, noshape=False)))
327
328    def __str__(self):
329        '''return json string format'''
330        return json.dumps(self.to_json())
331
332    def __eq__(self, other):
333        '''equal if xnd are equal'''
334        for xnda in self.xnd:
335            if xnda not in other:
336                return False
337        for xnda in other.xnd:
338            if xnda not in self:
339                return False
340        return True
341
342    def __len__(self):
343        '''number of Xndarray'''
344        return len(self.xnd)
345
346    def __contains__(self, item):
347        ''' item of xnd'''
348        return item in self.xnd
349
350    def __getitem__(self, selec):
351        ''' return Xndarray or tuple of Xndarray with selec:
352            - string : name of a xndarray,
353            - integer : index of a xndarray,
354            - index selector : index interval
355            - tuple : names or index '''
356        if selec is None or selec == '' or selec in ([], ()):
357            return self
358        if isinstance(selec, (list, tuple)) and len(selec) == 1:
359            selec = selec[0]
360        if isinstance(selec, tuple):
361            return [self[i] for i in selec]
362        if isinstance(selec, str):
363            return self.dic_xnd[selec]
364        if isinstance(selec, list):
365            return self[selec[0]][selec[1:]]
366        return self.xnd[selec]
367
368    def __delitem__(self, ind):
369        '''remove a Xndarray (ind is index, name or tuple of names).'''
370        if isinstance(ind, int):
371            del self.xnd[ind]
372        elif isinstance(ind, str):
373            del self.xnd[self.names.index(ind)]
374        elif isinstance(ind, tuple):
375            ind_n = [self.names[i] if isinstance(i, int) else i for i in ind]
376            for i in ind_n:
377                del self[i]
378
379    def __copy__(self):
380        ''' Copy all the data '''
381        return self.__class__(self)
382
383    def parent(self, var):
384        '''return the Xndarray parent (where the full_name is equal to the name)'''
385        if var.name in self.names:
386            return self[var.name]
387        return var
388
389    def dims(self, var, json_name=False):
390        '''return the list of parent namedarrays of the links of a Xndarray
391
392        *parameters*
393
394        - **var**: string - full_name of the Xndarray
395        - **json_name**: boolean (defaut False) - if True return json_name else full_name
396        '''
397        if var not in self.names:
398            return None
399        if self[var].add_name and not self[var].links:
400            return self.dims(self[var].name, json_name)
401        if var in self.namedarrays:
402            return [self[var].json_name if json_name else var]
403        if var not in self.variables + self.additionals:
404            return None
405        list_dims = []
406        for link in self[var].links:
407            list_dims += self.dims(link, json_name) if self.dims(link,
408                                                                 json_name) else [link]
409        return list_dims
410
411    def shape_dims(self, var):
412        '''return a shape with the dimensions associated to the var full_name'''
413        return [len(self[dim]) for dim in self.dims(var)
414                ] if set(self.dims(var)) <= set(self.names) else None
415
416    @property
417    def validity(self):
418        '''return the validity state: 'inconsistent', 'undifined' or 'valid' '''
419        for xnda in self:
420            if xnda.mode in ['relative', 'inconsistent']:
421                return 'undefined'
422        if self.undef_links or self.undef_vars:
423            return 'inconsistent'
424        return 'valid'
425
426    @property
427    def xtype(self):
428        '''return the Xdataset type: 'meta', 'group', 'mono', 'multi' '''
429        if self.metadata and not (self.additionals or self.variables or
430                                  self.namedarrays):
431            return 'meta'
432        if self.validity != 'valid':
433            return 'group'
434        match len(self.data_vars):
435            case 0:
436                return 'group'
437            case 1:
438                return 'mono'
439            case _:
440                return 'multi'
441
442    @property
443    def dic_xnd(self):
444        '''return a dict of Xndarray where key is the full_name'''
445        return {xnda.full_name: xnda for xnda in self.xnd}
446
447    @property
448    def length(self):
449        '''return the max length of Xndarray'''
450        return max(len(xnda) for xnda in self.xnd)
451
452    @property
453    def names(self):
454        '''return a tuple with the Xndarray full_name'''
455        return tuple(xnda.full_name for xnda in self.xnd)
456
457    @property
458    def partition(self):
459        '''return a dict of Xndarray grouped with category'''
460        dic = {}
461        dic |= {'data_vars': list(self.data_vars)} if self.data_vars else {}
462        dic |= {'data_arrays': list(self.data_arrays)
463                } if self.data_arrays else {}
464        dic |= {'dimensions': list(self.dimensions)} if self.dimensions else {}
465        dic |= {'coordinates': list(self.coordinates)
466                } if self.coordinates else {}
467        dic |= {'additionals': list(self.additionals)
468                } if self.additionals else {}
469        dic |= {'metadata': list(self.metadata)} if self.metadata else {}
470        dic |= {'uniques': list(self.uniques)} if self.uniques else {}
471        return dic
472
473    @property
474    def info(self):
475        '''return a dict with Xdataset information '''
476        inf = {'name': self.name, 'xtype': self.xtype} | self.partition
477        inf['validity'] = self.validity
478        inf['length'] = len(self[self.data_vars[0]]) if self.data_vars else 0
479        inf['width'] = len(self)
480        data = {name: {key: val for key, val in self[name].info.items() if key != 'name'}
481                for name in self.names}
482        return {'structure': {key: val for key, val in inf.items() if val},
483                'data': {key: val for key, val in data.items() if val}}
484
485    @property
486    def tab_info(self):
487        '''return a dict with Xdataset information for tabular interface'''
488        info = self.info
489        data = info['data']
490        t_info = {}
491        if 'dimensions' in info['structure']:
492            t_info['dimensions'] = info['structure']['dimensions']
493        t_info['data'] = {name: {key: val for key, val in data[name].items()
494                                 if key in ['shape', 'xtype', 'meta', 'links']}
495                          for name in data}
496        return t_info
497
498    def to_canonical(self):
499        '''remove optional links of the included Xndarray'''
500        for name in self.names:
501            if self[name].links in ([self[name].name], [name]):
502                self[name].links = None
503        for add in self.additionals:
504            if self[add].links in [self[self[add].name].links,
505                                   [self[add].name]]:
506                self[add].links = None
507        for unic in self.uniques:
508            self[unic].links = None
509        return self
510
511    def to_ndarray(self, full_name):
512        '''convert a Xndarray from a Xdataset in a np.ndarray'''
513        if self.shape_dims(full_name) is None:
514            data = self[full_name].ndarray
515        else:
516            data = self[full_name].darray.reshape(self.shape_dims(full_name))
517        if data.dtype.name[:8] == 'datetime':
518            data = data.astype('datetime64[ns]')
519        return data
520
521    def to_darray(self, full_name):
522        '''convert a Xndarray from a Xdataset in a flattened np.ndarray'''
523        data = self[full_name].darray
524        if data.dtype.name[:8] == 'datetime':
525            data = data.astype('datetime64[ns]')
526        return data