ntv-numpy.ntv_numpy.xdataset

  1# -*- coding: utf-8 -*-
  2"""
  3@author: Philippe@loco-labs.io
  4
  5The `xdataset` module is part of the `ntv-numpy.ntv_numpy` package ([specification document](
  6https://loco-philippe.github.io/ES/JSON%20semantic%20format%20(JSON-NTV).htm)).
  7
  8It contains the classes `Xdataset`, `XdatasetInterface`, `XdatasetCategory` for 
  9the multidimensional dataset.
 10
 11For more information, see the
 12[user guide](https://loco-philippe.github.io/ntv-numpy/docs/user_guide.html)
 13 or the [github repository](https://github.com/loco-philippe/ntv-numpy).
 14"""
 15from abc import ABC, abstractmethod
 16import json
 17import pprint
 18from json_ntv import Ntv
 19from ntv_numpy.ndarray import Nutil
 20from ntv_numpy.xndarray import Xndarray
 21from ntv_numpy.xconnector import XarrayConnec, ScippConnec, AstropyNDDataConnec
 22from ntv_numpy.xconnector import PandasConnec
 23
 24
 25class XdatasetCategory(ABC):
 26    ''' category of Xndarray (dynamic tuple of full_name) - see Xdataset docstring'''
 27
 28    xnd: list = NotImplemented
 29    names: list = NotImplemented
 30
 31    @abstractmethod
 32    def dims(self, var, json_name=False):
 33        '''method defined in Xdataset class'''
 34
 35    @property
 36    def data_arrays(self):
 37        '''return a tuple of data_arrays Xndarray full_name'''
 38        return tuple(sorted(nda for nda in self.namedarrays
 39                            if not nda in self.dimensions + self.uniques))
 40
 41    @property
 42    def dimensions(self):
 43        '''return a tuple of dimensions Xndarray full_name'''
 44        dimable = []
 45        for var in self.variables:
 46            dimable += self.dims(var)
 47        return tuple(sorted(set(nda for nda in dimable if nda in self.namedarrays)))
 48
 49    @property
 50    def shape(self):
 51        '''return an array with the length of dimensions'''
 52        return [len(self[dim]) for dim in self.dimensions]
 53
 54    @property
 55    def coordinates(self):
 56        '''return a tuple of coordinates Xndarray full_name'''
 57        dims = set(self.dimensions)
 58        if not dims:
 59            return ()
 60        return tuple(sorted(set(xnda.name for xnda in self.xnd
 61                                if xnda.xtype == 'variable' and set(xnda.links) != dims)))
 62
 63    @property
 64    def data_vars(self):
 65        '''return a tuple of data_vars Xndarray full_name'''
 66        dims = set(self.dimensions)
 67        if not dims:
 68            return self.variables
 69        return tuple(sorted(xnda.name for xnda in self.xnd
 70                            if xnda.xtype == 'variable' and set(xnda.links) == dims))
 71
 72    @property
 73    def namedarrays(self):
 74        '''return a tuple of namedarray Xndarray full_name'''
 75        return tuple(sorted(xnda.name for xnda in self.xnd if xnda.xtype == 'namedarray'))
 76
 77    @property
 78    def variables(self):
 79        '''return a tuple of variables Xndarray full_name'''
 80        return tuple(sorted(xnda.name for xnda in self.xnd if xnda.xtype == 'variable'))
 81
 82    @property
 83    def undef_vars(self):
 84        '''return a tuple of variables Xndarray full_name with inconsistent shape'''
 85        return tuple(sorted(var for var in self.variables if self[var].shape !=
 86                            [len(self[dim]) for dim in self.dims(var)]))
 87
 88    @property
 89    def undef_links(self):
 90        '''return a tuple of variables Xndarray full_name with inconsistent links'''
 91        return tuple(sorted(link for var in self.variables for link in self[var].links
 92                            if not link in self.names))
 93
 94    @property
 95    def masks(self):
 96        '''return a tuple of additional Xndarray full_name with boolean ntv_type'''
 97        return tuple(sorted(xnda.full_name for xnda in self.xnd
 98                            if xnda.xtype == 'additional' and xnda.ntv_type == 'boolean'))
 99
100    @property
101    def data_add(self):
102        '''return a tuple of additional Xndarray full_name with not boolean ntv_type'''
103        return tuple(sorted(xnda.full_name for xnda in self.xnd
104                            if xnda.xtype == 'additional' and xnda.ntv_type != 'boolean'))
105
106    @property
107    def metadata(self):
108        '''return a tuple of metadata Xndarray full_name'''
109        return tuple(sorted(xnda.full_name for xnda in self.xnd if xnda.xtype == 'meta'))
110
111    @property
112    def uniques(self):
113        '''return a tuple of unique Xndarray full_name'''
114        return tuple(full_name for full_name in self.namedarrays if len(self[full_name]) == 1)
115
116    @property
117    def additionals(self):
118        '''return a tuple of additionals Xndarray full_name'''
119        return tuple(sorted(xnda.full_name for xnda in self.xnd if xnda.xtype == 'additional'))
120
121    def group(self, grp):
122        '''return a tuple of Xndarray full_name with the same name'''
123        if isinstance(grp, str):
124            return tuple(sorted(xnda.full_name for xnda in self.xnd
125                                if grp in (xnda.name, xnda.full_name)))
126        return tuple(sorted(nam for gr_nam in grp for nam in self.group(gr_nam)))
127
128    def add_group(self, add_name):
129        '''return a tuple of Xndarray full_name with the same add_name'''
130        return tuple(sorted(xnda.full_name for xnda in self.xnd if xnda.add_name == add_name))
131
132
133class XdatasetInterface(ABC):
134    ''' Xdataset interface - see Xdataset docstring'''
135
136    name: str = NotImplemented
137    xnd: list = NotImplemented
138
139    @staticmethod
140    def read_json(jsn, **kwargs):
141        ''' convert json data into a Xdataset.
142
143        *Parameters*
144
145        - **convert** : boolean (default True) - If True, convert json data with
146        non Numpy ntv_type into Xndarray with python type
147        '''
148        option = {'convert': True} | kwargs
149        jso = json.loads(jsn) if isinstance(jsn, str) else jsn
150        value, name = Ntv.decode_json(jso)[:2]
151
152        xnd = [Xndarray.read_json({key: val}, **option)
153               for key, val in value.items()]
154        return Xdataset(xnd, name)
155
156    def to_json(self, **kwargs):
157        ''' convert a Xdataset into json-value.
158
159        *Parameters*
160
161        - **encoded** : Boolean (default False) - json value if False else json text
162        - **header** : Boolean (default True) - including 'xdataset' type
163        - **notype** : list of Boolean (default list of None) - including data type if False
164        - **novalue** : Boolean (default False) - including value if False
165        - **noshape** : Boolean (default True) - if True, without shape if dim < 1
166        - **format** : list of string (default list of 'full') - representation
167        format of the ndarray,
168        '''
169        notype = kwargs['notype'] if ('notype' in kwargs and isinstance(kwargs['notype'], list) and
170                                      len(kwargs['notype']) == len(self)) else [False] * len(self)
171        forma = kwargs['format'] if ('format' in kwargs and isinstance(kwargs['format'], list) and
172                                     len(kwargs['format']) == len(self)) else ['full'] * len(self)
173        noshape = kwargs.get('noshape', True)
174        dic_xnd = {}
175        for xna, notyp, forma in zip(self.xnd, notype, forma):
176            dic_xnd |= xna.to_json(notype=notyp, novalue=kwargs.get('novalue', False),
177                                   noshape=noshape, format=forma, header=False)
178        return Nutil.json_ntv(self.name, 'xdataset', dic_xnd,
179                              header=kwargs.get('header', True),
180                              encoded=kwargs.get('encoded', False))
181
182    def to_xarray(self, **kwargs):
183        '''return a DataArray or a Dataset from a Xdataset
184
185        *Parameters*
186
187        - **dataset** : Boolean (default True) - if False and a single data_var, return a DataArray
188        '''
189        return XarrayConnec.xexport(self, **kwargs)
190
191    @staticmethod
192    def from_xarray(xar, **kwargs):
193        '''return a Xdataset from a DataArray or a Dataset'''
194        return XarrayConnec.ximport(xar, Xdataset, **kwargs)
195
196    def to_scipp(self, **kwargs):
197        '''return a sc.DataArray or a sc.Dataset from a Xdataset
198
199        *Parameters*
200
201        - **dataset** : Boolean (default True) - if False and a single data_var,
202        return a DataArray
203        - **datagroup** : Boolean (default True) - if True return a DataGroup with
204        metadata and data_arrays
205        - **ntv_type** : Boolean (default True) - if True add ntv-type to the name
206        '''
207        return ScippConnec.xexport(self, **kwargs)
208
209    @staticmethod
210    def from_scipp(sci, **kwargs):
211        '''return a Xdataset from a scipp object DataArray, Dataset or DataGroup'''
212        return ScippConnec.ximport(sci, Xdataset, **kwargs)
213
214    def to_nddata(self, **kwargs):
215        '''return a NDData from a Xdataset'''
216        return AstropyNDDataConnec.xexport(self, **kwargs)
217
218    @staticmethod
219    def from_nddata(ndd, **kwargs):
220        '''return a Xdataset from a NDData'''
221        return AstropyNDDataConnec.ximport(ndd, Xdataset, **kwargs)
222
223    def to_dataframe(self, **kwargs):
224        '''return a pd.DataFrame from a Xdataset'''
225        return PandasConnec.xexport(self, **kwargs)
226
227    @staticmethod
228    def from_dataframe(dfr, **kwargs):
229        '''return a Xdataset from a pd.DataFrame'''
230        return PandasConnec.ximport(dfr, Xdataset, **kwargs)
231
232
233class Xdataset(XdatasetCategory, XdatasetInterface):
234    ''' Representation of a multidimensional Dataset
235
236    *Attributes :*
237    - **name** :  String - name of the Xdataset
238    - **xnd**:   list of Xndarray
239
240    *dynamic values (@property)*
241    - `xtype`
242    - `validity`
243    - `dic_xnd`
244    - `partition`
245    - `length`
246    - `info`
247
248    *methods*
249    - `parent`
250    - `dims`
251    - `shape_dims`
252    - `to_canonical`
253    - `to_ndarray`
254    - `to_darray`
255
256    *XdatasetCategory (@property)*
257    - `names`
258    - `data_arrays`
259    - `dimensions`
260    - `coordinates`
261    - `data_vars`
262    - `namedarrays`
263    - `variables`
264    - `undef_vars`
265    - `undef_links`
266    - `masks`
267    - `data_add`
268    - `meta`
269    - `metadata`
270    - `uniques`
271    - `additionals`
272    - `group`
273    - `add_group`
274
275    *XdatasetInterface methods *
276    - `read_json` (static)
277    - `to_json`
278    - `from_xarray` (static)
279    - `to_xarray`
280    - `from_scipp` (static)
281    - `to_scipp`
282    - `from_nddata` (static)
283    - `to_nddata`
284    - `from_dataframe` (static)
285    - `to_dataframe`
286    '''
287
288    def __init__(self, xnd=None, name=None):
289        '''Xdataset constructor
290
291            *Parameters*
292
293            - **xnd** : Xdataset/Xndarray/list of Xndarray (default None),
294            - **name** : String (default None) - name of the Xdataset
295        '''
296        self.name = name
297        match xnd:
298            case list():
299                self.xnd = xnd
300            case xdat if isinstance(xdat, Xdataset):
301                self.name = xdat.name
302                self.xnd = xdat.xnd
303            case xnda if isinstance(xnda, Xndarray):
304                self.xnd = [xnda]
305            case _:
306                self.xnd = []
307
308    def __repr__(self):
309        '''return classname and number of value'''
310        return (self.__class__.__name__ + '[' + str(len(self)) + ']\n' +
311                pprint.pformat(self.to_json(novalue=True, header=False, noshape=False)))
312
313    def __str__(self):
314        '''return json string format'''
315        return json.dumps(self.to_json())
316
317    def __eq__(self, other):
318        '''equal if xnd are equal'''
319        for xnda in self.xnd:
320            if not xnda in other:
321                return False
322        for xnda in other.xnd:
323            if not xnda in self:
324                return False
325        return True
326
327    def __len__(self):
328        '''number of Xndarray'''
329        return len(self.xnd)
330
331    def __contains__(self, item):
332        ''' item of xnd'''
333        return item in self.xnd
334
335    def __getitem__(self, selec):
336        ''' return Xndarray or tuple of Xndarray with selec:
337            - string : name of a xndarray,
338            - integer : index of a xndarray,
339            - index selector : index interval
340            - tuple : names or index '''
341        if selec is None or selec == '' or selec in ([], ()):
342            return self
343        if isinstance(selec, (list, tuple)) and len(selec) == 1:
344            selec = selec[0]
345        if isinstance(selec, tuple):
346            return [self[i] for i in selec]
347        if isinstance(selec, str):
348            return self.dic_xnd[selec]
349        if isinstance(selec, list):
350            return self[selec[0]][selec[1:]]
351        return self.xnd[selec]
352
353    def __delitem__(self, ind):
354        '''remove a Xndarray (ind is index, name or tuple of names).'''
355        if isinstance(ind, int):
356            del self.xnd[ind]
357        elif isinstance(ind, str):
358            del self.xnd[self.names.index(ind)]
359        elif isinstance(ind, tuple):
360            ind_n = [self.names[i] if isinstance(i, int) else i for i in ind]
361            for i in ind_n:
362                del self[i]
363
364    def __copy__(self):
365        ''' Copy all the data '''
366        return self.__class__(self)
367
368    def parent(self, var):
369        '''return the Xndarray parent (where the full_name is equal to the name)'''
370        if var.name in self.names:
371            return self[var.name]
372        return var
373
374    def dims(self, var, json_name=False):
375        '''return the list of parent namedarrays of the links of a Xndarray
376
377        *parameters*
378
379        - **var**: string - full_name of the Xndarray
380        - **json_name**: boolean (defaut False) - if True return json_name else full_name
381        '''
382        if not var in self.names:
383            return None
384        if self[var].add_name and not self[var].links:
385            return self.dims(self[var].name, json_name)
386        if var in self.namedarrays:
387            return [self[var].json_name if json_name else var]
388        if not var in self.variables + self.additionals:
389            return None
390        list_dims = []
391        for link in self[var].links:
392            list_dims += self.dims(link, json_name) if self.dims(link,
393                                                                 json_name) else [link]
394        return list_dims
395
396    def shape_dims(self, var):
397        '''return a shape with the dimensions associated to the var full_name'''
398        return [len(self[dim]) for dim in self.dims(var)
399                ] if set(self.dims(var)) <= set(self.names) else None
400
401    @property
402    def validity(self):
403        '''return the validity state: 'inconsistent', 'undifined' or 'valid' '''
404        for xnda in self:
405            if xnda.mode in ['relative', 'inconsistent']:
406                return 'undefined'
407        if self.undef_links or self.undef_vars:
408            return 'inconsistent'
409        return 'valid'
410
411    @property
412    def xtype(self):
413        '''return the Xdataset type: 'meta', 'group', 'mono', 'multi' '''
414        if self.metadata and not (self.additionals or self.variables or
415                                  self.namedarrays):
416            return 'meta'
417        if self.validity != 'valid':
418            return 'group'
419        match len(self.data_vars):
420            case 0:
421                return 'group'
422            case 1:
423                return 'mono'
424            case _:
425                return 'multi'
426
427    @property
428    def dic_xnd(self):
429        '''return a dict of Xndarray where key is the full_name'''
430        return {xnda.full_name: xnda for xnda in self.xnd}
431
432    @property
433    def length(self):
434        '''return the max length of Xndarray'''
435        return max(len(xnda) for xnda in self.xnd)
436
437    @property
438    def names(self):
439        '''return a tuple with the Xndarray full_name'''
440        return tuple(xnda.full_name for xnda in self.xnd)
441
442    @property
443    def partition(self):
444        '''return a dict of Xndarray grouped with category'''
445        dic = {}
446        dic |= {'data_vars': list(self.data_vars)} if self.data_vars else {}
447        dic |= {'data_arrays': list(self.data_arrays)
448                } if self.data_arrays else {}
449        dic |= {'dimensions': list(self.dimensions)} if self.dimensions else {}
450        dic |= {'coordinates': list(self.coordinates)
451                } if self.coordinates else {}
452        dic |= {'additionals': list(self.additionals)
453                } if self.additionals else {}
454        dic |= {'metadata': list(self.metadata)} if self.metadata else {}
455        dic |= {'uniques': list(self.uniques)} if self.uniques else {}
456        return dic
457
458    @property
459    def info(self):
460        '''return a dict with Xdataset information '''
461        inf = {'name': self.name, 'xtype': self.xtype} | self.partition
462        inf['validity'] = self.validity
463        inf['length'] = len(self[self.data_vars[0]]) if self.data_vars else 0
464        inf['width'] = len(self)
465        data = {name: {key: val for key, val in self[name].info.items() if key != 'name'}
466                for name in self.names}
467        return {'structure': {key: val for key, val in inf.items() if val},
468                'data': {key: val for key, val in data.items() if val}}
469
470    @property
471    def tab_info(self):
472        '''return a dict with Xdataset information for tabular interface'''
473        info = self.info
474        data = info['data']
475        t_info = {}
476        if 'dimensions' in info['structure']:
477            t_info['dimensions'] = info['structure']['dimensions']
478        t_info['data'] = {name: {key: val for key, val in data[name].items()
479                                 if key in ['shape', 'xtype', 'meta', 'links']}
480                          for name in data}
481        return t_info
482
483    def to_canonical(self):
484        '''remove optional links of the included Xndarray'''
485        for name in self.names:
486            if self[name].links in ([self[name].name], [name]):
487                self[name].links = None
488        for add in self.additionals:
489            if self[add].links in [self[self[add].name].links,
490                                   [self[add].name]]:
491                self[add].links = None
492        for unic in self.uniques:
493            self[unic].links = None
494        return self
495
496    def to_ndarray(self, full_name):
497        '''convert a Xndarray from a Xdataset in a np.ndarray'''
498        if self.shape_dims(full_name) is None:
499            data = self[full_name].ndarray
500        else:
501            data = self[full_name].darray.reshape(self.shape_dims(full_name))
502        if data.dtype.name[:8] == 'datetime':
503            data = data.astype('datetime64[ns]')
504        return data
505
506    def to_darray(self, full_name):
507        '''convert a Xndarray from a Xdataset in a flattened np.ndarray'''
508        data = self[full_name].darray
509        if data.dtype.name[:8] == 'datetime':
510            data = data.astype('datetime64[ns]')
511        return data