ntv-numpy.ntv_numpy.data_array

  1# -*- coding: utf-8 -*-
  2"""
  3@author: Philippe@loco-labs.io
  4
  5The `data_array` module is part of the `ntv-numpy.ntv_numpy` package ([specification document](
  6https://loco-philippe.github.io/ES/JSON%20semantic%20format%20(JSON-NTV).htm)).
  7
  8It contains the classes `Darray` (abstract), `Dfull`, `Dcomplete` for the
  9representation of unidimensional arrays.
 10
 11For more information, see the
 12[user guide](https://loco-philippe.github.io/ntv-numpy/docs/user_guide.html)
 13 or the [github repository](https://github.com/loco-philippe/ntv-numpy).
 14
 15"""
 16
 17#import datetime
 18#from decimal import Decimal
 19
 20from abc import ABC, abstractmethod
 21import json
 22import numpy as np
 23from json_ntv import Ntv, NtvConnector
 24#from json_ntv import Ntv, ShapelyConnec, NtvConnector
 25import pandas as pd
 26
 27
 28class Darray(ABC):
 29    ''' The Darray class is an abstract class used by `Dfull`and `Dcomplete` classes.
 30
 31    *Attributes :*
 32    - **data** :  np.ndarray - data after coding
 33    - **ref**:  int or string - reference to another Darray data
 34    - **coding**: np.ndarray of int - mapping between data and the values
 35
 36    *dynamic values (@property)*
 37    - `values`
 38
 39    *methods*
 40    - `read_json` (staticmethod)
 41    - `to_json`
 42    '''
 43
 44    def __init__(self, data, ref=None, coding=None, dtype=None, unidim=False):
 45        '''Darray constructor.
 46
 47        *Parameters*
 48
 49        - **data**: list, Darray or np.ndarray - data to represent (after coding)
 50        - **ref** : String or integer (default None) - name or index of another Darray
 51        - **coding**: List of integer (default None) - mapping between data and the list of values
 52        - **dtype**: string (default None) - numpy.dtype to apply
 53        '''
 54        if isinstance(data, Darray):
 55            self.data = data.data
 56            self.ref = data.ref
 57            self.coding = data.coding
 58            return
 59        data = data if isinstance(data, (list, np.ndarray)) else [data]
 60        if (len(data) > 0 and isinstance(data[0], (list, np.ndarray))) or unidim:
 61            self.data = np.fromiter(data, dtype='object')
 62        else:
 63            self.data = np.array(data, dtype=dtype).reshape(-1)
 64        self.ref = ref
 65        self.coding = np.array(coding)
 66
 67    def __repr__(self):
 68        '''return classname and number of value'''
 69        return self.__class__.__name__ + '[' + str(len(self)) + ']'
 70
 71    def __str__(self):
 72        '''return json string format'''
 73        return json.dumps(self.to_json())
 74
 75    def __eq__(self, other):
 76        ''' equal if values are equal'''
 77        return np.array_equal(self.values, other.values, equal_nan=False)
 78
 79    def __len__(self):
 80        ''' len of values'''
 81        return self._len_val
 82
 83    def __contains__(self, item):
 84        ''' item of values'''
 85        return item in self.values
 86
 87    def __getitem__(self, ind):
 88        ''' return value item'''
 89        if isinstance(ind, tuple):
 90            return [self.values[i] for i in ind]
 91            # return [copy(self.values[i]) for i in ind]
 92        return self.values[ind]
 93        # return copy(self.values[ind])
 94
 95    def __copy__(self):
 96        ''' Copy all the data '''
 97        return self.__class__(self)
 98
 99    @staticmethod
100    def read_json(val, dtype=None, unidim=False):
101        ''' return a Darray entity from a list of data.
102
103        *Parameters*
104
105        - **val**: list of data
106        - **dtype** : string (default None) - numpy.dtype to apply
107        '''
108        val = val if isinstance(val, list) else [val]
109        if not val or not isinstance(val[0], list):
110            return Dfull(val, dtype=dtype, unidim=unidim)
111        match val:
112            case [data, ref, list(coding)] if (isinstance(ref, (int, str)) and
113                                               isinstance(coding[0], int) and
114                                               max(coding) < len(data)):
115                return None
116            case [data, ref] if (isinstance(data, list) and
117                                 isinstance(ref, (int, str))):
118                return None
119            case [data, list(coef)] if len(coef) == 1:
120                return None
121            case [data, list(coding)] if (isinstance(coding[0], int) and
122                                          max(coding) < len(data)):
123                return Dcomplete(data, None, coding, dtype=dtype, unidim=unidim)
124            case _:
125                return Dfull(val, dtype=dtype, unidim=unidim)
126
127    @abstractmethod
128    def to_json(self):
129        ''' return a JsonValue'''
130
131    @property
132    @abstractmethod
133    def values(self):
134        ''' return the list of values'''
135
136    @property
137    @abstractmethod
138    def _len_val(self):
139        '''return the length of the entity'''
140
141
142class Dfull(Darray):
143    ''' Representation of a one dimensional Array with full representation
144
145    *dynamic values (@property)*
146    - `values`
147
148    *methods*
149    - `read_json` (staticmethod)
150    - `to_json`
151    '''
152
153    def __init__(self, data, ref=None, coding=None, dtype=None, unidim=False):
154        '''Dfull constructor.
155
156        *Parameters*
157
158        - **data**: list, Darray or np.ndarray - data to represent (after coding)
159        - **ref** : unused
160        - **coding**: unused
161        - **dtype**: string (default None) - numpy.dtype to apply
162        '''
163        super().__init__(data, dtype=dtype, unidim=unidim)
164
165    def to_json(self):
166        ''' return a JsonValue of the Dfull entity.'''
167        #return self.data.tolist()
168        return Dutil.list_json(self.data)
169
170    @property
171    def values(self):
172        ''' return the list of values'''
173        return self.data
174
175    @property
176    def _len_val(self):
177        '''return the length of the Dfull entity'''
178        return len(self.data) if self.data.ndim > 0 else 0
179
180
181class Dcomplete(Darray):
182    ''' Representation of a one dimensional Array with full representation
183
184    *dynamic values (@property)*
185    - `values`
186
187    *methods*
188    - `read_json` (staticmethod)
189    - `to_json`
190    '''
191
192    def __init__(self, data, ref=None, coding=None, dtype=None, unidim=False):
193        '''Dcomplete constructor.
194
195        *Parameters*
196
197        - **data**: list, Darray or np.ndarray - data to represent (after coding)
198        - **ref** : unused
199        - **coding**: List of integer (default None) - mapping between data and the list of values
200        - **dtype**: string (default None) - numpy.dtype to apply
201        '''
202        if coding is None:
203            try:
204                data, coding = np.unique(data, return_inverse=True)
205            except (TypeError, ValueError):
206                dat, idx, coding = np.unique(np.frompyfunc(Ntv.from_obj, 1, 1)(data),
207                                             return_index=True, return_inverse=True)
208                data = data[idx]
209        super().__init__(data, coding=coding, dtype=dtype, unidim=unidim)
210
211    def to_json(self):
212        ''' return a JsonValue of the Dcomplete entity.'''
213        #return [self.data.tolist(), self.coding.tolist()]
214        return [Dutil.list_json(self.data), self.coding.tolist()]
215
216    @property
217    def values(self):
218        ''' return the list of values'''
219        return self.data[self.coding]
220
221    @property
222    def _len_val(self):
223        '''return the length of the Dcomplete entity'''
224        return len(self.coding) if self.coding.ndim > 0 else 0
225
226class Dutil:
227    '''np.ndarray utilities.
228
229    *static methods*
230    - `convert`
231    - `is_json`
232    - `ntv_val`
233    - `add_ext`
234    - `split_type`
235    - `ntv_type`
236    - `nda_ntv_type`
237    - `dtype`
238    - `json_ntv`
239    - `split_name`
240    - `split_json_name`
241    '''
242    """DATATION_DT = {'date': 'datetime64[D]', 'year': 'datetime64[Y]',
243                   'yearmonth': 'datetime64[M]',
244                   'datetime': 'datetime64[s]', 'datetime[ms]': 'datetime64[ms]',
245                   'datetime[us]': 'datetime64[us]', 'datetime[ns]': 'datetime64[ns]',
246                   'datetime[ps]': 'datetime64[ps]', 'datetime[fs]': 'datetime64[fs]',
247                   'timedelta': 'timedelta64[s]', 'timedelta[ms]': 'timedelta64[ms]',
248                   'timedelta[us]': 'timedelta64[us]', 'timedelta[ns]': 'timedelta64[ns]',
249                   'timedelta[ps]': 'timedelta64[ps]', 'timedelta[fs]': 'timedelta64[fs]',
250                   'timedelta[D]': 'timedelta64[D]', 'timedelta[Y]': 'timedelta64[Y]',
251                   'timedelta[M]': 'timedelta64[M]'}
252    DT_DATATION = {val: key for key, val in DATATION_DT.items()}
253
254    #CONNECTOR_DT = {'field': 'Series', 'tab': 'DataFrame'}
255    CONNECTOR_DT = {'field': 'Series', 'tab': 'DataFrame'}
256    DT_CONNECTOR = {val: key for key, val in CONNECTOR_DT.items()}
257
258    PYTHON_DT = {'array': 'list', 'time': 'datetime.time',
259                 'object': 'dict', 'null': 'NoneType', 'decimal64': 'Decimal',
260                 'ndarray': 'ndarray', 'narray': 'narray'}
261    DT_PYTHON = {val: key for key, val in PYTHON_DT.items()}
262
263    #OTHER_DT = {'boolean': 'bool', 'string': 'str'}
264    OTHER_DT = {'boolean': 'bool', 'string': 'str', 'base16': 'bytes'}
265    DT_OTHER = {val: key for key, val in OTHER_DT.items()}
266
267    LOCATION_DT = {'point': 'Point',
268                   'line': 'LineString', 'polygon': 'Polygon'}
269    DT_LOCATION = {val: key for key, val in LOCATION_DT.items()}
270
271    NUMBER_DT = {'json': 'object', 'number': None, 'month': 'int', 'day': 'int',
272                 'wday': 'int', 'yday': 'int', 'week': 'hour', 'minute': 'int',
273                 'second': 'int'}
274    #STRING_DT = {'base16': 'str', 'base32': 'str', 'base64': 'str',
275    STRING_DT = {'base32': 'str', 'base64': 'str',
276                 'period': 'str', 'duration': 'str', 'jpointer': 'str',
277                 'uri': 'str', 'uriref': 'str', 'iri': 'str', 'iriref': 'str',
278                 'email': 'str', 'regex': 'str', 'hostname': 'str', 'ipv4': 'str',
279                 'ipv6': 'str', 'file': 'str', 'geojson': 'str', }
280    FORMAT_CLS = {'full': Dfull, 'complete': Dcomplete}
281    CONVERT_DT = {'object': 'object', 'array': 'object', 'json': 'object',
282                  'number': 'float', 'boolean': 'bool', 'null': 'object',
283                  'string': 'str', 'integer': 'int'}
284    STRUCT_DT = {'Ntv': 'object', 'NtvSingle': 'object', 'NtvList': 'object'}
285
286    DT_NTVTYPE = DT_DATATION | DT_LOCATION | DT_OTHER | DT_CONNECTOR | DT_PYTHON"""
287
288    @staticmethod
289    def equals(nself, nother):
290        '''return True if all elements are equals and dtype are equal'''
291        if not (isinstance(nself, np.ndarray) and isinstance(nother, np.ndarray)):
292            return False
293        if nself.dtype != nother.dtype or nself.shape != nother.shape:
294            return False
295        if len(nself.shape) == 0:
296            return True
297        if len(nself) != len(nother):
298            return False
299        if len(nself) == 0:
300            return True
301        if isinstance(nself[0], (np.ndarray, pd.Series, pd.DataFrame)):
302            SeriesConnec = NtvConnector.connector().get('SeriesConnec')
303            DataFrameConnec = NtvConnector.connector().get('DataFrameConnec')
304            equal = {np.ndarray: Dutil.equals,
305                     pd.Series: SeriesConnec.equals,
306                     pd.DataFrame: DataFrameConnec.equals}
307            for nps, npo in zip(nself, nother):
308                if not equal[type(nself[0])](nps, npo):
309                    return False
310            return True
311        return np.array_equal(nself, nother)
312
313    @staticmethod
314    def list_json(nda):
315        '''return a JSON representation of a unidimensional np.ndarray'''
316        if len(nda) == 0:
317            return []
318        if isinstance(nda[0], np.ndarray):
319            return [Dutil.list_json(arr) for arr in nda]
320        return nda.tolist()
321
322    """@staticmethod
323    def convert(ntv_type, nda, tojson=True, convert=True):
324        ''' convert np.ndarray with external NTVtype.
325
326        *Parameters*
327
328        - **ntv_type** : string - NTVtype deduced from the np.ndarray name_type and dtype,
329        - **nda** : np.ndarray to be converted.
330        - **tojson** : boolean (default True) - apply to json function
331        - **convert** : boolean (default True) - If True, convert json data with
332        non Numpy ntv_type into data with python type
333        '''
334        if tojson:
335            match ntv_type:
336                case dat if dat in Dutil.DATATION_DT:
337                    return nda.astype(Dutil.DATATION_DT[dat]).astype(str)
338                case 'bytes':
339                    return nda.astype('bytes').astype(str)
340                case 'time':
341                    return nda.astype(str)
342                case 'decimal64':
343                    return nda.astype(float)
344                case 'geojson':
345                    return np.frompyfunc(ShapelyConnec.to_geojson, 1, 1)(nda)
346                case _:
347                    return nda
348        else:
349            match [ntv_type, convert]:
350                case [None, _]:
351                    return nda
352                case [dat, _] if dat in Dutil.DATATION_DT:
353                    return nda.astype(Dutil.DATATION_DT[dat])
354                case [std, _] if std in Dutil.OTHER_DT:
355                    return nda.astype(Dutil.OTHER_DT[std])
356                case ['time', True]:
357                    return np.frompyfunc(datetime.time.fromisoformat, 1, 1)(nda)
358                case ['decimal64', True]:
359                    return np.frompyfunc(Decimal, 1, 1)(nda)
360                case ['narray', True]:
361                    nar = np.frompyfunc(Ndarray.read_json, 1, 1)(nda)
362                    return np.frompyfunc(Ndarray.to_ndarray, 1, 1)(nar)
363                case ['ndarray', True]:
364                    return np.frompyfunc(Ndarray.read_json, 1, 1)(nda)
365                case [python, _] if python in Dutil.PYTHON_DT:
366                    return nda.astype('object')
367                case [connec, True] if connec in Dutil.CONNECTOR_DT:
368                    return np.fromiter([NtvConnector.uncast(nd, None, connec)[0]
369                                        for nd in nda], dtype='object')
370                case [('point' | 'line' | 'polygon' | 'geometry'), True]:
371                    return np.frompyfunc(ShapelyConnec.to_geometry, 1, 1)(nda)
372                case [_, False]:
373                    return nda.astype(Dutil.CONVERT_DT[
374                        Dutil.dtype(ntv_type, convert=False)])
375                case _:
376                    return nda.astype(Dutil.dtype(ntv_type))
377
378        # float.fromhex(x.hex()) == x, bytes(bytearray.fromhex(x.hex())) == x"""