ntv-numpy.ntv_numpy.ndarray

  1# -*- coding: utf-8 -*-
  2"""
  3@author: Philippe@loco-labs.io
  4
  5The `ndarray` module is part of the `ntv-numpy.ntv_numpy` package
  6([specification document](
  7https://loco-philippe.github.io/ES/JSON%20semantic%20format%20(JSON-NTV).htm)).
  8
  9It contains the classes `Ndarray`, `Nutil`, `NdarrayError` for the JSON interface
 10of numpy.ndarrays.
 11
 12For more information, see the
 13[user guide](https://loco-philippe.github.io/ntv-numpy/docs/user_guide.html)
 14 or the [github repository](https://github.com/loco-philippe/ntv-numpy).
 15
 16"""
 17
 18import datetime
 19import json
 20
 21from decimal import Decimal
 22import numpy as np
 23from json_ntv import Ntv, ShapelyConnec, Datatype, NtvConnector
 24from ntv_numpy.data_array import Dfull, Dcomplete, Darray, Dutil
 25
 26
 27class Ndarray:
 28    ''' The Ndarray class is the JSON interface of numpy.ndarrays.
 29
 30    *static methods*
 31    - `read_json`
 32    - `to_json`
 33    - `equals`
 34    '''
 35
 36    def __init__(self, dar, ntv_type=None, shape=None):
 37        '''Ndarray constructor.
 38
 39        *Parameters*
 40
 41        - **dar**: Darray or np.ndarray - data to represent
 42        - **shape** : list of integer (default None) - length of dimensions
 43        - **ntv_type**: string (default None) - NTVtype to apply
 44        '''
 45        dar = [None] if isinstance(dar, list) and len(dar) == 0 else dar
 46        if isinstance(dar, Ndarray):
 47            self.uri = dar.uri
 48            self.is_json = dar.is_json
 49            self.ntvtype = dar.ntvtype
 50            self.shape = dar.shape
 51            self.darray = dar.darray
 52            return
 53        if isinstance(dar, str):
 54            self.uri = dar
 55            self.is_json = True
 56            self.ntvtype = Datatype(ntv_type) if ntv_type else None
 57            self.shape = shape
 58            self.darray = None
 59            return
 60        if shape:
 61            dar = Dfull(dar, dtype=Nutil.dtype(ntv_type), unidim=True).data
 62        else:
 63            # dar = np.array(dar if isinstance(dar, (list, np.ndarray)) else [dar],
 64            dar = np.array(dar, dtype=Nutil.dtype(ntv_type))
 65            shape = list(dar.shape)
 66        dar = np.array(dar).reshape(-1)
 67        # ntv_type = Nutil.nda_ntv_type(dar) if not (ntv_type or dar is None) else ntv_type
 68        ntv_type = Nutil.nda_ntv_type(dar, ntv_type)
 69        self.uri = None
 70        self.is_json = Nutil.is_json(dar[0])
 71        self.ntvtype = Datatype(ntv_type)
 72        self.shape = shape
 73        self.darray = dar.astype(Nutil.dtype(str(self.ntvtype)))
 74
 75    def __repr__(self):
 76        '''return classname, the shape and the ntv_type'''
 77        uri = self.uri if self.uri else ''
 78        typ = self.ntv_type if self.ntv_type else ''
 79        sha = str(self.shape) if self.shape else ''
 80        u_t = ', ' if uri and typ + sha else ''
 81        t_s = ', ' if typ and sha else ''
 82        return self.__class__.__name__ + '(' + uri + u_t + typ + t_s + sha + ')'
 83
 84    def __str__(self):
 85        '''return json string format'''
 86        return json.dumps(self.to_json())
 87
 88    def __eq__(self, other):
 89        ''' equal if attributes are equal'''
 90        if self.ntv_type != other.ntv_type:
 91            return False
 92        if self.uri != other.uri:
 93            return False
 94        if self.shape != other.shape:
 95            return False
 96        if self.darray is None and other.darray is None:
 97            return True
 98        if self.darray is None or other.darray is None:
 99            return False
100        return Dutil.equals(self.darray, other.darray)
101
102    def __len__(self):
103        ''' len of ndarray'''
104        # return len(self.darray) if self.darray is not None else Ndarray.len_shape(self.shape)
105        return len(self.darray) if self.darray is not None else 0
106
107    def __contains__(self, item):
108        ''' item of darray values'''
109        return item in self.darray if self.darray is not None else None
110
111    def __getitem__(self, ind):
112        ''' return darray value item'''
113        if self.darray is None:
114            return None
115        if isinstance(ind, tuple):
116            return [self.darray[i] for i in ind]
117        return self.darray[ind]
118
119    def __copy__(self):
120        ''' Copy all the data '''
121        return self.__class__(self)
122
123    def __array__(self):
124        '''numpy array interface'''
125        return self.ndarray
126
127    @property
128    def ntv_type(self):
129        ''' string representation of ntvtype'''
130        return str(self.ntvtype) if self.ntvtype else None
131
132    @property
133    def ndarray(self):
134        '''representation with a np.ndarray not flattened'''
135        return self.darray.reshape(self.shape) if not self.darray is None else None
136
137    def update(self, nda, nda_uri=True):
138        '''update uri and darray and return the result (True, False)
139
140        *Parameters*
141
142        - **nda** : string, list, np.ndarray, Ndarray - data to include
143        - **nda_uri** : boolean (default True) - if True, existing shape and
144        ntv_type are not updated (but are created if not existing)'''
145        if not nda_uri and not (self.shape is None or nda.shape is None
146                                ) and self.shape != nda.shape:
147            return False
148        if not nda_uri and not (self.ntv_type is None or nda.ntv_type is None
149                                ) and self.ntv_type != nda.ntv_type:
150            return False
151        if nda_uri:
152            len_s = self.len_shape(self.shape)
153            if len_s and len(nda) and len_s != len(nda):
154                return False
155            self.ntvtype = nda.ntvtype if self.ntv_type is None else self.ntvtype
156            self.shape = nda.shape if self.shape is None else self.shape
157        else:
158            self.ntvtype = nda.ntvtype if not nda.ntv_type is None else self.ntvtype
159            self.shape = nda.shape if not nda.shape is None else self.shape
160        self.uri, self.darray = (
161            nda.uri, None) if nda.uri else (None, nda.darray)
162        return True
163
164    def set_array(self, darray):
165        '''set a new darray and remove uri, return the result (True, False)
166
167        *Parameters*
168
169        - **darray** : list, np.ndarray, Ndarray - data to include'''
170        ndarray = Ndarray(darray)
171        darray = ndarray.darray
172        ntv_type = ndarray.ntv_type
173        shape = ndarray.shape
174        new_shape = shape if self.shape is None else self.shape
175        new_ntv_type = ntv_type if self.ntv_type is None else self.ntv_type
176        if (len(darray) != Ndarray.len_shape(new_shape) or
177                new_ntv_type != ntv_type or new_shape != shape):
178            return False
179        self.uri = None
180        self.darray = darray
181        self.ntvtype = Datatype(new_ntv_type)
182        self.shape = new_shape
183        return True
184
185    def set_uri(self, uri, no_ntv_type=False, no_shape=False):
186        '''set a new uri and remove ndarray and optionaly ntv_type and shape.
187        Return the result (True, False)
188
189        *Parameters*
190
191        - **uri** : string - URI of the Ndarray
192        - **no_ntv_type** : boolean (default False) - If True, ntv_type is None
193        - **no_shape** : boolean (default False) - If True, shape is None
194        '''
195        if not isinstance(uri, str) or not uri:
196            return False
197        self.uri = uri
198        self.darray = None
199        self.ntvtype = None if no_ntv_type else self.ntvtype
200        self.shape = None if no_shape else self.shape
201        return True
202
203    def to_ndarray(self):
204        '''representation with a np.ndarray not flattened'''
205        return self.ndarray
206
207    @property
208    def mode(self):
209        '''representation mode of the darray/uri data (relative, absolute,
210        undefined, inconsistent)'''
211        match [self.darray, self.uri]:
212            case [None, str()]:
213                return 'relative'
214            case [None, None]:
215                return 'undefined'
216            case [_, None]:
217                return 'absolute'
218            case _:
219                return 'inconsistent'
220
221    @staticmethod
222    def read_json(jsn, **kwargs):
223        ''' convert json ntv_value into a ndarray.
224
225        *Parameters*
226
227        - **convert** : boolean (default True) - If True, convert json data with
228        non Numpy ntv_type into data with python type
229        '''
230        option = {'convert': True} | kwargs
231        jso = json.loads(jsn) if isinstance(jsn, str) else jsn
232        ntv_value, = Ntv.decode_json(jso)[:1]
233
234        ntv_type = None
235        shape = None
236        match ntv_value[:-1]:
237            case []: ...
238            case [ntv_type, shape]: ...
239            case [str(ntv_type)]: ...
240            case [list(shape)]: ...
241        unidim = not shape is None
242        if isinstance(ntv_value[-1], str):
243            return Ndarray(ntv_value[-1], shape=shape, ntv_type=ntv_type)
244        darray = Darray.read_json(ntv_value[-1], dtype=Nutil.dtype(ntv_type),
245                                  unidim=unidim)
246        darray.data = Nutil.convert(ntv_type, darray.data, tojson=False,
247                                    convert=option['convert'])
248        return Ndarray(darray.values, shape=shape, ntv_type=ntv_type)
249
250    def to_json(self, **kwargs):
251        ''' convert a Ndarray into json-value
252
253        *Parameters*
254
255        - **noshape** : Boolean (default True) - if True, without shape if dim < 1
256        - **notype** : Boolean (default False) - including data type if False
257        - **novalue** : Boolean (default False) - including value if False
258        - **format** : string (default 'full') - representation format of the ndarray,
259        - **encoded** : Boolean (default False) - json-value if False else json-text
260        - **header** : Boolean (default True) - including ndarray type
261        '''
262        option = {'format': 'full', 'header': True, 'encoded': False,
263                  'notype': False, 'noshape': True, 'novalue': False} | kwargs
264        if self.mode in ['undefined', 'inconsistent']:
265            return None
266        if self.mode == 'absolute' and len(self.darray) == 0:
267            return [[]]
268
269        shape = None if not self.shape or (len(self.shape) < 2 and
270                                           option['noshape']) else self.shape
271
272        if self.mode == 'relative':
273            js_val = self.uri
274        else:
275            js_val = Nutil.ntv_val(self.ntv_type, self.darray, option['format'],
276                                   self.is_json) if not option['novalue'] else ['-']
277
278        lis = [self.ntv_type if not option['notype'] else None, shape, js_val]
279        return Nutil.json_ntv(None, 'ndarray',
280                              [val for val in lis if not val is None],
281                              header=option['header'], encoded=option['encoded'])
282
283    @property
284    def info(self):
285        ''' infos of the Ndarray'''
286        inf = {'shape': self.shape}
287        inf['length'] = len(self)
288        inf['ntvtype'] = self.ntv_type
289        inf['shape'] = self.shape
290        inf['uri'] = self.uri
291        return {key: val for key, val in inf.items() if val}
292
293    @staticmethod
294    def len_shape(shape):
295        '''return a length from a shape (product of dimensions)'''
296        if not shape:
297            return 0
298        prod = 1
299        for dim in shape:
300            prod *= dim
301        return prod
302
303
304class Nutil:
305    '''ntv-ndarray utilities.
306
307    *static methods*
308    - `convert`
309    - `is_json`
310    - `ntv_val`
311    - `add_ext`
312    - `split_type`
313    - `ntv_type`
314    - `nda_ntv_type`
315    - `dtype`
316    - `json_ntv`
317    - `split_name`
318    - `split_json_name`
319
320    '''
321
322    DATATION_DT = {'date': 'datetime64[D]', 'year': 'datetime64[Y]',
323                   'yearmonth': 'datetime64[M]',
324                   'datetime': 'datetime64[s]', 'datetime[ms]': 'datetime64[ms]',
325                   'datetime[us]': 'datetime64[us]', 'datetime[ns]': 'datetime64[ns]',
326                   'datetime[ps]': 'datetime64[ps]', 'datetime[fs]': 'datetime64[fs]',
327                   'timedelta': 'timedelta64[s]', 'timedelta[ms]': 'timedelta64[ms]',
328                   'timedelta[us]': 'timedelta64[us]', 'timedelta[ns]': 'timedelta64[ns]',
329                   'timedelta[ps]': 'timedelta64[ps]', 'timedelta[fs]': 'timedelta64[fs]',
330                   'timedelta[D]': 'timedelta64[D]', 'timedelta[Y]': 'timedelta64[Y]',
331                   'timedelta[M]': 'timedelta64[M]'}
332    DT_DATATION = {val: key for key, val in DATATION_DT.items()}
333
334    # CONNECTOR_DT = {'field': 'Series', 'tab': 'DataFrame'}
335    CONNECTOR_DT = {'field': 'Series', 'tab': 'DataFrame'}
336    DT_CONNECTOR = {val: key for key, val in CONNECTOR_DT.items()}
337
338    PYTHON_DT = {'array': 'list', 'time': 'datetime.time',
339                 'object': 'dict', 'null': 'NoneType', 'decimal64': 'Decimal',
340                 'ndarray': 'ndarray', 'narray': 'narray'}
341    DT_PYTHON = {val: key for key, val in PYTHON_DT.items()}
342
343    # OTHER_DT = {'boolean': 'bool', 'string': 'str'}
344    OTHER_DT = {'boolean': 'bool', 'string': 'str', 'base16': 'bytes'}
345    DT_OTHER = {val: key for key, val in OTHER_DT.items()}
346
347    LOCATION_DT = {'point': 'Point',
348                   'line': 'LineString', 'polygon': 'Polygon'}
349    DT_LOCATION = {val: key for key, val in LOCATION_DT.items()}
350
351    NUMBER_DT = {'json': 'object', 'number': None, 'month': 'int', 'day': 'int',
352                 'wday': 'int', 'yday': 'int', 'week': 'hour', 'minute': 'int',
353                 'second': 'int'}
354    # STRING_DT = {'base16': 'str', 'base32': 'str', 'base64': 'str',
355    STRING_DT = {'base32': 'str', 'base64': 'str',
356                 'period': 'str', 'duration': 'str', 'jpointer': 'str',
357                 'uri': 'str', 'uriref': 'str', 'iri': 'str', 'iriref': 'str',
358                 'email': 'str', 'regex': 'str', 'hostname': 'str', 'ipv4': 'str',
359                 'ipv6': 'str', 'file': 'str', 'geojson': 'str', }
360    FORMAT_CLS = {'full': Dfull, 'complete': Dcomplete}
361    CONVERT_DT = {'object': 'object', 'array': 'object', 'json': 'object',
362                  'number': 'float', 'boolean': 'bool', 'null': 'object',
363                  'string': 'str', 'integer': 'int'}
364    STRUCT_DT = {'Ntv': 'object', 'NtvSingle': 'object', 'NtvList': 'object'}
365
366    DT_NTVTYPE = DT_DATATION | DT_LOCATION | DT_OTHER | DT_CONNECTOR | DT_PYTHON
367
368    @staticmethod
369    def is_json(obj):
370        ''' check if obj is a json structure and return True if obj is a json-value
371
372        *Parameters*
373
374        - **obj** : object to check'''
375        if obj is None:
376            return True
377        is_js = NtvConnector.is_json
378        match obj:
379            case str() | int() | float() | bool():
380                return True
381            case list() | tuple() as obj:
382                if not obj:
383                    return True
384                return min(is_js(obj_in) for obj_in in obj)
385            case dict() as obj:
386                if not obj:
387                    return True
388                if not min(isinstance(key, str) for key in obj.keys()):
389                    return False
390                return min(is_js(obj_in) for obj_in in obj.values())
391            case _:
392                return False
393
394    @staticmethod
395    def convert(ntv_type, nda, tojson=True, convert=True):
396        ''' convert np.ndarray with external NTVtype.
397
398        *Parameters*
399
400        - **ntv_type** : string - NTVtype deduced from the np.ndarray name_type and dtype,
401        - **nda** : np.ndarray to be converted.
402        - **tojson** : boolean (default True) - apply to json function
403        - **convert** : boolean (default True) - If True, convert json data with
404        non Numpy ntv_type into data with python type
405        '''
406        if tojson:
407            match ntv_type:
408                case dat if dat in Nutil.DATATION_DT:
409                    return nda.astype(Nutil.DATATION_DT[dat]).astype(str)
410                case 'bytes':
411                    return nda.astype('bytes').astype(str)
412                case 'time':
413                    return nda.astype(str)
414                case 'decimal64':
415                    return nda.astype(float)
416                case 'geojson':
417                    return np.frompyfunc(ShapelyConnec.to_geojson, 1, 1)(nda)
418                case _:
419                    return nda
420        else:
421            match [ntv_type, convert]:
422                case [None, _]:
423                    return nda
424                case [dat, _] if dat in Nutil.DATATION_DT:
425                    return nda.astype(Nutil.DATATION_DT[dat])
426                case [std, _] if std in Nutil.OTHER_DT:
427                    return nda.astype(Nutil.OTHER_DT[std])
428                case ['time', True]:
429                    return np.frompyfunc(datetime.time.fromisoformat, 1, 1)(nda)
430                case ['decimal64', True]:
431                    return np.frompyfunc(Decimal, 1, 1)(nda)
432                case ['narray', True]:
433                    nar = np.frompyfunc(Ndarray.read_json, 1, 1)(nda)
434                    return np.frompyfunc(Ndarray.to_ndarray, 1, 1)(nar)
435                case ['ndarray', True]:
436                    return np.frompyfunc(Ndarray.read_json, 1, 1)(nda)
437                case [python, _] if python in Nutil.PYTHON_DT:
438                    return nda.astype('object')
439                case [connec, True] if connec in Nutil.CONNECTOR_DT:
440                    return np.fromiter([NtvConnector.uncast(nd, None, connec)[0]
441                                        for nd in nda], dtype='object')
442                case [('point' | 'line' | 'polygon' | 'geometry'), True]:
443                    return np.frompyfunc(ShapelyConnec.to_geometry, 1, 1)(nda)
444                case [_, False]:
445                    return nda.astype(Nutil.CONVERT_DT[
446                        Nutil.dtype(ntv_type, convert=False)])
447                case _:
448                    return nda.astype(Nutil.dtype(ntv_type))
449
450        # float.fromhex(x.hex()) == x, bytes(bytearray.fromhex(x.hex())) == x
451    @staticmethod
452    def ntv_val(ntv_type, nda, form, is_json=False):
453        ''' convert a np.ndarray into NTV json-value.
454
455        *Parameters*
456
457        - **ntv_type** : string - NTVtype deduced from the ndarray, name_type and dtype,
458        - **nda** : ndarray to be converted.
459        - **form** : format of data ('full', 'complete', 'sparse', 'primary').
460        - **is_json** : boolean (defaut False) - True if nda data is Json data
461        '''
462        if form == 'complete' and len(nda) < 2:
463            raise NdarrayError(
464                "complete format is not available with ndarray length < 2")
465        Format = Nutil.FORMAT_CLS[form]
466        darray = Format(nda)
467        ref = darray.ref
468        coding = darray.coding
469        if is_json:
470            return Format(darray.data, ref=ref, coding=coding).to_json()
471        match ntv_type:
472            case 'narray':
473                data = [Ndarray(nd).to_json(header=False)
474                        for nd in darray.data]
475            case 'ndarray':
476                # data = [Ndarray.to_json(nd) for nd in darray.data]
477                data = [Ndarray(nd).to_json(header=False)
478                        for nd in darray.data]
479            case connec if connec in Nutil.CONNECTOR_DT:
480                data = [NtvConnector.cast(nd, None, connec)[0]
481                        for nd in darray.data]
482            case 'point' | 'line' | 'polygon' | 'geometry':
483                data = np.frompyfunc(ShapelyConnec.to_coord, 1, 1)(darray.data)
484            case None:
485                data = nda
486            case _:
487                data = Nutil.convert(ntv_type, darray.data)
488        return Format(data, ref=ref, coding=coding).to_json()
489
490    @staticmethod
491    def add_ext(typ, ext):
492        '''return extended type : "typ[ext]"'''
493        ext = '[' + ext + ']' if ext else ''
494        return '' if not typ else typ + ext
495
496    @staticmethod
497    def split_type(typ):
498        '''return a tuple with typ and extension'''
499        if not isinstance(typ, str):
500            return (None, None)
501        spl = typ.split('[', maxsplit=1)
502        return (spl[0], None) if len(spl) == 1 else (spl[0], spl[1][:-1])
503
504    @staticmethod
505    def split_json_name(string, notnone=False):
506        '''return a tuple with name, ntv_type from string'''
507        null = '' if notnone else None
508        if not string or string == ':':
509            return (null, null)
510        spl = string.rsplit(':', maxsplit=1)
511        if len(spl) == 1:
512            return (string, null)
513        if spl[0] == '':
514            return (null, spl[1])
515        sp0 = spl[0][:-1] if spl[0][-1] == ':' else spl[0]
516        return (null if sp0 == '' else sp0, null if spl[1] == '' else spl[1])
517
518    @staticmethod
519    def split_name(string):
520        '''return a list with name, add_name from string'''
521        if not string or string == '.':
522            return ['', '']
523        spl = string.split('.', maxsplit=1)
524        spl = [spl[0], ''] if len(spl) < 2 else spl
525        return spl
526
527    @staticmethod
528    def ntv_type(dtype, ntv_type=None, ext=None):
529        ''' return NTVtype from dtype, additional type and extension.
530
531        *Parameters*
532
533        - **dtype** : string - dtype of the ndarray
534        - **ntv_type** : string - additional type
535        - **ext** : string - type extension
536        '''
537        if ntv_type:
538            return Nutil.add_ext(ntv_type, ext)
539        match dtype:
540            case dat if dat in Nutil.DT_NTVTYPE:
541                return Nutil.add_ext(Nutil.DT_NTVTYPE[dat], ext)
542            case string if string[:3] == 'str':
543                return Nutil.add_ext('string', ext)
544            case byte if byte[:5] == 'bytes':
545                # return Nutil.add_ext('bytes', ext)
546                return Nutil.add_ext('base16', ext)
547            case _:
548                return Nutil.add_ext(dtype, ext)
549
550    @staticmethod
551    def nda_ntv_type(nda, ntv_type=None, ext=None):
552        '''return ntv_type from an ndarray, additional type and extension.
553
554        *Parameters*
555
556        - **nda** : ndarray - data used to calculate the ntv_type
557        - **ntv_type** : string - additional type
558        - **ext** : string - type extension
559        '''
560        if ntv_type or nda is None:
561            return ntv_type
562        dtype = nda.dtype.name
563        pytype = nda.flat[0].__class__.__name__
564        dtype = pytype if dtype == 'object' and not pytype in Nutil.STRUCT_DT else dtype
565        # dtype = pytype if dtype == 'object' and pytype in Nutil.DT_NTVTYPE else dtype
566        # dtype = nda.flat[0].__class__.__name__ if dtype == 'object' else dtype
567        return Nutil.ntv_type(dtype, ntv_type, ext)
568
569    @staticmethod
570    def dtype(ntv_type, convert=True):
571        ''' return dtype from ntv_type
572
573        *parameters*
574
575        - **convert** : boolean (default True) - if True, dtype if from converted data
576        '''
577        DTYPE = (Nutil.DATATION_DT | Nutil.NUMBER_DT | Nutil.OTHER_DT |
578                 Nutil.STRING_DT)
579        OBJECT = Nutil.LOCATION_DT | Nutil.CONNECTOR_DT | Nutil.PYTHON_DT
580        type_base = Nutil.split_type(ntv_type)[0]
581        if convert:
582            if type_base in OBJECT:
583                return 'object'
584            return DTYPE.get(ntv_type, DTYPE.get(type_base, type_base))
585        return Datatype(ntv_type).json_type
586
587    @staticmethod
588    def json_ntv(ntv_name, ntv_type, ntv_value, **kwargs):
589        ''' return the JSON representation of a NTV entity
590
591        *parameters*
592
593        - **ntv_name** : string - name of the NTV
594        - **ntv_type** : string - type of the NTV
595        - **ntv_value** : string - Json value of the NTV
596        - **encoded** : boolean (default False) - if True return JsonText else JsonValue
597        - **header** : boolean (default True) - if True include ntv_name + ntv_type
598        '''
599        name = ntv_name if ntv_name else ''
600        option = {'encoded': False, 'header': True} | kwargs
601        if option['header'] or name:
602            typ = ':' + ntv_type if option['header'] and ntv_type else ''
603            jsn = {name + typ: ntv_value} if name + typ else ntv_value
604        else:
605            jsn = ntv_value
606        if option['encoded']:
607            return json.dumps(jsn)
608        return jsn
609
610
611class NdarrayError(Exception):
612    '''Multidimensional exception'''