ntv-numpy.ntv_numpy.ndarray

  1# -*- coding: utf-8 -*-
  2"""
  3@author: Philippe@loco-labs.io
  4
  5The `ndarray` module is part of the `ntv-numpy.ntv_numpy` package
  6([specification document](
  7https://loco-philippe.github.io/ES/JSON%20semantic%20format%20(JSON-NTV).htm)).
  8
  9It contains the classes `Ndarray`, `Nutil`, `NdarrayError` for the JSON interface
 10of numpy.ndarrays.
 11
 12For more information, see the
 13[user guide](https://loco-philippe.github.io/ntv-numpy/docs/user_guide.html)
 14 or the [github repository](https://github.com/loco-philippe/ntv-numpy).
 15
 16"""
 17
 18import datetime
 19import json
 20
 21from decimal import Decimal
 22import numpy as np
 23from json_ntv import Ntv, ShapelyConnec, NtvConnector  # , Datatype
 24from ntv_numpy.data_array import Dfull, Dcomplete, Darray, Dutil
 25from ntv_numpy.ndtype import Ndtype, NP_NTYPE
 26
 27
 28class Ndarray:
 29    ''' The Ndarray class is the JSON interface of numpy.ndarrays.
 30
 31    *static methods*
 32    - `read_json`
 33    - `to_json`
 34    - `set_shape`
 35    '''
 36
 37    def __init__(self, dar, ntv_type=None, shape=None, str_uri=True):
 38        '''Ndarray constructor.
 39
 40        *Parameters*
 41
 42        - **dar**: Darray or np.ndarray - data to represent
 43        - **shape** : list of integer (default None) - length of dimensions
 44        - **ntv_type**: string (default None) - NTVtype to apply
 45        - **str_uri**: boolean(default True) - if True and dar is a string,
 46        dar is an uri else a np.array
 47        '''
 48        dar = [None] if isinstance(dar, list) and len(dar) == 0 else dar
 49        if isinstance(dar, Ndarray):
 50            self.uri = dar.uri
 51            self.is_json = dar.is_json
 52            self.ntvtype = dar.ntvtype
 53            self.shape = dar.shape
 54            self.darray = dar.darray
 55            return
 56        if isinstance(dar, str) and str_uri:
 57            self.uri = dar
 58            self.is_json = True
 59            self.ntvtype = Ndtype(ntv_type) if ntv_type else None
 60            self.shape = shape
 61            self.darray = None
 62            return
 63        if shape:
 64            dar = Dfull(dar, dtype=Nutil.dtype(ntv_type), unidim=True).data
 65        else:
 66            dar = np.array(dar, dtype=Nutil.dtype(ntv_type))
 67            shape = list(dar.shape)
 68        dar = np.array(dar).reshape(-1)
 69        ntv_type = Nutil.nda_ntv_type(dar, ntv_type)
 70        self.uri = None
 71        self.is_json = Nutil.is_json(dar[0])
 72        self.ntvtype = Ndtype(ntv_type)
 73        self.shape = shape
 74        self.darray = dar.astype(Nutil.dtype(str(self.ntvtype)))
 75
 76    def __repr__(self):
 77        '''return classname, the shape and the ntv_type'''
 78        uri = self.uri if self.uri else ''
 79        typ = self.ntv_type if self.ntv_type else ''
 80        sha = str(self.shape) if self.shape else ''
 81        u_t = ', ' if uri and typ + sha else ''
 82        t_s = ', ' if typ and sha else ''
 83        return self.__class__.__name__ + '(' + uri + u_t + typ + t_s + sha + ')'
 84
 85    def __str__(self):
 86        '''return json string format'''
 87        return json.dumps(self.to_json())
 88
 89    def __eq__(self, other):
 90        ''' equal if attributes are equal'''
 91        if self.ntv_type != other.ntv_type:
 92            return False
 93        if self.uri != other.uri:
 94            return False
 95        if self.shape != other.shape:
 96            return False
 97        if self.darray is None and other.darray is None:
 98            return True
 99        if self.darray is None or other.darray is None:
100            return False
101        return Dutil.equals(self.darray, other.darray)
102
103    def __len__(self):
104        ''' len of ndarray'''
105        return len(self.darray) if self.darray is not None else 0
106
107    def __contains__(self, item):
108        ''' item of darray values'''
109        return item in self.darray if self.darray is not None else None
110
111    def __getitem__(self, ind):
112        ''' return darray value item'''
113        if self.darray is None:
114            return None
115        if isinstance(ind, tuple):
116            return [self.darray[i] for i in ind]
117        return self.darray[ind]
118
119    def __copy__(self):
120        ''' Copy all the data '''
121        return self.__class__(self)
122
123    def __array__(self):
124        '''numpy array interface'''
125        return self.ndarray
126
127    @property
128    def ntv_type(self):
129        ''' string representation of ntvtype'''
130        return str(self.ntvtype) if self.ntvtype else None
131
132    @property
133    def ndarray(self):
134        '''representation with a np.ndarray not flattened'''
135        return self.darray.reshape(self.shape) if self.darray is not None else None
136
137    def set_shape(self, shape):
138        '''update the shape'''
139        if Ndarray.len_shape(shape) != len(self.darray):
140            raise NdarrayError(
141                "shape is not consistent with the ndarray length")
142        self.shape = list(shape)
143
144    def update(self, nda, nda_uri=True):
145        '''update uri and darray and return the result (True, False)
146
147        *Parameters*
148
149        - **nda** : string, list, np.ndarray, Ndarray - data to include
150        - **nda_uri** : boolean (default True) - if True, existing shape and
151        ntv_type are not updated (but are created if not existing)'''
152        if not nda_uri and not (self.shape is None or nda.shape is None
153                                ) and self.shape != nda.shape:
154            return False
155        if not nda_uri and not (self.ntv_type is None or nda.ntv_type is None
156                                ) and self.ntv_type != nda.ntv_type:
157            return False
158        if nda_uri:
159            len_s = self.len_shape(self.shape)
160            if len_s and len(nda) and len_s != len(nda):
161                return False
162            self.ntvtype = nda.ntvtype if self.ntv_type is None else self.ntvtype
163            self.shape = nda.shape if self.shape is None else self.shape
164        else:
165            self.ntvtype = nda.ntvtype if nda.ntv_type is not None else self.ntvtype
166            self.shape = nda.shape if nda.shape is not None else self.shape
167        self.uri, self.darray = (
168            nda.uri, None) if nda.uri else (None, nda.darray)
169        return True
170
171    def set_array(self, darray):
172        '''set a new darray and remove uri, return the result (True, False)
173
174        *Parameters*
175
176        - **darray** : list, np.ndarray, Ndarray - data to include'''
177        ndarray = Ndarray(darray)
178        darray = ndarray.darray
179        ntv_type = ndarray.ntv_type
180        shape = ndarray.shape
181        new_shape = shape if self.shape is None else self.shape
182        new_ntv_type = ntv_type if self.ntv_type is None else self.ntv_type
183        if (len(darray) != Ndarray.len_shape(new_shape) or
184                new_ntv_type != ntv_type or new_shape != shape):
185            return False
186        self.uri = None
187        self.darray = darray
188        self.ntvtype = Ndtype(new_ntv_type)
189        self.shape = new_shape
190        return True
191
192    def set_uri(self, uri, no_ntv_type=False, no_shape=False):
193        '''set a new uri and remove ndarray and optionaly ntv_type and shape.
194        Return the result (True, False)
195
196        *Parameters*
197
198        - **uri** : string - URI of the Ndarray
199        - **no_ntv_type** : boolean (default False) - If True, ntv_type is None
200        - **no_shape** : boolean (default False) - If True, shape is None
201        '''
202        if not isinstance(uri, str) or not uri:
203            return False
204        self.uri = uri
205        self.darray = None
206        self.ntvtype = None if no_ntv_type else self.ntvtype
207        self.shape = None if no_shape else self.shape
208        return True
209
210    def to_ndarray(self):
211        '''representation with a np.ndarray not flattened'''
212        return self.ndarray
213
214    @property
215    def mode(self):
216        '''representation mode of the darray/uri data (relative, absolute,
217        undefined, inconsistent)'''
218        match [self.darray, self.uri]:
219            case [None, str()]:
220                return 'relative'
221            case [None, None]:
222                return 'undefined'
223            case [_, None]:
224                return 'absolute'
225            case _:
226                return 'inconsistent'
227
228    @staticmethod
229    def read_json(jsn, **kwargs):
230        ''' convert json ntv_value into a ndarray.
231
232        *Parameters*
233
234        - **convert** : boolean (default True) - If True, convert json data with
235        non Numpy ntv_type into data with python type
236        '''
237        option = {'convert': True} | kwargs
238        jso = json.loads(jsn) if isinstance(jsn, str) else jsn
239        ntv_value, = Ntv.decode_json(jso)[:1]
240
241        ntv_type = None
242        shape = None
243        match ntv_value[:-1]:
244            case []: ...
245            case [ntv_type, shape]: ...
246            case [str(ntv_type)]: ...
247            case [list(shape)]: ...
248        unidim = shape is not None
249        if isinstance(ntv_value[-1], str):
250            return Ndarray(ntv_value[-1], shape=shape, ntv_type=ntv_type)
251        darray = Darray.read_json(ntv_value[-1], dtype=Nutil.dtype(ntv_type),
252                                  unidim=unidim)
253        darray.data = Nutil.convert(ntv_type, darray.data, tojson=False,
254                                    convert=option['convert'])
255        return Ndarray(darray.values, shape=shape, ntv_type=ntv_type)
256
257    def to_json(self, **kwargs):
258        ''' convert a Ndarray into json-value
259
260        *Parameters*
261
262        - **noshape** : Boolean (default True) - if True, without shape if dim < 1
263        - **notype** : Boolean (default False) - including data type if False
264        - **novalue** : Boolean (default False) - including value if False
265        - **format** : string (default 'full') - representation format of the ndarray,
266        - **encoded** : Boolean (default False) - json-value if False else json-text
267        - **header** : Boolean (default True) - including ndarray type
268        '''
269        option = {'format': 'full', 'header': True, 'encoded': False,
270                  'notype': False, 'noshape': True, 'novalue': False} | kwargs
271        if self.mode in ['undefined', 'inconsistent']:
272            return None
273        if self.mode == 'absolute' and len(self.darray) == 0:
274            return [[]]
275
276        shape = None if not self.shape or (len(self.shape) < 2 and
277                                           option['noshape']) else self.shape
278
279        if self.mode == 'relative':
280            js_val = self.uri
281        else:
282            js_val = Nutil.ntv_val(self.ntv_type, self.darray, option['format'],
283                                   self.is_json) if not option['novalue'] else ['-']
284
285        lis = [self.ntv_type if not option['notype'] else None, shape, js_val]
286        return Nutil.json_ntv(None, 'ndarray',
287                              [val for val in lis if val is not None],
288                              header=option['header'], encoded=option['encoded'])
289
290    @property
291    def info(self):
292        ''' infos of the Ndarray'''
293        inf = {'shape': self.shape}
294        inf['length'] = len(self)
295        inf['ntvtype'] = self.ntv_type
296        inf['shape'] = self.shape
297        inf['uri'] = self.uri
298        return {key: val for key, val in inf.items() if val}
299
300    @staticmethod
301    def len_shape(shape):
302        '''return a length from a shape (product of dimensions)'''
303        if not shape:
304            return 0
305        prod = 1
306        for dim in shape:
307            prod *= dim
308        return prod
309
310
311class Nutil:
312    '''ntv-ndarray utilities.
313
314    *static methods*
315    - `convert`
316    - `is_json`
317    - `ntv_val`
318    - `add_ext`
319    - `split_type`
320    - `ntv_type`
321    - `nda_ntv_type`
322    - `dtype`
323    - `json_ntv`
324    - `split_name`
325    - `split_json_name`
326
327    '''
328    CONNECTOR_DT = {'field': 'Series', 'tab': 'DataFrame'}
329    PYTHON_DT = {'array': 'list', 'time': 'datetime.time',
330                 'object': 'dict', 'null': 'NoneType', 'decimal64': 'Decimal',
331                 'ndarray': 'ndarray', 'narray': 'narray'}
332    LOCATION_DT = {'point': 'Point',
333                   'line': 'LineString', 'polygon': 'Polygon'}
334    DT_CONNECTOR = {val: key for key, val in CONNECTOR_DT.items()}
335    DT_PYTHON = {val: key for key, val in PYTHON_DT.items()}
336    DT_LOCATION = {val: key for key, val in LOCATION_DT.items()}
337    DT_NTVTYPE = DT_LOCATION | DT_CONNECTOR | DT_PYTHON
338
339    FORMAT_CLS = {'full': Dfull, 'complete': Dcomplete}
340    STRUCT_DT = {'Ntv': 'object', 'NtvSingle': 'object', 'NtvList': 'object'}
341    CONVERT_DT = {'object': 'object', 'array': 'object', 'json': 'object',
342                  'number': 'float', 'boolean': 'bool', 'null': 'object',
343                  'string': 'str', 'integer': 'int'}
344
345    @staticmethod
346    def is_json(obj):
347        ''' check if obj is a json structure and return True if obj is a json-value
348
349        *Parameters*
350
351        - **obj** : object to check'''
352        if obj is None:
353            return True
354        is_js = NtvConnector.is_json
355        match obj:
356            case str() | int() | float() | bool():
357                return True
358            case list() | tuple() as obj:
359                if not obj:
360                    return True
361                return min(is_js(obj_in) for obj_in in obj)
362            case dict() as obj:
363                if not obj:
364                    return True
365                if not min(isinstance(key, str) for key in obj.keys()):
366                    return False
367                return min(is_js(obj_in) for obj_in in obj.values())
368            case _:
369                return False
370
371    @staticmethod
372    def extend_array(arr, til, shap, order):
373        '''return a flattened np.ndarray extended in additional dimensions
374
375        parameters:
376
377        - arr: np.array to extend
378        - til: integer - parameter to apply to np.tile function
379        - shap: list of integer - shape of the array
380        - order: list of integer - order of dimensions to apply
381        '''
382        old_order = list(range(len(order)))
383        arr_tab = np.tile(arr, til).reshape(shap)
384        return np.moveaxis(arr_tab, old_order, order).flatten()
385
386    @staticmethod
387    def convert(ntv_type, nda, tojson=True, convert=True):
388        ''' convert np.ndarray with external NTVtype.
389
390        *Parameters*
391
392        - **ntv_type** : string - NTVtype deduced from the np.ndarray name_type and dtype,
393        - **nda** : np.ndarray to be converted.
394        - **tojson** : boolean (default True) - apply to json function
395        - **convert** : boolean (default True) - If True, convert json data with
396        non Numpy ntv_type into data with python type
397        '''
398
399        dtype = Nutil.dtype(ntv_type)
400        jtype = Nutil.dtype(ntv_type, convert=False)
401        if tojson:
402            match ntv_type:
403                case dat if Ndtype(dat).category == 'datation':
404                    return nda.astype(dtype).astype(jtype)
405                case 'base16':
406                    return nda.astype(dtype)
407                case 'time' | 'decimal64':
408                    return nda.astype(jtype)
409                case 'geojson':
410                    return np.frompyfunc(ShapelyConnec.to_geojson, 1, 1)(nda)
411                case _:
412                    return nda
413        else:
414            match [ntv_type, convert]:
415                case [None, _]:
416                    return nda
417                case [_, False]:
418                    return nda.astype(jtype)
419                case ['time', _]:
420                    return np.frompyfunc(datetime.time.fromisoformat, 1, 1)(nda)
421                case ['decimal64', _]:
422                    return np.frompyfunc(Decimal, 1, 1)(nda)
423                case ['narray', _]:
424                    nar = np.frompyfunc(Ndarray.read_json, 1, 1)(nda)
425                    return np.frompyfunc(Ndarray.to_ndarray, 1, 1)(nar)
426                case ['ndarray', _]:
427                    return np.frompyfunc(Ndarray.read_json, 1, 1)(nda)
428                case [('point' | 'line' | 'polygon' | 'geometry'), _]:
429                    return np.frompyfunc(ShapelyConnec.to_geometry, 1, 1)(nda)
430                case [connec, _] if connec in Nutil.CONNECTOR_DT:
431                    return np.fromiter([NtvConnector.uncast(nd, None, connec)[0]
432                                        for nd in nda], dtype='object')
433                case _:
434                    return nda.astype(dtype)
435
436        # float.fromhex(x.hex()) == x, bytes(bytearray.fromhex(x.hex())) == x
437    @staticmethod
438    def ntv_val(ntv_type, nda, form, is_json=False):
439        ''' convert a np.ndarray into NTV json-value.
440
441        *Parameters*
442
443        - **ntv_type** : string - NTVtype deduced from the ndarray, name_type and dtype,
444        - **nda** : ndarray to be converted.
445        - **form** : format of data ('full', 'complete', 'sparse', 'primary').
446        - **is_json** : boolean (defaut False) - True if nda data is Json data
447        '''
448        if form == 'complete' and len(nda) < 2:
449            raise NdarrayError(
450                "complete format is not available with ndarray length < 2")
451        Format = Nutil.FORMAT_CLS[form]
452        darray = Format(nda)
453        ref = darray.ref
454        coding = darray.coding
455        if is_json:
456            return Format(darray.data, ref=ref, coding=coding).to_json()
457        match ntv_type:
458            case 'narray':
459                data = [Ndarray(nd).to_json(header=False)
460                        for nd in darray.data]
461            case 'ndarray':
462                data = [Ndarray(nd).to_json(header=False)
463                        for nd in darray.data]
464            case connec if connec in Nutil.CONNECTOR_DT:
465                data = [NtvConnector.cast(nd, None, connec)[0]
466                        for nd in darray.data]
467            case 'point' | 'line' | 'polygon' | 'geometry':
468                data = np.frompyfunc(ShapelyConnec.to_coord, 1, 1)(darray.data)
469            case None:
470                data = nda
471            case _:
472                data = Nutil.convert(ntv_type, darray.data)
473        return Format(data, ref=ref, coding=coding).to_json()
474
475    @staticmethod
476    def add_ext(typ, ext):
477        '''return extended type string: "typ[ext]"'''
478        ext = '[' + ext + ']' if ext else ''
479        return '' if not typ else typ + ext
480
481    @staticmethod
482    def split_type(typ):
483        '''return a tuple with typ and extension'''
484        if not isinstance(typ, str):
485            return (None, None)
486        spl = typ.split('[', maxsplit=1)
487        return (spl[0], None) if len(spl) == 1 else (spl[0], spl[1][:-1])
488
489    @staticmethod
490    def split_json_name(string, notnone=False):
491        '''return a tuple with name, ntv_type from string'''
492        null = '' if notnone else None
493        if not string or string == ':':
494            return (null, null)
495        spl = string.rsplit(':', maxsplit=1)
496        if len(spl) == 1:
497            return (string, null)
498        if spl[0] == '':
499            return (null, spl[1])
500        sp0 = spl[0][:-1] if spl[0][-1] == ':' else spl[0]
501        return (null if sp0 == '' else sp0, null if spl[1] == '' else spl[1])
502
503    @staticmethod
504    def split_name(string):
505        '''return a list with name, add_name from string'''
506        if not string or string == '.':
507            return ['', '']
508        spl = string.split('.', maxsplit=1)
509        spl = [spl[0], ''] if len(spl) < 2 else spl
510        return spl
511
512    @staticmethod
513    def ntv_type(dtype, ntv_type=None, ext=None):
514        ''' return ntv_type string from dtype, additional type and extension.
515
516        *Parameters*
517
518        - **dtype** : string - dtype of the ndarray
519        - **ntv_type** : string - additional type
520        - **ext** : string - type extension
521        '''
522        np_ntype = NP_NTYPE | Nutil.DT_NTVTYPE | {
523            'int': 'int', 'object': 'object'}
524        if ntv_type:
525            return Nutil.add_ext(ntv_type, ext)
526        match dtype:
527            case string if string[:3] == 'str':
528                return Nutil.add_ext('string', ext)
529            case bytesxx if bytesxx[:5] == 'bytes':
530                return Nutil.add_ext('base16', ext)
531            case dtyp if dtyp in np_ntype:
532                return Nutil.add_ext(np_ntype[dtyp], ext)
533            case date if date[:10] == 'datetime64':
534                return 'datetime' + date[10:]
535            case delta if delta[:11] == 'timedelta64':
536                return 'timedelta' + delta[11:]
537            case _:
538                return Nutil.add_ext(dtype, ext)
539
540    @staticmethod
541    def nda_ntv_type(nda, ntv_type=None, ext=None):
542        '''return ntv_type string from an ndarray, additional type and extension.
543
544        *Parameters*
545
546        - **nda** : ndarray - data used to calculate the ntv_type
547        - **ntv_type** : string - additional type
548        - **ext** : string - type extension
549        '''
550        if ntv_type or nda is None:
551            return ntv_type
552        dtype = nda.dtype.name
553        pytype = nda.flat[0].__class__.__name__
554        dtype = pytype if dtype == 'object' and pytype not in Nutil.STRUCT_DT else dtype
555        return Nutil.ntv_type(dtype, ntv_type, ext)
556
557    @staticmethod
558    def dtype(ntv_type, convert=True):
559        ''' return dtype from ntv_type
560
561        *parameters*
562
563        - **convert** : boolean (default True) - if True, dtype if from converted data
564        '''
565        if not ntv_type:
566            return None
567        if convert:
568            if ntv_type[:8] == 'datetime' and ntv_type[8:]:
569                return 'datetime64' + ntv_type[8:]
570            return Ndtype(ntv_type).dtype
571        return Nutil.CONVERT_DT[Ndtype(ntv_type).json_type]
572
573    @staticmethod
574    def json_ntv(ntv_name, ntv_type, ntv_value, **kwargs):
575        ''' return the JSON representation of a NTV entity
576
577        *parameters*
578
579        - **ntv_name** : string - name of the NTV
580        - **ntv_type** : string - type of the NTV
581        - **ntv_value** : string - Json value of the NTV
582        - **encoded** : boolean (default False) - if True return JsonText else JsonValue
583        - **header** : boolean (default True) - if True include ntv_name + ntv_type
584        '''
585        name = ntv_name if ntv_name else ''
586        option = {'encoded': False, 'header': True} | kwargs
587        if option['header'] or name:
588            typ = ':' + ntv_type if option['header'] and ntv_type else ''
589            jsn = {name + typ: ntv_value} if name + typ else ntv_value
590        else:
591            jsn = ntv_value
592        if option['encoded']:
593            return json.dumps(jsn)
594        return jsn
595
596
597class NdarrayError(Exception):
598    '''Multidimensional exception'''