python.observation.dataset_analysis

  1# -*- coding: utf-8 -*-
  2"""
  3Created on Sun Oct  2 22:24:59 2022
  4
  5@author: philippe@loco-labs.io
  6
  7The `python.observation.dataset_analysis` module contains the `Analysis` class.
  8
  9"""
 10
 11# %% declarations
 12from copy import copy
 13import pprint
 14from collections import Counter
 15
 16from observation.util import util
 17
 18
 19class Analysis:
 20    '''This class analyses relationships included in a tabular object 
 21    (Pandas DataFrame, Dataset, Observation, list of list).
 22
 23    The Analysis class includes the following functions:
 24    - identification and qualification of the relationships between Field,
 25    - generation of the global properties of the structure
 26    - data actualization based on structure updates
 27
 28    *Attributes* :
 29
 30    - **iobj** : Dataset or Observation associated to the Analysis object
 31    - **hashi** : internal Id of the iobj
 32    - **matrix** : square matrix with relationship properties between two fields
 33    - **infos** : list of characteristics (matrix synthesis)
 34    - **primary** : list of 'primary' fields row 
 35    - **secondary** : list of 'secondary' fields row 
 36    - **lvarname** : list of 'variable' fields name 
 37
 38    The methods defined in this class are :
 39
 40    - `Analysis.actualize`
 41    - `Analysis.actualize`
 42    - `Analysis.check_relationship`
 43    - `Analysis.getmatrix`
 44    - `Analysis.getvarname`
 45    - `Analysis.getsecondary`
 46    - `Analysis.getprimary`
 47    - `Analysis.getpartition`
 48    - `Analysis.tree`
 49    '''
 50    # %% methods
 51
 52    def __init__(self, iobj):
 53        '''Analysis constructor.
 54
 55         *Parameters*
 56
 57        - **iobj** : object - tabular object (Pandas DataFrame, Dataset, Observation, 
 58        list of list)
 59
 60        Note: The Analysis data can be update only if tabular object is Dataset or 
 61        Observation.
 62        '''
 63        if iobj.__class__.__name__ in ('Dataset', 'Observation', 'Ndataset', 'Sdataset'):
 64            self.iobj = iobj
 65        elif iobj.__class__.__name__ == 'DataFrame':
 66            from observation import Sdataset
 67            self.iobj = Sdataset(iobj)
 68        else:
 69            from dataset import Dataset
 70            self.iobj = Dataset.obj(iobj)
 71        self.hashi = None
 72        self.matrix = None
 73        self.infos = None
 74        self.primary = None
 75        self.secondary = None
 76        self.lvarname = None
 77        self.partition = []
 78        self.groups = []
 79
 80    def actualize(self, partition=None):
 81        ''' update all data with new values of iobj
 82
 83         *Parameters*
 84
 85        - **partition** : list of int (default None) - partition to be used '''
 86        self.matrix = self._setmatrix()
 87        self._setinfos()
 88        self._setparent()
 89        self._setgroups()
 90        self._setpartition()
 91        self._setinfospartition(partition)
 92        self.hashi = self.iobj._hashi()
 93        self.lvarname = [idx['name']
 94                         for idx in self.infos if idx['cat'] == 'variable']
 95        coupledvar = [idx['name'] for idx in self.infos if idx['cat'] == 'coupled'
 96                      and self.infos[idx['parent']]['cat'] == 'variable']
 97        self.lvarname += coupledvar
 98        self.secondary = [idx['num']
 99                          for idx in self.infos if idx['cat'] == 'secondary']
100        coupledsec = [idx['num'] for idx in self.infos if idx['cat'] == 'coupled'
101                      and self.infos[idx['parent']]['cat'] in ('primary', 'secondary')]
102        self.secondary += coupledsec
103        #infosidx = [idx for idx in self.infos if idx['cat'] != 'variable']
104        infosidx = [idx for idx in self.infos if idx['cat'] != 'variable' and 
105                    not (idx['cat'] == 'coupled' and 
106                         self.infos[idx['parent']]['cat'] == 'variable') ]
107        self.primary = [infosidx.index(idx)
108                        for idx in infosidx if idx['cat'] == 'primary']
109
110    def check_relationship(self, relations):
111        '''get the list of inconsistent records for each relationship defined in relations
112
113         *Parameters*
114
115        - **relations** : list of dict - list of fields with relationship property
116        
117        *Returns* : dict with for each relationship: key = pair of name, 
118        and value = list of inconsistent records'''
119        if not isinstance(relations, (list, dict)):
120            raise AnalysisError("relations is not correct")
121        if isinstance(relations, dict):
122            relations = [relations]
123        dic_res = {}
124        for field in relations:
125            if not 'relationship' in field or not 'name' in field:
126                continue
127            if not 'parent' in field['relationship'] or not 'link' in field['relationship']:
128                raise AnalysisError("relationship is not correct")
129            rel = field['relationship']['link']
130            f_parent = self.iobj.nindex(field['relationship']['parent'])
131            f_field = self.iobj.nindex(field['name'])
132            name_rel = field['name'] + ' - ' + field['relationship']['parent']
133            if f_parent is None or f_field is None:
134                raise AnalysisError("field's name are  not present in data")
135            match rel:
136                case 'derived':
137                    dic_res[name_rel] = f_parent.coupling(f_field, reindex=True)                
138                case 'coupled':
139                    dic_res[name_rel] = f_parent.coupling(f_field, derived=False, reindex=True)    
140                case _:
141                    raise AnalysisError(rel + "is not a valid relationship")
142        return dic_res          
143    
144    def getinfos(self, keys=None):
145        '''return attribute infos
146
147         *Parameters*
148
149        - **keys** : string, list or tuple (default None) - list of attributes to returned
150        if 'all' or None, all attributes are returned
151        if 'struct', only structural attributes are returned'''
152        if self.hashi != self.iobj._hashi():
153            self.actualize()
154        if keys == 'struct':
155            keys = ['num', 'name', 'cat', 'child', 'crossed', 'distparent',
156                    'diffdistparent', 'parent', 'pparent', 'rateder', 'ratecpl']
157        if not keys or keys == 'all':
158            return self.infos
159        return [{k: v for k, v in inf.items() if k in keys} for inf in self.infos]
160
161    def getmatrix(self, name=None):
162        '''return attribute matrix or only one value of the matrix defined by two names
163
164         *Parameters*
165
166        - **name** : list or tuple (default None) - list of two fields names        
167        '''
168        if self.hashi != self.iobj._hashi():
169            self.actualize()
170        if not name or not isinstance(name, list):
171            return self.matrix
172        if name[0] in self.iobj.lname:
173            ind0 = self.iobj.lname.index(name[0])
174            if len(name) == 1:
175                return self.matrix[ind0]
176            if len(name) > 1 and name[1] in self.iobj.lname:
177                return self.matrix[ind0][self.iobj.lname.index(name[1])]
178        return None
179
180    def getvarname(self):
181        '''return variable Field name'''
182        if self.hashi != self.iobj._hashi():
183            self.actualize()
184        return self.lvarname
185
186    def getprimary(self):
187        '''return attribute primary'''
188        if self.hashi != self.iobj._hashi():
189            self.actualize()
190        return self.primary
191
192    def getsecondary(self):
193        '''return attribute secondary'''
194        if self.hashi != self.iobj._hashi():
195            self.actualize()
196        return self.secondary
197
198    def getpartition(self):
199        '''return attribute partition'''
200        if self.hashi != self.iobj._hashi():
201            self.actualize()
202        return self.partition
203
204    def getgroups(self):
205        '''return attribute groups'''
206        if self.hashi != self.iobj._hashi():
207            self.actualize()
208        return self.groups
209
210    def tree(self, mode='derived', width=5, lname=20, string=True):
211        '''return a string with a tree of derived Field.
212
213         *Parameters*
214
215        - **lname** : integer (default 20) - length of the names        
216        - **width** : integer (default 5) - length of the lines        
217        - **mode** : string (default 'derived') - kind of tree :
218            'derived' : derived tree
219            'distance': min distance tree
220            'diff': min dist rate tree
221        '''
222        if mode == 'derived':
223            modeparent = 'parent'
224        elif mode == 'distance':
225            modeparent = 'minparent'
226        elif mode == 'diff':
227            modeparent = 'distparent'
228        else:
229            raise AnalysisError('mode is unknown')
230        if self.hashi != self.iobj._hashi():
231            self.actualize()
232        child = [None] * (len(self.infos) + 1)
233        for i in range(len(self.infos)):
234            parent = self.infos[i][modeparent]
235            if child[parent + 1] is None:
236                child[parent + 1] = []
237            child[parent + 1].append(i)
238        tr = self._dic_noeud(-1, child, lname, mode)
239        if string:
240            tre = pprint.pformat(tr, indent=0, width=width)
241            tre = tre.replace('---', ' - ')
242            tre = tre.replace('  ', ' ')
243            tre = tre.replace('*', ' ')
244            for c in ["'", "\"", "{", "[", "]", "}", ","]:
245                tre = tre.replace(c, "")
246            return tre
247        return tr
248
249    # %% internal methods
250    def _setmatrix(self):
251        '''set and return matrix attributes (coupling infos between each idx)'''
252        lenindex = self.iobj.lenindex
253        mat = [[None for i in range(lenindex)] for i in range(lenindex)]
254        for i in range(lenindex):
255            for j in range(i, lenindex):
256                mat[i][j] = self.iobj.lindex[i].couplinginfos(
257                    self.iobj.lindex[j])
258            for j in range(i):
259                mat[i][j] = copy(mat[j][i])
260                if mat[i][j]['typecoupl'] == 'derived':
261                    mat[i][j]['typecoupl'] = 'derive'
262                elif mat[i][j]['typecoupl'] == 'derive':
263                    mat[i][j]['typecoupl'] = 'derived'
264                elif mat[i][j]['typecoupl'] == 'linked':
265                    mat[i][j]['typecoupl'] = 'link'
266                elif mat[i][j]['typecoupl'] == 'link':
267                    mat[i][j]['typecoupl'] = 'linked'
268        return mat
269
270    def _setinfos(self):
271        '''set and return attribute 'infos'. 
272        Infos is an array with infos of each index :
273            - num, name, cat, child, crossed, distparent, diffdistparent, 
274            parent, pparent, rateder'''
275        lenindex = self.iobj.lenindex
276        leniobj = len(self.iobj)
277        self.infos = [{} for i in range(lenindex)]
278        for i in range(lenindex):
279            self.infos[i]['num'] = i
280            self.infos[i]['name'] = self.iobj.lname[i]
281            self.infos[i]['cat'] = 'null'
282            self.infos[i]['parent'] = -1
283            self.infos[i]['distparent'] = -1
284            self.infos[i]['minparent'] = -1
285            self.infos[i]['pparent'] = -2
286            self.infos[i]['diffdistparent'] = -1
287            self.infos[i]['distance'] = leniobj * leniobj
288            self.infos[i]['ratecpl'] = 1
289            self.infos[i]['rateder'] = 1
290            self.infos[i]['child'] = []
291            self.infos[i]['crossed'] = []
292            self.infos[i] |= self.iobj.lindex[i].infos
293            if self.infos[i]['typecodec'] == 'unique':
294                self.infos[i]['pparent'] = -1
295                self.infos[i]['cat'] = 'unique'
296                self.infos[i]['diffdistparent'] = leniobj - 1
297                self.infos[i]['rateder'] = 0
298        for i in range(lenindex):
299            for j in range(i+1, lenindex):
300                if self.matrix[i][j]['typecoupl'] == 'coupled' and \
301                        self.infos[j]['parent'] == -1:
302                    self.infos[j]['parent'] = i
303                    self.infos[j]['distparent'] = i
304                    self.infos[j]['diffdistparent'] = 0
305                    self.infos[j]['rateder'] = 0
306                    self.infos[j]['cat'] = 'coupled'
307                    self.infos[i]['child'].append(j)
308        return
309
310    def _setinfospartition(self, partition=None):
311        '''add partition data into infos attribute'''
312        if not partition is None and not partition in self.partition:
313            raise AnalysisError('partition is not a valid partition')
314        lenindex = self.iobj.lenindex
315        infosp = self.infos
316        if not partition and len(self.partition) > 0:
317            partition = self.partition[0]
318        if partition:
319            for i in partition:
320                infosp[i]['cat'] = 'primary'
321                infosp[i]['pparent'] = i
322        for i in range(lenindex):
323            if infosp[i]['cat'] == 'null':
324                util.pparent2(i, infosp)
325                if infosp[i]['pparent'] == -1 and partition:
326                    infosp[i]['cat'] = 'variable'
327                else:
328                    infosp[i]['cat'] = 'secondary'
329        for i in range(lenindex):
330            if infosp[i]['cat'] == 'coupled':
331                infosp[i]['pparent'] = infosp[infosp[i]['parent']]['pparent']
332
333    def _setparent(self):
334        '''set parent (Field with minimal diff) for each Field'''
335        # parent : min(diff) -> child
336        # distparent : min(rateder) -> diffdistparent, rateder(rateA)
337        # minparent : min(distance) -> rate(rateB), distance
338        lenindex = self.iobj.lenindex
339        leniobj = len(self.iobj)
340        for i in range(lenindex):
341            mindiff = leniobj
342            ratedermin = 1
343            distancemin = leniobj * leniobj
344            distparent = None
345            minparent = None
346            parent = None
347            infoi = self.infos[i]
348            for j in range(lenindex):
349                matij = self.matrix[i][j]
350                if not infoi['cat'] in ['unique', 'coupled']:
351                    if i != j and not i in self._listparent(j, 'parent') and \
352                            matij['typecoupl'] in ('coupled', 'derived') and \
353                            matij['diff'] < mindiff:
354                        mindiff = matij['diff']
355                        parent = j
356                    elif i != j and matij['typecoupl'] == 'crossed' and \
357                            self.infos[j]['cat'] != 'coupled':
358                        infoi['crossed'].append(j)
359                    if i != j and not i in self._listparent(j, 'distparent') and \
360                        matij['typecoupl'] in ('coupled', 'derived', 'linked', 'crossed') and \
361                       matij['rateder'] < ratedermin:
362                        ratedermin = matij['rateder']
363                        distparent = j
364                if i != j and not i in self._listparent(j, 'minparent') and \
365                    matij['distance'] < distancemin and \
366                    infoi['lencodec'] <= self.infos[j]['lencodec'] and \
367                        self.infos[j]['cat'] != 'coupled':
368                    distancemin = matij['distance']
369                    minparent = j
370            if not infoi['cat'] in ['unique', 'coupled']:
371                if not parent is None:
372                    infoi['parent'] = parent
373                    self.infos[parent]['child'].append(i)
374                if not distparent is None:
375                    infoi['distparent'] = distparent
376                    infoi['diffdistparent'] = self.matrix[i][distparent]['diff']
377                    infoi['rateder'] = self.matrix[i][distparent]['rateder']
378            if not minparent is None:
379                infoi['minparent'] = minparent
380                infoi['distance'] = self.matrix[i][minparent]['distance']
381                infoi['ratecpl'] = self.matrix[i][minparent]['ratecpl']
382            else:
383                infoi['distance'] = leniobj - infoi['lencodec']
384        return
385
386    def _listparent(self, idx, typeparent):
387        parent = idx
388        listparent = []
389        while not parent is None and parent >= 0:
390            parent = self.infos[parent][typeparent]
391            if not parent is None and parent >= 0:
392                listparent.append(parent)
393        return listparent
394
395    def _dic_noeud(self, n, child, lname, mode):
396        '''generate a dict with nodes data defined by 'child' '''
397        if n == -1:
398            lis = ['root-' + mode + '*(' + str(len(self.iobj)) + ')']
399        else:
400            adding = ''
401            if mode == 'distance':
402                adding = str(self.infos[n]['distance']) + ' - '
403            elif mode == 'diff':
404                adding = str(format(self.infos[n]['rateder'], '.2e')) + ' - '
405            adding += str(self.infos[n]['lencodec'])
406            name = self.infos[n]['name'] + ' (' + adding + ')'
407            lis = [name.replace(' ', '*').replace("'", '*')]
408        if child[n+1]:
409            for ch in child[n+1]:
410                if ch != n:
411                    lis.append(self._dic_noeud(ch, child, lname, mode))
412        return {str(n).ljust(2, '*'): lis}
413
414    def _setgroups(self):
415        '''set groups (list of crossed Field groups)'''
416        self.groups = []
417        crossed = {info['num'] for info in self.infos if info['crossed']}
418        remove = set()
419        for num in crossed:
420            for num2 in crossed:
421                if num != num2 and self.infos[num]['parent'] in crossed:
422                    remove.add(num)
423        crossed -= remove     
424        setcrossed = set()
425        for num in crossed:
426            info = self.infos[num]
427            if not info['name'] in setcrossed:
428                setname = {self.infos[cros]['name'] for cros in info['crossed']
429                           if cros in crossed} | {info['name']}
430                self.groups.append(setname)
431                setcrossed |= setname
432        return None
433
434    def _setpartition(self):
435        '''set partition (list of Field partitions)'''
436        brother = {idx['num']: idx['crossed']
437                   for idx in self.infos if idx['crossed']}
438        self.partition = []
439        chemin = []
440        for cros in brother:
441            chemin = []
442            self._addchemin(chemin, cros, 1, brother)
443        childroot = [idx['num'] for idx in self.infos if idx['parent'] == -1
444                     and idx['typecodec'] in ('complete', 'full')]
445        if childroot:
446            self.partition.append(childroot)
447        return None
448
449    def _addchemin(self, chemin, node, lchemin, brother):
450        '''extend 'chemin' with new nodes and add it to 'partition' '''
451        if lchemin == len(self.iobj) and node == chemin[0] and \
452                max(Counter(zip(*[self.iobj.lindex[idx].keys for idx in chemin])).values()) == 1:
453            part = sorted(chemin)
454            if not part in self.partition:
455                if not self.partition or len(part) > len(self.partition[0]):
456                    self.partition.insert(0, part)
457                else:
458                    self.partition.append(part)
459        if node in chemin[1:]:
460            return
461        lnode = self.infos[node]['lencodec']
462        if lchemin * lnode <= len(self.iobj):
463            newchemin = chemin + [node]
464            for broth in brother[node]:
465                self._addchemin(newchemin, broth, lchemin * lnode, brother)
466
467
468class AnalysisError(Exception):
469    ''' Analysis Exception'''
470    # pass