From fb286af354ab03dbe1ad53585de3d22266a2dc51 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 31 Oct 2019 10:45:34 +0100 Subject: [PATCH 01/38] [skip sc] first draft --- processing/post/addCauchy.py | 57 +++----------------------- python/damask/__init__.py | 1 + python/damask/table.py | 77 ++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 52 deletions(-) create mode 100644 python/damask/table.py diff --git a/processing/post/addCauchy.py b/processing/post/addCauchy.py index 18c4ec215..9037567f8 100755 --- a/processing/post/addCauchy.py +++ b/processing/post/addCauchy.py @@ -1,11 +1,8 @@ #!/usr/bin/env python3 import os -import sys from optparse import OptionParser -import numpy as np - import damask @@ -37,53 +34,9 @@ parser.set_defaults(defgrad = 'f', (options,filenames) = parser.parse_args() -# --- loop over input files ------------------------------------------------------------------------- - -if filenames == []: filenames = [None] - for name in filenames: - try: - table = damask.ASCIItable(name = name, buffered = False) - except: - continue - damask.util.report(scriptName,name) - -# ------------------------------------------ read header ------------------------------------------ - - table.head_read() - -# ------------------------------------------ sanity checks ---------------------------------------- - - errors = [] - column = {} - - for tensor in [options.defgrad,options.stress]: - dim = table.label_dimension(tensor) - if dim < 0: errors.append('column {} not found.'.format(tensor)) - elif dim != 9: errors.append('column {} is not a tensor.'.format(tensor)) - else: - column[tensor] = table.label_index(tensor) - - if errors != []: - damask.util.croak(errors) - table.close(dismiss = True) - continue - -# ------------------------------------------ assemble header -------------------------------------- - - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - table.labels_append(['{}_Cauchy'.format(i+1) for i in range(9)]) # extend ASCII header with new labels - table.head_write() - -# ------------------------------------------ process data ------------------------------------------ - - outputAlive = True - while outputAlive and table.data_read(): # read next data line of ASCII table - F = np.array(list(map(float,table.data[column[options.defgrad]:column[options.defgrad]+9])),'d').reshape(3,3) - P = np.array(list(map(float,table.data[column[options.stress ]:column[options.stress ]+9])),'d').reshape(3,3) - table.data_append(list(1.0/np.linalg.det(F)*np.dot(P,F.T).reshape(9))) # [Cauchy] = (1/det(F)) * [P].[F_transpose] - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close input ASCII table (works for stdin) + table = damask.Table(name) + table.add_array('Cauchy',damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3), + table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9), + scriptID) + table.to_ASCII() diff --git a/python/damask/__init__.py b/python/damask/__init__.py index f876d1417..da699be29 100644 --- a/python/damask/__init__.py +++ b/python/damask/__init__.py @@ -9,6 +9,7 @@ name = 'damask' # classes from .environment import Environment # noqa from .asciitable import ASCIItable # noqa +from .table import Table # noqa from .config import Material # noqa from .colormaps import Colormap, Color # noqa diff --git a/python/damask/table.py b/python/damask/table.py new file mode 100644 index 000000000..81901c252 --- /dev/null +++ b/python/damask/table.py @@ -0,0 +1,77 @@ +import re + +import pandas as pd +import numpy as np + +class Table(): + """Read and write to ASCII tables""" + + def __init__(self,name): + self.name = name + with open(self.name) as f: + header,keyword = f.readline().split() + if keyword == 'header': + header = int(header) + else: + raise Exception + self.comments = [f.readline()[:-1] for i in range(header-1)] + labels_raw = f.readline().split() + self.data = pd.read_csv(f,delim_whitespace=True,header=None) + + labels_repeated = [l.split('_',1)[1] if '_' in l else l for l in labels_raw] + self.data.rename(columns=dict(zip([l for l in self.data.columns],labels_repeated)),inplace=True) + + self.shape = {} + for l in labels_raw: + tensor_column = re.search(':.*?_',l) + if tensor_column: + my_shape = tensor_column.group()[1:-1].split('x') + self.shape[l.split('_',1)[1]] = tuple([int(d) for d in my_shape]) + else: + vector_column = re.match('.*?_',l) + if vector_column: + self.shape[l.split('_',1)[1]] = (int(l.split('_',1)[0]),) + else: + self.shape[l]=(1,) + + self.labels = list(dict.fromkeys(labels_repeated)) + + + def get_array(self,label): + return self.data[label].to_numpy().reshape((-1,)+self.shape[label]) + + + def add_array(self,label,array,info): + if np.product(array.shape[1:],dtype=int) == 1: + self.comments.append('{}: {}'.format(label,info)) + + else: + self.comments.append('{} {}: {}'.format(label,array.shape[1:],info)) + + self.shape[label] = array.shape[1:] + self.labels.append(label) + size = np.product(array.shape[1:]) + new_data = pd.DataFrame(data=array.reshape(-1,size), + columns=[label for l in range(size)]) + self.data = pd.concat([self.data,new_data],axis=1) + + + def to_ASCII(self,name=None): + labels = [] + for l in self.labels: + if(self.shape[l] == (1,)): + labels.append('{}'.format(l)) + elif(len(self.shape[l]) == 1): + labels+=['{}_{}'.format(i+1,l)\ + for i in range(self.shape[l][0])] + else: + labels+=['{}:{}_{}'.format(i+1,'x'.join([str(d) for d in self.shape[l]]),l)\ + for i in range(np.product(self.shape[l]))] + + header = ['{} header'.format(len(self.comments)+1)]\ + + self.comments\ + + [' '.join(labels)] + + with open(name if name is not None else self.name,'w') as f: + for line in header: f.write(line+'\n') + self.data.to_csv(f,sep=' ',index=False,header=False) From 845cfc34ecc09fdb4e78c2a9c6378157b9cf0fe9 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Tue, 26 Nov 2019 14:26:25 +0100 Subject: [PATCH 02/38] similar logic as in geom class - filename is not part of the object - transparent handling of files, strings, and path-like objects for file IO --- processing/post/addCauchy.py | 2 +- python/damask/table.py | 70 +++++++++++++++++++----------------- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/processing/post/addCauchy.py b/processing/post/addCauchy.py index 9037567f8..3a0e14da7 100755 --- a/processing/post/addCauchy.py +++ b/processing/post/addCauchy.py @@ -39,4 +39,4 @@ for name in filenames: table.add_array('Cauchy',damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3), table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9), scriptID) - table.to_ASCII() + table.to_ASCII(name) diff --git a/python/damask/table.py b/python/damask/table.py index 81901c252..e77ae2b9c 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -5,36 +5,39 @@ import numpy as np class Table(): """Read and write to ASCII tables""" - - def __init__(self,name): - self.name = name - with open(self.name) as f: - header,keyword = f.readline().split() - if keyword == 'header': - header = int(header) + + def __init__(self,fname): + try: + f = open(fname) + except TypeError: + f = fname + + header,keyword = f.readline().split() + if keyword == 'header': + header = int(header) + else: + raise Exception + self.comments = [f.readline()[:-1] for i in range(header-1)] + labels_raw = f.readline().split() + self.data = pd.read_csv(f,delim_whitespace=True,header=None) + + labels_repeated = [l.split('_',1)[1] if '_' in l else l for l in labels_raw] + self.data.rename(columns=dict(zip([l for l in self.data.columns],labels_repeated)),inplace=True) + + self.shape = {} + for l in labels_raw: + tensor_column = re.search(':.*?_',l) + if tensor_column: + my_shape = tensor_column.group()[1:-1].split('x') + self.shape[l.split('_',1)[1]] = tuple([int(d) for d in my_shape]) else: - raise Exception - self.comments = [f.readline()[:-1] for i in range(header-1)] - labels_raw = f.readline().split() - self.data = pd.read_csv(f,delim_whitespace=True,header=None) - - labels_repeated = [l.split('_',1)[1] if '_' in l else l for l in labels_raw] - self.data.rename(columns=dict(zip([l for l in self.data.columns],labels_repeated)),inplace=True) - - self.shape = {} - for l in labels_raw: - tensor_column = re.search(':.*?_',l) - if tensor_column: - my_shape = tensor_column.group()[1:-1].split('x') - self.shape[l.split('_',1)[1]] = tuple([int(d) for d in my_shape]) + vector_column = re.match('.*?_',l) + if vector_column: + self.shape[l.split('_',1)[1]] = (int(l.split('_',1)[0]),) else: - vector_column = re.match('.*?_',l) - if vector_column: - self.shape[l.split('_',1)[1]] = (int(l.split('_',1)[0]),) - else: - self.shape[l]=(1,) - - self.labels = list(dict.fromkeys(labels_repeated)) + self.shape[l]=(1,) + + self.labels = list(dict.fromkeys(labels_repeated)) def get_array(self,label): @@ -56,7 +59,7 @@ class Table(): self.data = pd.concat([self.data,new_data],axis=1) - def to_ASCII(self,name=None): + def to_ASCII(self,fname): labels = [] for l in self.labels: if(self.shape[l] == (1,)): @@ -72,6 +75,9 @@ class Table(): + self.comments\ + [' '.join(labels)] - with open(name if name is not None else self.name,'w') as f: - for line in header: f.write(line+'\n') - self.data.to_csv(f,sep=' ',index=False,header=False) + try: + f = open(fname,'w') + except TypeError: + f = fname + for line in header: f.write(line+'\n') + self.data.to_csv(f,sep=' ',index=False,header=False) From 925a4f73d649eee0276e102e0c95da2c119d18fa Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Tue, 26 Nov 2019 20:32:54 +0100 Subject: [PATCH 03/38] staticmethod better suited than class method a classmethod changes the class, i.e. it assigns attributes and gives them specific values. a staticmethod does not alter the class https://www.geeksforgeeks.org/class-method-vs-static-method-python --- python/damask/geom.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/damask/geom.py b/python/damask/geom.py index 1c9e10cd1..32ea2ed89 100644 --- a/python/damask/geom.py +++ b/python/damask/geom.py @@ -239,8 +239,8 @@ class Geom(): header.append('homogenization {}'.format(self.get_homogenization())) return header - @classmethod - def from_file(cls,fname): + @staticmethod + def from_file(fname): """ Reads a geom file. @@ -300,7 +300,7 @@ class Geom(): if not np.any(np.mod(microstructure.flatten(),1) != 0.0): # no float present microstructure = microstructure.astype('int') - return cls(microstructure.reshape(grid),size,origin,homogenization,comments) + return Geom(microstructure.reshape(grid),size,origin,homogenization,comments) def to_file(self,fname,pack=None): From 5661f60552e25ffe19ce2bb505e9b2a8404bad76 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Tue, 26 Nov 2019 21:36:24 +0100 Subject: [PATCH 04/38] fname seems to be the common name --- python/damask/dadf5.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/python/damask/dadf5.py b/python/damask/dadf5.py index d879946eb..5ecc8e619 100644 --- a/python/damask/dadf5.py +++ b/python/damask/dadf5.py @@ -18,17 +18,17 @@ class DADF5(): """ # ------------------------------------------------------------------ - def __init__(self,filename): + def __init__(self,fname): """ Opens an existing DADF5 file. Parameters ---------- - filename : str + fname : str name of the DADF5 file to be openend. """ - with h5py.File(filename,'r') as f: + with h5py.File(fname,'r') as f: if f.attrs['DADF5-major'] != 0 or not 2 <= f.attrs['DADF5-minor'] <= 3: raise TypeError('Unsupported DADF5 version {} '.format(f.attrs['DADF5-version'])) @@ -64,7 +64,7 @@ class DADF5(): 'con_physics': self.con_physics, 'mat_physics': self.mat_physics} - self.filename = filename + self.fname = fname def __manage_visible(self,datasets,what,action): @@ -298,7 +298,7 @@ class DADF5(): groups = [] - with h5py.File(self.filename,'r') as f: + with h5py.File(self.fname,'r') as f: for i in self.iter_visible('increments'): for o,p in zip(['constituents','materialpoints'],['con_physics','mat_physics']): for oo in self.iter_visible(o): @@ -315,7 +315,7 @@ class DADF5(): def list_data(self): """Return information on all active datasets in the file.""" message = '' - with h5py.File(self.filename,'r') as f: + with h5py.File(self.fname,'r') as f: for s,i in enumerate(self.iter_visible('increments')): message+='\n{} ({}s)\n'.format(i,self.times[s]) for o,p in zip(['constituents','materialpoints'],['con_physics','mat_physics']): @@ -336,7 +336,7 @@ class DADF5(): def get_dataset_location(self,label): """Return the location of all active datasets with given label.""" path = [] - with h5py.File(self.filename,'r') as f: + with h5py.File(self.fname,'r') as f: for i in self.iter_visible('increments'): k = '/'.join([i,'geometry',label]) try: @@ -358,14 +358,14 @@ class DADF5(): def get_constituent_ID(self,c=0): """Pointwise constituent ID.""" - with h5py.File(self.filename,'r') as f: + with h5py.File(self.fname,'r') as f: names = f['/mapping/cellResults/constituent']['Name'][:,c].astype('str') return np.array([int(n.split('_')[0]) for n in names.tolist()],dtype=np.int32) def get_crystal_structure(self): # ToDo: extension to multi constituents/phase """Info about the crystal structure.""" - with h5py.File(self.filename,'r') as f: + with h5py.File(self.fname,'r') as f: return f[self.get_dataset_location('orientation')[0]].attrs['Lattice'].astype('str') # np.bytes_ to string @@ -375,7 +375,7 @@ class DADF5(): If more than one path is given, the dataset is composed of the individual contributions. """ - with h5py.File(self.filename,'r') as f: + with h5py.File(self.fname,'r') as f: shape = (self.Nmaterialpoints,) + np.shape(f[path[0]])[1:] if len(shape) == 1: shape = shape +(1,) dataset = np.full(shape,np.nan,dtype=np.dtype(f[path[0]])) @@ -418,7 +418,7 @@ class DADF5(): ) return np.concatenate((x[:,:,:,None],y[:,:,:,None],y[:,:,:,None]),axis = 3).reshape([np.product(self.grid),3]) else: - with h5py.File(self.filename,'r') as f: + with h5py.File(self.fname,'r') as f: return f['geometry/x_c'][()] @@ -798,7 +798,7 @@ class DADF5(): todo = [] # ToDo: It would be more memory efficient to read only from file when required, i.e. do to it in pool.add_task for group in self.groups_with_datasets([d['label'] for d in datasets_requested]): - with h5py.File(self.filename,'r') as f: + with h5py.File(self.fname,'r') as f: datasets_in = {} for d in datasets_requested: loc = f[group+'/'+d['label']] @@ -813,7 +813,7 @@ class DADF5(): N_not_calculated = len(todo) while N_not_calculated > 0: result = results.get() - with h5py.File(self.filename,'a') as f: # write to file + with h5py.File(self.fname,'a') as f: # write to file dataset_out = f[result['group']].create_dataset(result['label'],data=result['data']) for k in result['meta'].keys(): dataset_out.attrs[k] = result['meta'][k].encode() From 2d96136a0d9fb305a0bd43ce11e00077c9d900d1 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Tue, 26 Nov 2019 22:53:46 +0100 Subject: [PATCH 05/38] more general constructor for Table reading from file is just one case (now handled by static method). General constructor needs data and header information as dictionary. Works only with python 3.7 where dict keeps the insertion order. Earlier python versions/other implementations might fail. --- processing/post/addCauchy.py | 15 +++++-- python/damask/table.py | 82 ++++++++++++++++++++++++------------ 2 files changed, 66 insertions(+), 31 deletions(-) diff --git a/processing/post/addCauchy.py b/processing/post/addCauchy.py index 3a0e14da7..788f1b580 100755 --- a/processing/post/addCauchy.py +++ b/processing/post/addCauchy.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import os +import sys +from io import StringIO from optparse import OptionParser import damask @@ -34,9 +36,14 @@ parser.set_defaults(defgrad = 'f', (options,filenames) = parser.parse_args() +if filenames == []: filenames = [None] + for name in filenames: - table = damask.Table(name) - table.add_array('Cauchy',damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3), - table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9), + damask.util.report(scriptName,name) + + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + table.add_array('Cauchy', + damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3), + table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9), scriptID) - table.to_ASCII(name) + table.to_ASCII(sys.stdout if name is None else name) diff --git a/python/damask/table.py b/python/damask/table.py index e77ae2b9c..6c5103bc9 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -4,9 +4,42 @@ import pandas as pd import numpy as np class Table(): - """Read and write to ASCII tables""" - - def __init__(self,fname): + """Store spreadsheet-like data.""" + + def __init__(self,array,headings,comments=None): + """ + New spreadsheet data. + + Parameters + ---------- + array : numpy.ndarray + Data. + headings : dict + Column headings. Labels as keys and shape as tuple. Example 'F':(3,3) for a deformation gradient. + comments : iterable of str, optional + Additional, human-readable information + + """ + self.data = pd.DataFrame(data=array) + + d = {} + i = 0 + for label in headings: + for components in range(np.prod(headings[label])): + d[i] = label + i+=1 + + self.data.rename(columns=d,inplace=True) + + if comments is None: + self.comments = [] + else: + self.comments = [c for c in comments] + + self.headings = headings + + @staticmethod + def from_ASCII(fname): try: f = open(fname) except TypeError: @@ -17,43 +50,38 @@ class Table(): header = int(header) else: raise Exception - self.comments = [f.readline()[:-1] for i in range(header-1)] - labels_raw = f.readline().split() - self.data = pd.read_csv(f,delim_whitespace=True,header=None) - - labels_repeated = [l.split('_',1)[1] if '_' in l else l for l in labels_raw] - self.data.rename(columns=dict(zip([l for l in self.data.columns],labels_repeated)),inplace=True) - - self.shape = {} + comments = [f.readline()[:-1] for i in range(header-1)] + labels_raw = f.readline().split() + labels = [l.split('_',1)[1] if '_' in l else l for l in labels_raw] + + headings = {} for l in labels_raw: tensor_column = re.search(':.*?_',l) if tensor_column: my_shape = tensor_column.group()[1:-1].split('x') - self.shape[l.split('_',1)[1]] = tuple([int(d) for d in my_shape]) + headings[l.split('_',1)[1]] = tuple([int(d) for d in my_shape]) else: vector_column = re.match('.*?_',l) if vector_column: - self.shape[l.split('_',1)[1]] = (int(l.split('_',1)[0]),) + headings[l.split('_',1)[1]] = (int(l.split('_',1)[0]),) else: - self.shape[l]=(1,) + headings[l]=(1,) - self.labels = list(dict.fromkeys(labels_repeated)) + return Table(np.loadtxt(f),headings,comments) def get_array(self,label): - return self.data[label].to_numpy().reshape((-1,)+self.shape[label]) + return self.data[label].to_numpy().reshape((-1,)+self.headings[label]) def add_array(self,label,array,info): - if np.product(array.shape[1:],dtype=int) == 1: + if np.prod(array.shape[1:],dtype=int) == 1: self.comments.append('{}: {}'.format(label,info)) - else: self.comments.append('{} {}: {}'.format(label,array.shape[1:],info)) - self.shape[label] = array.shape[1:] - self.labels.append(label) - size = np.product(array.shape[1:]) + self.headings[label] = array.shape[1:] if len(array.shape) > 1 else (1,) + size = np.prod(array.shape[1:],dtype=int) new_data = pd.DataFrame(data=array.reshape(-1,size), columns=[label for l in range(size)]) self.data = pd.concat([self.data,new_data],axis=1) @@ -61,15 +89,15 @@ class Table(): def to_ASCII(self,fname): labels = [] - for l in self.labels: - if(self.shape[l] == (1,)): + for l in self.headings: + if(self.headings[l] == (1,)): labels.append('{}'.format(l)) - elif(len(self.shape[l]) == 1): + elif(len(self.headings[l]) == 1): labels+=['{}_{}'.format(i+1,l)\ - for i in range(self.shape[l][0])] + for i in range(self.headings[l][0])] else: - labels+=['{}:{}_{}'.format(i+1,'x'.join([str(d) for d in self.shape[l]]),l)\ - for i in range(np.product(self.shape[l]))] + labels+=['{}:{}_{}'.format(i+1,'x'.join([str(d) for d in self.headings[l]]),l)\ + for i in range(np.prod(self.headings[l],dtype=int))] header = ['{} header'.format(len(self.comments)+1)]\ + self.comments\ From 31d3958ca6d43b63270c6755def366fbe4824d02 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Tue, 26 Nov 2019 22:55:38 +0100 Subject: [PATCH 06/38] using fast new Table class more a proof-of-concept since shell scripts are deprecated. The detailed error handling of the former scripts is not implemented, i.e. the user need to ensure that the files exist and the data has the correct shape --- processing/post/addDeterminant.py | 66 ++++---------------------- processing/post/addDeviator.py | 79 +++++-------------------------- 2 files changed, 22 insertions(+), 123 deletions(-) diff --git a/processing/post/addDeterminant.py b/processing/post/addDeterminant.py index 14f0321be..090b9224a 100755 --- a/processing/post/addDeterminant.py +++ b/processing/post/addDeterminant.py @@ -4,20 +4,13 @@ import os import sys from optparse import OptionParser -import damask +import numpy as np +import damask scriptName = os.path.splitext(os.path.basename(__file__))[0] scriptID = ' '.join([scriptName,damask.version]) -def determinant(m): - return +m[0]*m[4]*m[8] \ - +m[1]*m[5]*m[6] \ - +m[2]*m[3]*m[7] \ - -m[2]*m[4]*m[6] \ - -m[1]*m[3]*m[8] \ - -m[0]*m[5]*m[7] - # -------------------------------------------------------------------- # MAIN @@ -43,52 +36,11 @@ if options.tensor is None: if filenames == []: filenames = [None] for name in filenames: - try: - table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ read header ------------------------------------------ - - table.head_read() - -# ------------------------------------------ sanity checks ---------------------------------------- - - items = { - 'tensor': {'dim': 9, 'shape': [3,3], 'labels':options.tensor, 'column': []}, - } - errors = [] - remarks = [] - - for type, data in items.items(): - for what in data['labels']: - dim = table.label_dimension(what) - if dim != data['dim']: remarks.append('column {} is not a {}...'.format(what,type)) - else: - items[type]['column'].append(table.label_index(what)) - table.labels_append('det({})'.format(what)) # extend ASCII header with new labels - - if remarks != []: damask.util.croak(remarks) - if errors != []: - damask.util.croak(errors) - table.close(dismiss = True) - continue - -# ------------------------------------------ assemble header -------------------------------------- - - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - table.head_write() - -# ------------------------------------------ process data ------------------------------------------ - - outputAlive = True - while outputAlive and table.data_read(): # read next data line of ASCII table - for type, data in items.items(): - for column in data['column']: - table.data_append(determinant(list(map(float,table.data[column: column+data['dim']])))) - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close input ASCII table (works for stdin) + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + for tensor in options.tensor: + table.add_array('det({})'.format(tensor), + np.linalg.det(table.get_array(tensor).reshape(-1,3,3)), + scriptID) + table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addDeviator.py b/processing/post/addDeviator.py index c9aeaacfd..c0bb77e6b 100755 --- a/processing/post/addDeviator.py +++ b/processing/post/addDeviator.py @@ -2,6 +2,7 @@ import os import sys +from io import StringIO from optparse import OptionParser import damask @@ -9,17 +10,6 @@ import damask scriptName = os.path.splitext(os.path.basename(__file__))[0] scriptID = ' '.join([scriptName,damask.version]) -oneThird = 1.0/3.0 - -def deviator(m,spherical = False): # Careful, do not change the value of m, its intent(inout)! - sph = oneThird*(m[0]+m[4]+m[8]) - dev = [ - m[0]-sph, m[1], m[2], - m[3], m[4]-sph, m[5], - m[6], m[7], m[8]-sph, - ] - return dev,sph if spherical else dev - # -------------------------------------------------------------------- # MAIN @@ -49,58 +39,15 @@ if options.tensor is None: if filenames == []: filenames = [None] for name in filenames: - try: - table = damask.ASCIItable(name = name, buffered = False) - except: - continue - damask.util.report(scriptName,name) - -# ------------------------------------------ read header ------------------------------------------ - - table.head_read() - -# ------------------------------------------ sanity checks ---------------------------------------- - - items = { - 'tensor': {'dim': 9, 'shape': [3,3], 'labels':options.tensor, 'active':[], 'column': []}, - } - errors = [] - remarks = [] - column = {} - - for type, data in items.items(): - for what in data['labels']: - dim = table.label_dimension(what) - if dim != data['dim']: remarks.append('column {} is not a {}.'.format(what,type)) - else: - items[type]['active'].append(what) - items[type]['column'].append(table.label_index(what)) - - if remarks != []: damask.util.croak(remarks) - if errors != []: - damask.util.croak(errors) - table.close(dismiss = True) - continue - -# ------------------------------------------ assemble header -------------------------------------- - - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - for type, data in items.items(): - for label in data['active']: - table.labels_append(['{}_dev({})'.format(i+1,label) for i in range(data['dim'])] + \ - (['sph({})'.format(label)] if options.spherical else [])) # extend ASCII header with new labels - table.head_write() - -# ------------------------------------------ process data ------------------------------------------ - - outputAlive = True - while outputAlive and table.data_read(): # read next data line of ASCII table - for type, data in items.items(): - for column in data['column']: - table.data_append(deviator(list(map(float,table.data[column: - column+data['dim']])),options.spherical)) - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close input ASCII table (works for stdin) + damask.util.report(scriptName,name) + + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + for tensor in options.tensor: + table.add_array('dev({})'.format(tensor), + damask.mechanics.deviatoric_part(table.get_array(tensor).reshape(-1,3,3)).reshape((-1,9)), + scriptID) + if options.spherical: + table.add_array('sph({})'.format(tensor), + damask.mechanics.spherical_part(table.get_array(tensor).reshape(-1,3,3)), + scriptID) + table.to_ASCII(sys.stdout if name is None else name) From 882a11c5f874b2491c88753c80b8cba69970f2f1 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 27 Nov 2019 08:43:20 +0100 Subject: [PATCH 07/38] get labels without shape information --- python/damask/table.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/damask/table.py b/python/damask/table.py index 6c5103bc9..7df642d02 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -69,10 +69,11 @@ class Table(): return Table(np.loadtxt(f),headings,comments) - def get_array(self,label): return self.data[label].to_numpy().reshape((-1,)+self.headings[label]) + def get_labels(self): + return [label for label in self.headings] def add_array(self,label,array,info): if np.prod(array.shape[1:],dtype=int) == 1: @@ -86,7 +87,6 @@ class Table(): columns=[label for l in range(size)]) self.data = pd.concat([self.data,new_data],axis=1) - def to_ASCII(self,fname): labels = [] for l in self.headings: From eb033e11b29522ccf9ad634ed6bbd52c8452eaf1 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 27 Nov 2019 08:44:38 +0100 Subject: [PATCH 08/38] polishing --- processing/post/addCauchy.py | 4 ++-- processing/post/addDeterminant.py | 8 +++----- processing/post/addDeviator.py | 12 +++++------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/processing/post/addCauchy.py b/processing/post/addCauchy.py index 788f1b580..ad29792a6 100755 --- a/processing/post/addCauchy.py +++ b/processing/post/addCauchy.py @@ -35,7 +35,6 @@ parser.set_defaults(defgrad = 'f', ) (options,filenames) = parser.parse_args() - if filenames == []: filenames = [None] for name in filenames: @@ -45,5 +44,6 @@ for name in filenames: table.add_array('Cauchy', damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3), table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9), - scriptID) + scriptID+' '+' '.join(sys.argv[1:])) + table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addDeterminant.py b/processing/post/addDeterminant.py index 090b9224a..e38261bf6 100755 --- a/processing/post/addDeterminant.py +++ b/processing/post/addDeterminant.py @@ -27,14 +27,11 @@ parser.add_option('-t','--tensor', help = 'heading of columns containing tensor field values') (options,filenames) = parser.parse_args() +if filenames == []: filenames = [None] if options.tensor is None: parser.error('no data column specified.') -# --- loop over input files ------------------------------------------------------------------------- - -if filenames == []: filenames = [None] - for name in filenames: damask.util.report(scriptName,name) @@ -42,5 +39,6 @@ for name in filenames: for tensor in options.tensor: table.add_array('det({})'.format(tensor), np.linalg.det(table.get_array(tensor).reshape(-1,3,3)), - scriptID) + scriptID+' '+' '.join(sys.argv[1:])) + table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addDeviator.py b/processing/post/addDeviator.py index c0bb77e6b..7807c7f5d 100755 --- a/processing/post/addDeviator.py +++ b/processing/post/addDeviator.py @@ -30,13 +30,10 @@ parser.add_option('-s','--spherical', help = 'report spherical part of tensor (hydrostatic component, pressure)') (options,filenames) = parser.parse_args() +if filenames == []: filenames = [None] if options.tensor is None: - parser.error('no data column specified...') - -# --- loop over input files ------------------------------------------------------------------------- - -if filenames == []: filenames = [None] + parser.error('no data column specified...') for name in filenames: damask.util.report(scriptName,name) @@ -45,9 +42,10 @@ for name in filenames: for tensor in options.tensor: table.add_array('dev({})'.format(tensor), damask.mechanics.deviatoric_part(table.get_array(tensor).reshape(-1,3,3)).reshape((-1,9)), - scriptID) + scriptID+' '+' '.join(sys.argv[1:])) if options.spherical: table.add_array('sph({})'.format(tensor), damask.mechanics.spherical_part(table.get_array(tensor).reshape(-1,3,3)), - scriptID) + scriptID+' '+' '.join(sys.argv[1:])) + table.to_ASCII(sys.stdout if name is None else name) From ab83dc2ebc91a44589f03040775ce11e38fe3952 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 27 Nov 2019 08:45:01 +0100 Subject: [PATCH 09/38] use central facilities --- processing/post/addMises.py | 80 +++++++------------------------------ 1 file changed, 14 insertions(+), 66 deletions(-) diff --git a/processing/post/addMises.py b/processing/post/addMises.py index be11b0f1c..9bc560f2d 100755 --- a/processing/post/addMises.py +++ b/processing/post/addMises.py @@ -2,10 +2,8 @@ import os import sys +from io import StringIO from optparse import OptionParser -from collections import OrderedDict - -import numpy as np import damask @@ -13,15 +11,6 @@ import damask scriptName = os.path.splitext(os.path.basename(__file__))[0] scriptID = ' '.join([scriptName,damask.version]) -def Mises(what,tensor): - - dev = tensor - np.trace(tensor)/3.0*np.eye(3) - symdev = 0.5*(dev+dev.T) - return np.sqrt(np.sum(symdev*symdev.T)* - { - 'stress': 3.0/2.0, - 'strain': 2.0/3.0, - }[what.lower()]) # -------------------------------------------------------------------- # MAIN @@ -47,62 +36,21 @@ parser.set_defaults(strain = [], (options,filenames) = parser.parse_args() if options.stress is [] and options.strain is []: - parser.error('no data column specified...') - -# --- loop over input files ------------------------------------------------------------------------- + parser.error('no data column specified...') if filenames == []: filenames = [None] for name in filenames: - try: - table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ read header ------------------------------------------ - - table.head_read() - -# ------------------------------------------ sanity checks ---------------------------------------- - - items = OrderedDict([ - ('strain', {'dim': 9, 'shape': [3,3], 'labels':options.strain, 'active':[], 'column': []}), - ('stress', {'dim': 9, 'shape': [3,3], 'labels':options.stress, 'active':[], 'column': []}) - ]) - errors = [] - remarks = [] - - for type, data in items.items(): - for what in data['labels']: - dim = table.label_dimension(what) - if dim != data['dim']: remarks.append('column {} is not a {}...'.format(what,type)) - else: - items[type]['active'].append(what) - items[type]['column'].append(table.label_index(what)) - table.labels_append('Mises({})'.format(what)) # extend ASCII header with new labels - - if remarks != []: damask.util.croak(remarks) - if errors != []: - damask.util.croak(errors) - table.close(dismiss = True) - continue - -# ------------------------------------------ assemble header -------------------------------------- - - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - table.head_write() - -# ------------------------------------------ process data ------------------------------------------ - - outputAlive = True - while outputAlive and table.data_read(): # read next data line of ASCII table - for type, data in items.items(): - for column in data['column']: - table.data_append(Mises(type, - np.array(table.data[column:column+data['dim']],'d').reshape(data['shape']))) - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close input ASCII table (works for stdin) + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + for strain in options.strain: + table.add_array('Mises({})'.format(strain), + damask.mechanics.Mises_strain(damask.mechanics.symmetric(table.get_array(strain).reshape(-1,3,3))), + scriptID+' '+' '.join(sys.argv[1:])) + for stress in options.stress: + table.add_array('Mises({})'.format(stress), + damask.mechanics.Mises_stress(damask.mechanics.symmetric(table.get_array(stress).reshape(-1,3,3))), + scriptID+' '+' '.join(sys.argv[1:])) + + table.to_ASCII(sys.stdout if name is None else name) From 9ad74745c15a43fe5cb0c884ac82ca74cde9047c Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 27 Nov 2019 08:48:41 +0100 Subject: [PATCH 10/38] missing import detected by prospector --- processing/post/addDeterminant.py | 1 + 1 file changed, 1 insertion(+) diff --git a/processing/post/addDeterminant.py b/processing/post/addDeterminant.py index e38261bf6..4c3f9d260 100755 --- a/processing/post/addDeterminant.py +++ b/processing/post/addDeterminant.py @@ -2,6 +2,7 @@ import os import sys +from io import StringIO from optparse import OptionParser import numpy as np From 96714089b19b3f38e247d06c79d075521c5b7d2e Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 27 Nov 2019 09:58:58 +0100 Subject: [PATCH 11/38] documented and tested handling of multi-dimensional data more precise regex expressions. get_array can handle individual components --- python/damask/table.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/python/damask/table.py b/python/damask/table.py index 7df642d02..bad2a60c3 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -40,6 +40,13 @@ class Table(): @staticmethod def from_ASCII(fname): + """ + Create table from ASCII file. + + The first line needs to indicate the number of subsequent header lines as 'n header'. + Vector data labels are indicated by '1_x, 2_x, ..., n_x'. + Tensor data labels are indicated by '3x3:1_x, 3x3:2_x, ..., 3x3:9_x'. + """ try: f = open(fname) except TypeError: @@ -50,29 +57,34 @@ class Table(): header = int(header) else: raise Exception - comments = [f.readline()[:-1] for i in range(header-1)] - labels_raw = f.readline().split() - labels = [l.split('_',1)[1] if '_' in l else l for l in labels_raw] + comments = [f.readline()[:-1] for i in range(header-1)] + labels = f.readline().split() headings = {} - for l in labels_raw: - tensor_column = re.search(':.*?_',l) + for label in labels: + tensor_column = re.search(r'[0-9,x]*?:[0-9]*?_',label) if tensor_column: - my_shape = tensor_column.group()[1:-1].split('x') - headings[l.split('_',1)[1]] = tuple([int(d) for d in my_shape]) + my_shape = tensor_column.group().split(':',1)[0].split('x') + headings[label.split('_',1)[1]] = tuple([int(d) for d in my_shape]) else: - vector_column = re.match('.*?_',l) + vector_column = re.match(r'[0-9]*?_',label) if vector_column: - headings[l.split('_',1)[1]] = (int(l.split('_',1)[0]),) + headings[label.split('_',1)[1]] = (int(label.split('_',1)[0]),) else: - headings[l]=(1,) + headings[label]=(1,) return Table(np.loadtxt(f),headings,comments) def get_array(self,label): - return self.data[label].to_numpy().reshape((-1,)+self.headings[label]) + """Return data as array.""" + if re.match(r'[0-9]*?_',label): + idx,key = label.split('_',1) + return self.data[key].to_numpy()[:,int(idx)-1] + else: + return self.data[label].to_numpy().reshape((-1,)+self.headings[label]) def get_labels(self): + """Return the labels of all columns.""" return [label for label in self.headings] def add_array(self,label,array,info): From 39734ef53cc81438ff69f84cfc87e89ec270fa0c Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 27 Nov 2019 10:56:29 +0100 Subject: [PATCH 12/38] alter data allows to access multiple columns ('f') and individual components ('1_f') --- python/damask/table.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/damask/table.py b/python/damask/table.py index bad2a60c3..2e426d6fb 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -83,6 +83,16 @@ class Table(): else: return self.data[label].to_numpy().reshape((-1,)+self.headings[label]) + def set_array(self,label,array): + """Set data.""" + if re.match(r'[0-9]*?_',label): + idx,key = label.split('_',1) + iloc = self.data.columns.get_loc(key).tolist().index(True) + int(idx) -1 + self.data.iloc[:,iloc] = array + else: + self.data[label] = array + + def get_labels(self): """Return the labels of all columns.""" return [label for label in self.headings] From 76c357737854fac8824fe798067fa8807d514ed9 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 27 Nov 2019 12:03:35 +0100 Subject: [PATCH 13/38] pandas.DataFrame needs to be a 2nd order array --- python/damask/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/damask/table.py b/python/damask/table.py index 2e426d6fb..3cd6ed330 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -90,7 +90,7 @@ class Table(): iloc = self.data.columns.get_loc(key).tolist().index(True) + int(idx) -1 self.data.iloc[:,iloc] = array else: - self.data[label] = array + self.data[label] = array.reshape(self.data[label].shape) def get_labels(self): From 7d4b982c736f63f532d26bd7b67a1dcc155ff5b9 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 27 Nov 2019 12:08:23 +0100 Subject: [PATCH 14/38] make information on modifications mandatory --- processing/post/scaleData.py | 59 ++++++------------------------------ python/damask/table.py | 7 ++++- 2 files changed, 15 insertions(+), 51 deletions(-) diff --git a/processing/post/scaleData.py b/processing/post/scaleData.py index 5b03f8e07..90eaeb0cc 100755 --- a/processing/post/scaleData.py +++ b/processing/post/scaleData.py @@ -35,58 +35,17 @@ parser.set_defaults(label = [], ) (options,filenames) = parser.parse_args() - -if len(options.label) != len(options.factor): - parser.error('number of column labels and factors do not match.') - -# --- loop over input files ------------------------------------------------------------------------- - if filenames == []: filenames = [None] +if len(options.label) != len(options.factor): + parser.error('number of column labels and factors do not match.') + for name in filenames: - try: - table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ read header ------------------------------------------ + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + for i,label in enumerate(options.label): + table.set_array(label,table.get_array(label)*float(options.factor[i]), + scriptID+' '+' '.join(sys.argv[1:])) - table.head_read() - - errors = [] - remarks = [] - columns = [] - dims = [] - factors = [] - - for what,factor in zip(options.label,options.factor): - col = table.label_index(what) - if col < 0: remarks.append('column {} not found...'.format(what,type)) - else: - columns.append(col) - factors.append(float(factor)) - dims.append(table.label_dimension(what)) - - if remarks != []: damask.util.croak(remarks) - if errors != []: - damask.util.croak(errors) - table.close(dismiss = True) - continue - -# ------------------------------------------ assemble header --------------------------------------- - - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - table.head_write() - -# ------------------------------------------ process data ------------------------------------------ - - outputAlive = True - while outputAlive and table.data_read(): # read next data line of ASCII table - for col,dim,factor in zip(columns,dims,factors): # loop over items - table.data[col:col+dim] = factor * np.array(table.data[col:col+dim],'d') - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close ASCII tables + table.to_ASCII(sys.stdout if name is None else name) diff --git a/python/damask/table.py b/python/damask/table.py index 3cd6ed330..0c9f3ac40 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -83,8 +83,13 @@ class Table(): else: return self.data[label].to_numpy().reshape((-1,)+self.headings[label]) - def set_array(self,label,array): + def set_array(self,label,array,info): """Set data.""" + if np.prod(array.shape[1:],dtype=int) == 1: + self.comments.append('{}: {}'.format(label,info)) + else: + self.comments.append('{} {}: {}'.format(label,array.shape[1:],info)) + if re.match(r'[0-9]*?_',label): idx,key = label.split('_',1) iloc = self.data.columns.get_loc(key).tolist().index(True) + int(idx) -1 From cee095b58ed19a2b45fdcd76b7b138a659b94b34 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 27 Nov 2019 12:19:37 +0100 Subject: [PATCH 15/38] better use centralized code --- processing/post/addPK2.py | 54 ++++++------------------------------ processing/post/scaleData.py | 3 +- python/damask/mechanics.py | 21 +++++++++++++- 3 files changed, 29 insertions(+), 49 deletions(-) diff --git a/processing/post/addPK2.py b/processing/post/addPK2.py index f38753619..45afdbcd1 100755 --- a/processing/post/addPK2.py +++ b/processing/post/addPK2.py @@ -2,10 +2,9 @@ import os import sys +from io import StringIO from optparse import OptionParser -import numpy as np - import damask @@ -36,53 +35,16 @@ parser.set_defaults(defgrad = 'f', ) (options,filenames) = parser.parse_args() - -# --- loop over input files ------------------------------------------------------------------------- - if filenames == []: filenames = [None] for name in filenames: - try: - table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ read header ------------------------------------------ + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) - table.head_read() + table.add_array('S', + damask.mechanics.PK2(table.get_array(options.defgrad).reshape(-1,3,3), + table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9), + scriptID+' '+' '.join(sys.argv[1:])) -# ------------------------------------------ sanity checks ---------------------------------------- - - errors = [] - column = {} - - for tensor in [options.defgrad,options.stress]: - dim = table.label_dimension(tensor) - if dim < 0: errors.append('column {} not found.'.format(tensor)) - elif dim != 9: errors.append('column {} is not a tensor.'.format(tensor)) - else: - column[tensor] = table.label_index(tensor) - - if errors != []: - damask.util.croak(errors) - table.close(dismiss = True) - continue - -# ------------------------------------------ assemble header -------------------------------------- - - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - table.labels_append(['{}_S'.format(i+1) for i in range(9)]) # extend ASCII header with new labels - table.head_write() - -# ------------------------------------------ process data ------------------------------------------ - outputAlive = True - while outputAlive and table.data_read(): # read next data line of ASCII table - F = np.array(list(map(float,table.data[column[options.defgrad]:column[options.defgrad]+9])),'d').reshape(3,3) - P = np.array(list(map(float,table.data[column[options.stress ]:column[options.stress ]+9])),'d').reshape(3,3) - table.data_append(list(np.dot(np.linalg.inv(F),P).reshape(9))) # [S] =[P].[F-1] - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close input ASCII table (works for stdin) + table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/scaleData.py b/processing/post/scaleData.py index 90eaeb0cc..ad57f4b7e 100755 --- a/processing/post/scaleData.py +++ b/processing/post/scaleData.py @@ -2,10 +2,9 @@ import os import sys +from io import StringIO from optparse import OptionParser -import numpy as np - import damask diff --git a/python/damask/mechanics.py b/python/damask/mechanics.py index 476682380..5503d7048 100644 --- a/python/damask/mechanics.py +++ b/python/damask/mechanics.py @@ -19,7 +19,26 @@ def Cauchy(F,P): else: sigma = np.einsum('i,ijk,ilk->ijl',1.0/np.linalg.det(F),P,F) return symmetric(sigma) - + + +def PK2(F,P): + """ + Return 2. Piola-Kirchhoff stress calculated from 1. Piola-Kirchhoff stress and deformation gradient. + + Parameters + ---------- + F : numpy.array of shape (:,3,3) or (3,3) + Deformation gradient. + P : numpy.array of shape (:,3,3) or (3,3) + 1. Piola-Kirchhoff stress. + + """ + if np.shape(F) == np.shape(P) == (3,3): + S = np.dot(np.linalg.inv(F),P) + else: + S = np.einsum('ijk,ikl->ijl',np.linalg.inv(F),P) + return S + def strain_tensor(F,t,m): """ From a8016d64bbce73cb3e44e08bc3e6ec2768506855 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 27 Nov 2019 12:31:32 +0100 Subject: [PATCH 16/38] simplified --- processing/post/shiftData.py | 62 ++++++------------------------------ 1 file changed, 10 insertions(+), 52 deletions(-) diff --git a/processing/post/shiftData.py b/processing/post/shiftData.py index 69a9696fa..591e68c5d 100755 --- a/processing/post/shiftData.py +++ b/processing/post/shiftData.py @@ -2,10 +2,9 @@ import os import sys +from io import StringIO from optparse import OptionParser -import numpy as np - import damask @@ -35,58 +34,17 @@ parser.set_defaults(label = [], ) (options,filenames) = parser.parse_args() - -if len(options.label) != len(options.offset): - parser.error('number of column labels and offsets do not match.') - -# --- loop over input files ------------------------------------------------------------------------- - if filenames == []: filenames = [None] +if len(options.label) != len(options.offset): + parser.error('number of column labels and offsets do not match.') + for name in filenames: - try: - table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ read header ------------------------------------------ + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + for i,label in enumerate(options.label): + table.set_array(label,table.get_array(label)+float(options.offset[i]), + scriptID+' '+' '.join(sys.argv[1:])) - table.head_read() - - errors = [] - remarks = [] - columns = [] - dims = [] - offsets = [] - - for what,offset in zip(options.label,options.offset): - col = table.label_index(what) - if col < 0: remarks.append('column {} not found...'.format(what,type)) - else: - columns.append(col) - offsets.append(float(offset)) - dims.append(table.label_dimension(what)) - - if remarks != []: damask.util.croak(remarks) - if errors != []: - damask.util.croak(errors) - table.close(dismiss = True) - continue - -# ------------------------------------------ assemble header --------------------------------------- - - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - table.head_write() - -# ------------------------------------------ process data ------------------------------------------ - - outputAlive = True - while outputAlive and table.data_read(): # read next data line of ASCII table - for col,dim,offset in zip(columns,dims,offsets): # loop over items - table.data[col:col+dim] = offset + np.array(table.data[col:col+dim],'d') - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close ASCII tables + table.to_ASCII(sys.stdout if name is None else name) From ca92400c2f55627349c1f23eb9e52b759e7ce038 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 28 Nov 2019 05:52:23 +0100 Subject: [PATCH 17/38] polishing columns is the term used py pandas --- python/damask/table.py | 84 +++++++++++++++++++++++++++++++----------- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/python/damask/table.py b/python/damask/table.py index 0c9f3ac40..2eacef58e 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -6,7 +6,7 @@ import numpy as np class Table(): """Store spreadsheet-like data.""" - def __init__(self,array,headings,comments=None): + def __init__(self,array,columns,comments=None): """ New spreadsheet data. @@ -14,8 +14,8 @@ class Table(): ---------- array : numpy.ndarray Data. - headings : dict - Column headings. Labels as keys and shape as tuple. Example 'F':(3,3) for a deformation gradient. + columns : dict + Column labels and shape. Example 'F':(3,3) for a deformation gradient. comments : iterable of str, optional Additional, human-readable information @@ -24,8 +24,8 @@ class Table(): d = {} i = 0 - for label in headings: - for components in range(np.prod(headings[label])): + for label in columns: + for components in range(np.prod(columns[label])): d[i] = label i+=1 @@ -36,7 +36,7 @@ class Table(): else: self.comments = [c for c in comments] - self.headings = headings + self.columns = columns @staticmethod def from_ASCII(fname): @@ -46,6 +46,12 @@ class Table(): The first line needs to indicate the number of subsequent header lines as 'n header'. Vector data labels are indicated by '1_x, 2_x, ..., n_x'. Tensor data labels are indicated by '3x3:1_x, 3x3:2_x, ..., 3x3:9_x'. + + Parameters + ---------- + fname : file, str, or pathlib.Path + Filename or file for reading. + """ try: f = open(fname) @@ -60,20 +66,20 @@ class Table(): comments = [f.readline()[:-1] for i in range(header-1)] labels = f.readline().split() - headings = {} + columns = {} for label in labels: tensor_column = re.search(r'[0-9,x]*?:[0-9]*?_',label) if tensor_column: my_shape = tensor_column.group().split(':',1)[0].split('x') - headings[label.split('_',1)[1]] = tuple([int(d) for d in my_shape]) + columns[label.split('_',1)[1]] = tuple([int(d) for d in my_shape]) else: vector_column = re.match(r'[0-9]*?_',label) if vector_column: - headings[label.split('_',1)[1]] = (int(label.split('_',1)[0]),) + columns[label.split('_',1)[1]] = (int(label.split('_',1)[0]),) else: - headings[label]=(1,) + columns[label]=(1,) - return Table(np.loadtxt(f),headings,comments) + return Table(np.loadtxt(f),columns,comments) def get_array(self,label): """Return data as array.""" @@ -81,10 +87,22 @@ class Table(): idx,key = label.split('_',1) return self.data[key].to_numpy()[:,int(idx)-1] else: - return self.data[label].to_numpy().reshape((-1,)+self.headings[label]) + return self.data[label].to_numpy().reshape((-1,)+self.columns[label]) def set_array(self,label,array,info): - """Set data.""" + """ + Modify data in the spreadsheet. + + Parameters + ---------- + label : str + Label for the new data + array : np.ndarray + New data + info : str + Human-readable information about the new data + + """ if np.prod(array.shape[1:],dtype=int) == 1: self.comments.append('{}: {}'.format(label,info)) else: @@ -97,34 +115,56 @@ class Table(): else: self.data[label] = array.reshape(self.data[label].shape) - def get_labels(self): """Return the labels of all columns.""" - return [label for label in self.headings] + return [label for label in self.columns] def add_array(self,label,array,info): + """ + Add data to the spreadsheet. + + Parameters + ---------- + label : str + Label for the new data + array : np.ndarray + New data + info : str + Human-readable information about the new data + + """ if np.prod(array.shape[1:],dtype=int) == 1: self.comments.append('{}: {}'.format(label,info)) else: self.comments.append('{} {}: {}'.format(label,array.shape[1:],info)) - self.headings[label] = array.shape[1:] if len(array.shape) > 1 else (1,) + self.columns[label] = array.shape[1:] if len(array.shape) > 1 else (1,) size = np.prod(array.shape[1:],dtype=int) new_data = pd.DataFrame(data=array.reshape(-1,size), columns=[label for l in range(size)]) self.data = pd.concat([self.data,new_data],axis=1) def to_ASCII(self,fname): + """ + Store as plain text file. + + Parameters + ---------- + fname : file, str, or pathlib.Path + Filename or file for reading. + + """ + labels = [] - for l in self.headings: - if(self.headings[l] == (1,)): + for l in self.columns: + if(self.columns[l] == (1,)): labels.append('{}'.format(l)) - elif(len(self.headings[l]) == 1): + elif(len(self.columns[l]) == 1): labels+=['{}_{}'.format(i+1,l)\ - for i in range(self.headings[l][0])] + for i in range(self.columns[l][0])] else: - labels+=['{}:{}_{}'.format(i+1,'x'.join([str(d) for d in self.headings[l]]),l)\ - for i in range(np.prod(self.headings[l],dtype=int))] + labels+=['{}:{}_{}'.format(i+1,'x'.join([str(d) for d in self.columns[l]]),l)\ + for i in range(np.prod(self.columns[l],dtype=int))] header = ['{} header'.format(len(self.comments)+1)]\ + self.comments\ From 76a15068931a225a92e53ebe52bff2f33c157a5f Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 28 Nov 2019 20:18:54 +0100 Subject: [PATCH 18/38] small improvements bugfix: writing correct labels (now consistent with readin) error handling for inconsistent initialization --- python/damask/table.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/damask/table.py b/python/damask/table.py index 0c9f3ac40..04a073cce 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -13,7 +13,7 @@ class Table(): Parameters ---------- array : numpy.ndarray - Data. + Data as two dimensional array headings : dict Column headings. Labels as keys and shape as tuple. Example 'F':(3,3) for a deformation gradient. comments : iterable of str, optional @@ -29,6 +29,9 @@ class Table(): d[i] = label i+=1 + if i != self.data.shape[1]: + raise IndexError('Mismatch between array shape and headings') + self.data.rename(columns=d,inplace=True) if comments is None: @@ -123,7 +126,7 @@ class Table(): labels+=['{}_{}'.format(i+1,l)\ for i in range(self.headings[l][0])] else: - labels+=['{}:{}_{}'.format(i+1,'x'.join([str(d) for d in self.headings[l]]),l)\ + labels+=['{}:{}_{}'.format('x'.join([str(d) for d in self.headings[l]]),i+1,l)\ for i in range(np.prod(self.headings[l],dtype=int))] header = ['{} header'.format(len(self.comments)+1)]\ From f5bbd3cf223b32f7090a01f0f2f78b48a3f3341d Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Tue, 3 Dec 2019 16:39:54 +0100 Subject: [PATCH 19/38] ensure functionality through unit testing --- python/tests/test_Table.py | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 python/tests/test_Table.py diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py new file mode 100644 index 000000000..577de84d2 --- /dev/null +++ b/python/tests/test_Table.py @@ -0,0 +1,46 @@ +import pytest +import numpy as np + +from damask import Table + +@pytest.fixture +def default(): + """Simple Table.""" + x = np.ones(65).reshape((5,13)) + return Table(x,{'F':(3,3),'v':(3,),'s':(1,)},['test data','contains only ones']) + + +class TestTable: + + def test_get_tensor(self,default): + d = default.get_array('F') + assert np.allclose(d,1.0) and d.shape[1:] == (3,3) + + def test_get_vector(self,default): + d = default.get_array('v') + assert np.allclose(d,1.0) and d.shape[1:] == (3,) + + def test_write_read_str(self,default,tmpdir): + default.to_ASCII(str(tmpdir.join('default.txt'))) + new = Table.from_ASCII(str(tmpdir.join('default.txt'))) + assert all(default.data==new.data) + + def test_write_read_file(self,default,tmpdir): + with open(tmpdir.join('default.txt'),'w') as f: + default.to_ASCII(f) + with open(tmpdir.join('default.txt')) as f: + new = Table.from_ASCII(f) + assert all(default.data==new.data) + + def test_set_array(self,default): + default.set_array('F',np.zeros((5,3,3)),'set to zero') + d=default.get_array('F') + assert np.allclose(d,0.0) and d.shape[1:] == (3,3) + + def test_get_labels(self,default): + assert default.get_labels() == ['F','v','s'] + + def test_add_array(self,default): + d = np.random.random((5,9)) + default.add_array('nine',d,'random data') + assert np.allclose(d,default.get_array('nine')) From d92d503b3c63b0d5178320e467fc40d377dce4c4 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Tue, 3 Dec 2019 16:49:27 +0100 Subject: [PATCH 20/38] pytest should be part of the test suite --- .gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 47cb2810c..6e82561c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ --- stages: - prepareAll + - python - preprocessing - postprocessing - compilePETSc @@ -103,6 +104,16 @@ checkout: - master - release +################################################################################################### +Pytest: + stage: python + script: + - cd $DAMASKROOT/python + - pytest + except: + - master + - release + ################################################################################################### OrientationRelationship: stage: preprocessing From 3effea8e1d046af30c033f064353052146ef9b41 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Tue, 3 Dec 2019 17:03:03 +0100 Subject: [PATCH 21/38] also check operations that should NOT work --- python/tests/test_Table.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index 577de84d2..801df0392 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -6,7 +6,7 @@ from damask import Table @pytest.fixture def default(): """Simple Table.""" - x = np.ones(65).reshape((5,13)) + x = np.ones((5,13)) return Table(x,{'F':(3,3),'v':(3,),'s':(1,)},['test data','contains only ones']) @@ -44,3 +44,14 @@ class TestTable: d = np.random.random((5,9)) default.add_array('nine',d,'random data') assert np.allclose(d,default.get_array('nine')) + + + def test_invalid_initialization(self,default): + x = default.get_array('v') + with pytest.raises(IndexError): + Table(x,{'F':(3,3)}) + + def test_invalid_set(self,default): + x = default.get_array('v') + with pytest.raises(ValueError): + default.set_array('F',x,'does not work') From 07e97787985281a7394fcc88c75ad226c2880f72 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 4 Dec 2019 10:20:03 +0100 Subject: [PATCH 22/38] one more test --- python/tests/test_Table.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index 801df0392..c34f9ee3a 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -55,3 +55,7 @@ class TestTable: x = default.get_array('v') with pytest.raises(ValueError): default.set_array('F',x,'does not work') + + def test_invalid_get_array(self,default): + with pytest.raises(KeyError): + default.get_array('n') From 19c55990ab7d3f61071e86e65c0eb7685d4646a2 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 4 Dec 2019 18:30:08 +0100 Subject: [PATCH 23/38] works again and uses DADF5 intrisic features --- processing/post/DADF5toDREAM3D.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/processing/post/DADF5toDREAM3D.py b/processing/post/DADF5toDREAM3D.py index 885545297..7ab04b934 100755 --- a/processing/post/DADF5toDREAM3D.py +++ b/processing/post/DADF5toDREAM3D.py @@ -49,7 +49,7 @@ Phase_types = {'Primary': 0} #further additions to these can be done by looking # -------------------------------------------------------------------- parser = argparse.ArgumentParser(description='Creating a file for DREAM3D from DAMASK data') parser.add_argument('filenames',nargs='+',help='HDF5 based output file') -parser.add_argument('--inc',nargs='+',help='Increment for which DREAM3D to be used, eg. 00025',type=int) +parser.add_argument('--inc',nargs='+',help='Increment for which DREAM3D to be used, eg. 25',type=int) parser.add_argument('-d','--dir', dest='dir',default='postProc',metavar='string', help='name of subdirectory to hold output') @@ -59,15 +59,13 @@ options = parser.parse_args() # loop over input files for filename in options.filenames: f = damask.DADF5(filename) #DAMASK output file - count = 0 - for increment in f.increments: - if int(increment[3:]) not in options.inc: - count = count + 1 + for increment in options.inc: + f.set_by_increment(increment,increment) + if len(f.visible['increments']) == 0: continue #-------output file creation------------------------------------- dirname = os.path.abspath(os.path.join(os.path.dirname(filename),options.dir)) - print(dirname) try: os.mkdir(dirname) except FileExistsError: @@ -90,11 +88,10 @@ for filename in options.filenames: # Phase information of DREAM.3D is constituent ID in DAMASK o[cell_data_label + '/Phases'] = f.get_constituent_ID().reshape(tuple(f.grid)+(1,)) # Data quaternions - DAMASK_quaternion = f.read_dataset(f.get_dataset_location('orientation'),0) - DREAM_3D_quaternion = np.empty((np.prod(f.grid),4),dtype=np.float32) + DAMASK_quaternion = f.read_dataset(f.get_dataset_location('orientation')) # Convert: DAMASK uses P = -1, DREAM.3D uses P = +1. Also change position of imagninary part DREAM_3D_quaternion = np.hstack((-DAMASK_quaternion['x'],-DAMASK_quaternion['y'],-DAMASK_quaternion['z'], - DAMASK_quaternion['w'])) + DAMASK_quaternion['w'])).astype(np.float32) o[cell_data_label + '/Quats'] = DREAM_3D_quaternion.reshape(tuple(f.grid)+(4,)) # Attributes to CellData group @@ -109,12 +106,14 @@ for filename in options.filenames: # phase attributes o[cell_data_label + '/Phases'].attrs['ComponentDimensions'] = np.array([1],np.uint64) o[cell_data_label + '/Phases'].attrs['ObjectType'] = 'DataArray' + o[cell_data_label + '/Phases'].attrs['TupleDimensions'] = f.grid.astype(np.uint64) # Quats attributes o[cell_data_label + '/Quats'].attrs['ComponentDimensions'] = np.array([4],np.uint64) o[cell_data_label + '/Quats'].attrs['ObjectType'] = 'DataArray' - - # Create EnsembleAttributeMatrix + o[cell_data_label + '/Quats'].attrs['TupleDimensions'] = f.grid.astype(np.uint64) + + # Create EnsembleAttributeMatrix ensemble_label = data_container_label + '/EnsembleAttributeMatrix' # Data CrystalStructures From 1c835725896b93fdd48c64a3aa87a5dccc3d240c Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 05:00:26 +0100 Subject: [PATCH 24/38] more descriptive names --- python/damask/table.py | 60 +++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/python/damask/table.py b/python/damask/table.py index 7d9e65961..bb195122d 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -6,40 +6,40 @@ import numpy as np class Table(): """Store spreadsheet-like data.""" - def __init__(self,array,headings,comments=None): + def __init__(self,data,shapes,comments=None): """ - New spreadsheet data. + New spreadsheet. Parameters ---------- - array : numpy.ndarray + data : numpy.ndarray Data. - headings : dict - Column headings. Labels as keys and shape as tuple. Example 'F':(3,3) for a deformation gradient. + shapes : dict with str:tuple pairs + Shapes of the columns. Example 'F':(3,3) for a deformation gradient. comments : iterable of str, optional - Additional, human-readable information + Additional, human-readable information. """ - self.data = pd.DataFrame(data=array) + self.data = pd.DataFrame(data=data) - d = {} + labels = {} i = 0 - for label in headings: - for components in range(np.prod(headings[label])): - d[i] = label + for label in shapes.keys(): + for components in range(np.prod(shapes[label])): + labels[i] = label i+=1 if i != self.data.shape[1]: - raise IndexError('Mismatch between array shape and headings') + raise IndexError('Shape mismatch between shapes and data') - self.data.rename(columns=d,inplace=True) + self.data.rename(columns=labels,inplace=True) if comments is None: self.comments = [] else: self.comments = [c for c in comments] - self.headings = headings + self.shapes = shapes @staticmethod def from_ASCII(fname): @@ -47,8 +47,8 @@ class Table(): Create table from ASCII file. The first line needs to indicate the number of subsequent header lines as 'n header'. - Vector data labels are indicated by '1_v, 2_v, ..., n_v'. - Tensor data labels are indicated by '3x3:1_T, 3x3:2_T, ..., 3x3:9_T'. + Vector data column labels are indicated by '1_v, 2_v, ..., n_v'. + Tensor data column labels are indicated by '3x3:1_T, 3x3:2_T, ..., 3x3:9_T'. Parameters ---------- @@ -69,20 +69,20 @@ class Table(): comments = [f.readline()[:-1] for i in range(header-1)] labels = f.readline().split() - headings = {} + shapes = {} for label in labels: tensor_column = re.search(r'[0-9,x]*?:[0-9]*?_',label) if tensor_column: my_shape = tensor_column.group().split(':',1)[0].split('x') - headings[label.split('_',1)[1]] = tuple([int(d) for d in my_shape]) + shapes[label.split('_',1)[1]] = tuple([int(d) for d in my_shape]) else: vector_column = re.match(r'[0-9]*?_',label) if vector_column: - headings[label.split('_',1)[1]] = (int(label.split('_',1)[0]),) + shapes[label.split('_',1)[1]] = (int(label.split('_',1)[0]),) else: - headings[label]=(1,) + shapes[label]=(1,) - return Table(np.loadtxt(f),headings,comments) + return Table(np.loadtxt(f),shapes,comments) def get_array(self,label): """ @@ -98,7 +98,7 @@ class Table(): idx,key = label.split('_',1) return self.data[key].to_numpy()[:,int(idx)-1] else: - return self.data[label].to_numpy().reshape((-1,)+self.headings[label]) + return self.data[label].to_numpy().reshape((-1,)+self.shapes[label]) def set_array(self,label,array,info): """ @@ -129,7 +129,7 @@ class Table(): def get_labels(self): """Return the labels of all columns.""" - return [label for label in self.headings] + return list(self.shapes.keys()) def add_array(self,label,array,info): """ @@ -150,7 +150,7 @@ class Table(): else: self.comments.append('{} {}: {}'.format(label,array.shape[1:],info)) - self.headings[label] = array.shape[1:] if len(array.shape) > 1 else (1,) + self.shapes[label] = array.shape[1:] if len(array.shape) > 1 else (1,) size = np.prod(array.shape[1:],dtype=int) new_data = pd.DataFrame(data=array.reshape(-1,size), columns=[label for l in range(size)]) @@ -167,15 +167,15 @@ class Table(): """ labels = [] - for l in self.headings: - if(self.headings[l] == (1,)): + for l in self.shapes: + if(self.shapes[l] == (1,)): labels.append('{}'.format(l)) - elif(len(self.headings[l]) == 1): + elif(len(self.shapes[l]) == 1): labels+=['{}_{}'.format(i+1,l)\ - for i in range(self.headings[l][0])] + for i in range(self.shapes[l][0])] else: - labels+=['{}:{}_{}'.format('x'.join([str(d) for d in self.headings[l]]),i+1,l)\ - for i in range(np.prod(self.headings[l],dtype=int))] + labels+=['{}:{}_{}'.format('x'.join([str(d) for d in self.shapes[l]]),i+1,l)\ + for i in range(np.prod(self.shapes[l],dtype=int))] header = ['{} header'.format(len(self.comments)+1)]\ + self.comments\ From 5fdf112f78de93711a5bd03fa11163c9d6536587 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 05:45:27 +0100 Subject: [PATCH 25/38] allow strings as data in ASCII file --- python/damask/table.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/damask/table.py b/python/damask/table.py index bb195122d..140665ab2 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -82,7 +82,10 @@ class Table(): else: shapes[label]=(1,) - return Table(np.loadtxt(f),shapes,comments) + data = pd.read_csv(f,names=[i for i in range(len(labels))],sep='\s+').to_numpy() + + return Table(data,shapes,comments) + def get_array(self,label): """ From 2172e92e7e2457633afc43fd9cd6740e14b9eafe Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 05:57:47 +0100 Subject: [PATCH 26/38] test exotic input files --- python/tests/reference/Table/datatype-mix.txt | 4 ++++ python/tests/reference/Table/whitespace-mix.txt | 6 ++++++ python/tests/test_Table.py | 12 ++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 python/tests/reference/Table/datatype-mix.txt create mode 100644 python/tests/reference/Table/whitespace-mix.txt diff --git a/python/tests/reference/Table/datatype-mix.txt b/python/tests/reference/Table/datatype-mix.txt new file mode 100644 index 000000000..2f6baa852 --- /dev/null +++ b/python/tests/reference/Table/datatype-mix.txt @@ -0,0 +1,4 @@ +1 header +a b +1.0 hallo +0.1 "hallo test" diff --git a/python/tests/reference/Table/whitespace-mix.txt b/python/tests/reference/Table/whitespace-mix.txt new file mode 100644 index 000000000..933a16e77 --- /dev/null +++ b/python/tests/reference/Table/whitespace-mix.txt @@ -0,0 +1,6 @@ +1 header +a b 1_c 2_c +1 2 3 4 +5 6 7 8 +9 10. 12. 12 + diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index c34f9ee3a..e7550984d 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -1,14 +1,21 @@ +import os + import pytest import numpy as np from damask import Table + @pytest.fixture def default(): """Simple Table.""" x = np.ones((5,13)) return Table(x,{'F':(3,3),'v':(3,),'s':(1,)},['test data','contains only ones']) +@pytest.fixture +def reference_dir(reference_dir_base): + """Directory containing reference results.""" + return os.path.join(reference_dir_base,'Table') class TestTable: @@ -31,6 +38,11 @@ class TestTable: with open(tmpdir.join('default.txt')) as f: new = Table.from_ASCII(f) assert all(default.data==new.data) + + @pytest.mark.parametrize('fname',['datatype-mix.txt','whitespace-mix.txt']) + def test_read_strange(self,reference_dir,fname): + with open(os.path.join(reference_dir,fname)) as f: + new = Table.from_ASCII(f) def test_set_array(self,default): default.set_array('F',np.zeros((5,3,3)),'set to zero') From e7a67262f809eacf6af7b8286740c861d5159bb2 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 06:00:49 +0100 Subject: [PATCH 27/38] clear enough --- python/damask/table.py | 2 +- python/tests/test_Table.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/damask/table.py b/python/damask/table.py index 140665ab2..56507847e 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -130,7 +130,7 @@ class Table(): self.data[label] = array.reshape(self.data[label].shape) - def get_labels(self): + def labels(self): """Return the labels of all columns.""" return list(self.shapes.keys()) diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index e7550984d..c8965d1bd 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -49,8 +49,8 @@ class TestTable: d=default.get_array('F') assert np.allclose(d,0.0) and d.shape[1:] == (3,3) - def test_get_labels(self,default): - assert default.get_labels() == ['F','v','s'] + def test_labels(self,default): + assert default.labels() == ['F','v','s'] def test_add_array(self,default): d = np.random.random((5,9)) From ee8e3386f4db6fea6b14926595a2f8c38a902750 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 06:10:27 +0100 Subject: [PATCH 28/38] shorter but still descriptive names --- processing/post/addCauchy.py | 8 ++-- processing/post/addDeterminant.py | 6 +-- processing/post/addDeviator.py | 12 +++--- processing/post/addMises.py | 12 +++--- processing/post/addPK2.py | 8 ++-- processing/post/scaleData.py | 5 ++- processing/post/shiftData.py | 5 ++- python/damask/table.py | 71 ++++++++++++++++--------------- python/tests/test_Table.py | 26 +++++------ 9 files changed, 79 insertions(+), 74 deletions(-) diff --git a/processing/post/addCauchy.py b/processing/post/addCauchy.py index ad29792a6..afc5a57be 100755 --- a/processing/post/addCauchy.py +++ b/processing/post/addCauchy.py @@ -41,9 +41,9 @@ for name in filenames: damask.util.report(scriptName,name) table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) - table.add_array('Cauchy', - damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3), - table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('Cauchy', + damask.mechanics.Cauchy(table.get(options.defgrad).reshape(-1,3,3), + table.get(options.stress ).reshape(-1,3,3)).reshape(-1,9), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addDeterminant.py b/processing/post/addDeterminant.py index 4c3f9d260..f2368559d 100755 --- a/processing/post/addDeterminant.py +++ b/processing/post/addDeterminant.py @@ -38,8 +38,8 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) for tensor in options.tensor: - table.add_array('det({})'.format(tensor), - np.linalg.det(table.get_array(tensor).reshape(-1,3,3)), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('det({})'.format(tensor), + np.linalg.det(table.get(tensor).reshape(-1,3,3)), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addDeviator.py b/processing/post/addDeviator.py index 7807c7f5d..ca06034b3 100755 --- a/processing/post/addDeviator.py +++ b/processing/post/addDeviator.py @@ -40,12 +40,12 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) for tensor in options.tensor: - table.add_array('dev({})'.format(tensor), - damask.mechanics.deviatoric_part(table.get_array(tensor).reshape(-1,3,3)).reshape((-1,9)), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('dev({})'.format(tensor), + damask.mechanics.deviatoric_part(table.get(tensor).reshape(-1,3,3)).reshape((-1,9)), + scriptID+' '+' '.join(sys.argv[1:])) if options.spherical: - table.add_array('sph({})'.format(tensor), - damask.mechanics.spherical_part(table.get_array(tensor).reshape(-1,3,3)), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('sph({})'.format(tensor), + damask.mechanics.spherical_part(table.get(tensor).reshape(-1,3,3)), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addMises.py b/processing/post/addMises.py index 9bc560f2d..0c2a6db50 100755 --- a/processing/post/addMises.py +++ b/processing/post/addMises.py @@ -45,12 +45,12 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) for strain in options.strain: - table.add_array('Mises({})'.format(strain), - damask.mechanics.Mises_strain(damask.mechanics.symmetric(table.get_array(strain).reshape(-1,3,3))), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('Mises({})'.format(strain), + damask.mechanics.Mises_strain(damask.mechanics.symmetric(table.get(strain).reshape(-1,3,3))), + scriptID+' '+' '.join(sys.argv[1:])) for stress in options.stress: - table.add_array('Mises({})'.format(stress), - damask.mechanics.Mises_stress(damask.mechanics.symmetric(table.get_array(stress).reshape(-1,3,3))), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('Mises({})'.format(stress), + damask.mechanics.Mises_stress(damask.mechanics.symmetric(table.get(stress).reshape(-1,3,3))), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addPK2.py b/processing/post/addPK2.py index 45afdbcd1..185160d79 100755 --- a/processing/post/addPK2.py +++ b/processing/post/addPK2.py @@ -42,9 +42,9 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) - table.add_array('S', - damask.mechanics.PK2(table.get_array(options.defgrad).reshape(-1,3,3), - table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('S', + damask.mechanics.PK2(table.get(options.defgrad).reshape(-1,3,3), + table.get(options.stress ).reshape(-1,3,3)).reshape(-1,9), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/scaleData.py b/processing/post/scaleData.py index ad57f4b7e..60ffa41fa 100755 --- a/processing/post/scaleData.py +++ b/processing/post/scaleData.py @@ -44,7 +44,8 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) for i,label in enumerate(options.label): - table.set_array(label,table.get_array(label)*float(options.factor[i]), - scriptID+' '+' '.join(sys.argv[1:])) + table.set(label, + table.get(label)*float(options.factor[i]), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/shiftData.py b/processing/post/shiftData.py index 591e68c5d..37c72919e 100755 --- a/processing/post/shiftData.py +++ b/processing/post/shiftData.py @@ -44,7 +44,8 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) for i,label in enumerate(options.label): - table.set_array(label,table.get_array(label)+float(options.offset[i]), - scriptID+' '+' '.join(sys.argv[1:])) + table.set(label, + table.get(label)+float(options.offset[i]), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/python/damask/table.py b/python/damask/table.py index 56507847e..171d15084 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -87,14 +87,19 @@ class Table(): return Table(data,shapes,comments) - def get_array(self,label): + def labels(self): + """Return the labels of all columns.""" + return list(self.shapes.keys()) + + + def get(self,label): """ - Return data as array. + Get column data. Parameters ---------- label : str - Label of the array. + Column label. """ if re.match(r'[0-9]*?_',label): @@ -103,62 +108,60 @@ class Table(): else: return self.data[label].to_numpy().reshape((-1,)+self.shapes[label]) - def set_array(self,label,array,info): + def set(self,label,data,info=None): """ - Modify data in the spreadsheet. + Set column data. Parameters ---------- label : str - Label for the new data. - array : np.ndarray + Column label. + data : np.ndarray New data. - info : str + info : str, optional Human-readable information about the new data. """ - if np.prod(array.shape[1:],dtype=int) == 1: - self.comments.append('{}: {}'.format(label,info)) - else: - self.comments.append('{} {}: {}'.format(label,array.shape[1:],info)) + if info is not None: + if np.prod(data.shape[1:],dtype=int) == 1: + self.comments.append('{}: {}'.format(label,info)) + else: + self.comments.append('{} {}: {}'.format(label,data.shape[1:],info)) if re.match(r'[0-9]*?_',label): idx,key = label.split('_',1) iloc = self.data.columns.get_loc(key).tolist().index(True) + int(idx) -1 - self.data.iloc[:,iloc] = array + self.data.iloc[:,iloc] = data else: - self.data[label] = array.reshape(self.data[label].shape) + self.data[label] = data.reshape(self.data[label].shape) - - def labels(self): - """Return the labels of all columns.""" - return list(self.shapes.keys()) - - def add_array(self,label,array,info): + def add(self,label,data,info=None): """ - Add data to the spreadsheet. + Add column data. Parameters ---------- label : str - Label for the new data. - array : np.ndarray - New data. - info : str - Human-readable information about the new data. + Column label. + data : np.ndarray + Modified data. + info : str, optional + Human-readable information about the modified data. """ - if np.prod(array.shape[1:],dtype=int) == 1: - self.comments.append('{}: {}'.format(label,info)) - else: - self.comments.append('{} {}: {}'.format(label,array.shape[1:],info)) + if info is not None: + if np.prod(data.shape[1:],dtype=int) == 1: + self.comments.append('{}: {}'.format(label,info)) + else: + self.comments.append('{} {}: {}'.format(label,data.shape[1:],info)) - self.shapes[label] = array.shape[1:] if len(array.shape) > 1 else (1,) - size = np.prod(array.shape[1:],dtype=int) - new_data = pd.DataFrame(data=array.reshape(-1,size), + self.shapes[label] = data.shape[1:] if len(data.shape) > 1 else (1,) + size = np.prod(data.shape[1:],dtype=int) + new_data = pd.DataFrame(data=data.reshape(-1,size), columns=[label for l in range(size)]) self.data = pd.concat([self.data,new_data],axis=1) - + + def to_ASCII(self,fname): """ Store as plain text file. diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index c8965d1bd..2fedfdc59 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -20,11 +20,11 @@ def reference_dir(reference_dir_base): class TestTable: def test_get_tensor(self,default): - d = default.get_array('F') + d = default.get('F') assert np.allclose(d,1.0) and d.shape[1:] == (3,3) def test_get_vector(self,default): - d = default.get_array('v') + d = default.get('v') assert np.allclose(d,1.0) and d.shape[1:] == (3,) def test_write_read_str(self,default,tmpdir): @@ -44,30 +44,30 @@ class TestTable: with open(os.path.join(reference_dir,fname)) as f: new = Table.from_ASCII(f) - def test_set_array(self,default): - default.set_array('F',np.zeros((5,3,3)),'set to zero') - d=default.get_array('F') + def test_set(self,default): + default.set('F',np.zeros((5,3,3)),'set to zero') + d=default.get('F') assert np.allclose(d,0.0) and d.shape[1:] == (3,3) def test_labels(self,default): assert default.labels() == ['F','v','s'] - def test_add_array(self,default): + def test_add(self,default): d = np.random.random((5,9)) - default.add_array('nine',d,'random data') - assert np.allclose(d,default.get_array('nine')) + default.add('nine',d,'random data') + assert np.allclose(d,default.get('nine')) def test_invalid_initialization(self,default): - x = default.get_array('v') + x = default.get('v') with pytest.raises(IndexError): Table(x,{'F':(3,3)}) def test_invalid_set(self,default): - x = default.get_array('v') + x = default.get('v') with pytest.raises(ValueError): - default.set_array('F',x,'does not work') + default.set('F',x,'does not work') - def test_invalid_get_array(self,default): + def test_invalid_get(self,default): with pytest.raises(KeyError): - default.get_array('n') + default.get('n') From 469d638afb87c2fa3dc6bc31c2be6df127e42596 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 06:50:06 +0100 Subject: [PATCH 29/38] making table class feature complete --- processing/post/addInfo.py | 29 +++-------------- processing/post/reLabel.py | 64 ++++++-------------------------------- python/damask/table.py | 34 ++++++++++++++++++++ python/tests/test_Table.py | 16 ++++++++++ 4 files changed, 65 insertions(+), 78 deletions(-) diff --git a/processing/post/addInfo.py b/processing/post/addInfo.py index 2d8192cc1..f7a54c3e2 100755 --- a/processing/post/addInfo.py +++ b/processing/post/addInfo.py @@ -24,35 +24,16 @@ parser.add_option('-i', dest = 'info', action = 'extend', metavar = '', help = 'items to add') - (options,filenames) = parser.parse_args() +if filenames == []: filenames = [None] if options.info is None: parser.error('no info specified.') -# --- loop over input files ------------------------------------------------------------------------ - -if filenames == []: filenames = [None] - for name in filenames: - try: table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ assemble header --------------------------------------- + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + table.comments += options.info - table.head_read() - table.info_append(options.info) - table.head_write() - -# ------------------------------------------ pass through data ------------------------------------- - - outputAlive = True - - while outputAlive and table.data_read(): # read next data line of ASCII table - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close ASCII tables + table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/reLabel.py b/processing/post/reLabel.py index e7ad1f1e9..4bd077586 100755 --- a/processing/post/reLabel.py +++ b/processing/post/reLabel.py @@ -35,62 +35,18 @@ parser.set_defaults(label = [], ) (options,filenames) = parser.parse_args() - -pattern = [re.compile('^()(.+)$'), # label pattern for scalar - re.compile('^(\d+_)?(.+)$'), # label pattern for multidimension - ] - -# --- loop over input files ------------------------------------------------------------------------- - if filenames == []: filenames = [None] +if len(options.label) != len(options.substitute): + parser.error('number of column labels and substitutes do not match.') + for name in filenames: - try: table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ read header ------------------------------------------ + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + for i,label in enumerate(options.label): + table.rename(label, + options.substitute[i], + scriptID+' '+' '.join(sys.argv[1:])) - table.head_read() - -# ------------------------------------------ process labels --------------------------------------- - - errors = [] - remarks = [] - - if len(options.label) == 0: - errors.append('no labels specified.') - elif len(options.label) != len(options.substitute): - errors.append('mismatch between number of labels ({}) and substitutes ({}).'.format(len(options.label), - len(options.substitute))) - else: - indices = table.label_index (options.label) - dimensions = table.label_dimension(options.label) - for i,index in enumerate(indices): - if index == -1: remarks.append('label "{}" not present...'.format(options.label[i])) - else: - m = pattern[int(dimensions[i]>1)].match(table.tags[index]) # isolate label name - for j in range(dimensions[i]): - table.tags[index+j] = table.tags[index+j].replace(m.group(2),options.substitute[i]) # replace name with substitute - - if remarks != []: damask.util.croak(remarks) - if errors != []: - damask.util.croak(errors) - table.close(dismiss = True) - continue - -# ------------------------------------------ assemble header --------------------------------------- - - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - table.head_write() - -# ------------------------------------------ process data ------------------------------------------ - - outputAlive = True - while outputAlive and table.data_read(): # read next data line of ASCII table - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close ASCII tables + table.to_ASCII(sys.stdout if name is None else name) diff --git a/python/damask/table.py b/python/damask/table.py index 171d15084..7d413812d 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -161,6 +161,40 @@ class Table(): columns=[label for l in range(size)]) self.data = pd.concat([self.data,new_data],axis=1) + def delete(self,label): + """ + Delete column data. + + Parameters + ---------- + label : str + Column label. + + """ + self.data.drop(columns=label,inplace=True) + + del self.shapes[label] + + def rename(self,label_old,label_new,info=None): + """ + Rename column data. + + Parameters + ---------- + label_old : str + Old column label. + label_new : str + New column label. + + """ + self.data.rename(columns={label_old:label_new},inplace=True) + + comments = '{} => {}'.format(label_old,label_new) + comments += ': {}'.format(info) if info is not None else '' + self.comments.append(comments) + + self.shapes[label_new] = self.shapes.pop(label_old) + def to_ASCII(self,fname): """ diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index 2fedfdc59..b628abe9f 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -57,6 +57,22 @@ class TestTable: default.add('nine',d,'random data') assert np.allclose(d,default.get('nine')) + def test_rename_equivalent(self,default): + v = default.get('v') + default.rename('v','u') + u = default.get('u') + assert np.all(v == u) + + def test_rename_gone(self,default): + default.rename('v','V') + with pytest.raises(KeyError): + default.get('v') + + def test_delete(self,default): + default.delete('v') + with pytest.raises(KeyError): + default.get('v') + def test_invalid_initialization(self,default): x = default.get('v') From 3af696995ddcf65d6a8e8f44df248dc7beab3fa5 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 08:35:34 +0100 Subject: [PATCH 30/38] did not work when piping --- processing/post/addInfo.py | 2 ++ processing/post/reLabel.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/processing/post/addInfo.py b/processing/post/addInfo.py index f7a54c3e2..5e32510db 100755 --- a/processing/post/addInfo.py +++ b/processing/post/addInfo.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import os +import sys +from io import StringIO from optparse import OptionParser import damask diff --git a/processing/post/reLabel.py b/processing/post/reLabel.py index 4bd077586..85d16acea 100755 --- a/processing/post/reLabel.py +++ b/processing/post/reLabel.py @@ -2,8 +2,8 @@ import os import sys +from io import StringIO from optparse import OptionParser -import re import damask From 1bf4946e677858cceac2d7ac2f3032195e4dea5e Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 08:37:39 +0100 Subject: [PATCH 31/38] invalid escape sequence --- python/damask/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/damask/table.py b/python/damask/table.py index 7d413812d..b2be3ae20 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -82,7 +82,7 @@ class Table(): else: shapes[label]=(1,) - data = pd.read_csv(f,names=[i for i in range(len(labels))],sep='\s+').to_numpy() + data = pd.read_csv(f,names=[i for i in range(len(labels))],sep=r'\s+').to_numpy() return Table(data,shapes,comments) From f885748a589239a9e9d64607202423d07c29e315 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 08:43:14 +0100 Subject: [PATCH 32/38] variable was never used --- python/tests/test_Table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index b628abe9f..67bcab557 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -42,7 +42,7 @@ class TestTable: @pytest.mark.parametrize('fname',['datatype-mix.txt','whitespace-mix.txt']) def test_read_strange(self,reference_dir,fname): with open(os.path.join(reference_dir,fname)) as f: - new = Table.from_ASCII(f) + Table.from_ASCII(f) def test_set(self,default): default.set('F',np.zeros((5,3,3)),'set to zero') From 53a38e3a97a2b61d56c857fe3d3970f34f405745 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 10:37:13 +0100 Subject: [PATCH 33/38] polishing --- processing/post/scaleData.py | 6 +++--- processing/post/shiftData.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/processing/post/scaleData.py b/processing/post/scaleData.py index 60ffa41fa..4484c5300 100755 --- a/processing/post/scaleData.py +++ b/processing/post/scaleData.py @@ -22,7 +22,7 @@ Uniformly scale column values by given factor. """, version = scriptID) parser.add_option('-l','--label', - dest = 'label', + dest = 'labels', action = 'extend', metavar = '', help ='column(s) to scale') parser.add_option('-f','--factor', @@ -31,7 +31,7 @@ parser.add_option('-f','--factor', help = 'factor(s) per column') parser.set_defaults(label = [], - ) + factor = []) (options,filenames) = parser.parse_args() if filenames == []: filenames = [None] @@ -43,7 +43,7 @@ for name in filenames: damask.util.report(scriptName,name) table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) - for i,label in enumerate(options.label): + for i,label in enumerate(options.labels): table.set(label, table.get(label)*float(options.factor[i]), scriptID+' '+' '.join(sys.argv[1:])) diff --git a/processing/post/shiftData.py b/processing/post/shiftData.py index 37c72919e..fb2ba48f9 100755 --- a/processing/post/shiftData.py +++ b/processing/post/shiftData.py @@ -22,7 +22,7 @@ Uniformly shift column values by given offset. """, version = scriptID) parser.add_option('-l','--label', - dest = 'label', + dest = 'labels', action = 'extend', metavar = '', help ='column(s) to shift') parser.add_option('-o','--offset', @@ -31,7 +31,7 @@ parser.add_option('-o','--offset', help = 'offset(s) per column') parser.set_defaults(label = [], - ) + offset = []) (options,filenames) = parser.parse_args() if filenames == []: filenames = [None] @@ -43,7 +43,7 @@ for name in filenames: damask.util.report(scriptName,name) table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) - for i,label in enumerate(options.label): + for i,label in enumerate(options.labels): table.set(label, table.get(label)+float(options.offset[i]), scriptID+' '+' '.join(sys.argv[1:])) From 1018b892960692b26f30655c652822309a01a82e Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 10:47:36 +0100 Subject: [PATCH 34/38] nice to have.. --- processing/post/sortTable.py | 45 +++++++----------------------------- python/damask/table.py | 29 ++++++++++++++++++++++- python/tests/test_Table.py | 38 +++++++++++++++++++++++++++--- 3 files changed, 71 insertions(+), 41 deletions(-) diff --git a/processing/post/sortTable.py b/processing/post/sortTable.py index 53a357226..3a3738d18 100755 --- a/processing/post/sortTable.py +++ b/processing/post/sortTable.py @@ -2,10 +2,9 @@ import os import sys +from io import StringIO from optparse import OptionParser -import numpy as np - import damask @@ -26,7 +25,7 @@ With coordinates in columns "x", "y", and "z"; sorting with x slowest and z fast parser.add_option('-l','--label', - dest = 'keys', + dest = 'labels', action = 'extend', metavar = '', help = 'list of column labels (a,b,c,...)') parser.add_option('-r','--reverse', @@ -38,42 +37,14 @@ parser.set_defaults(reverse = False, ) (options,filenames) = parser.parse_args() - - -# --- loop over input files ------------------------------------------------------------------------- - if filenames == []: filenames = [None] +if options.labels is None: + parser.error('no labels specified.') for name in filenames: - try: table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ assemble header --------------------------------------- + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + table.sort_by(options.labels,not options.reverse) - table.head_read() - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - table.head_write() - -# ------------------------------------------ process data --------------------------------------- - - table.data_readArray() - - keys = table.labels(raw = True)[::-1] if options.keys is None else options.keys[::-1] # numpy sorts with most significant column as last - - cols = [] - remarks = [] - for i,column in enumerate(table.label_index(keys)): - if column < 0: remarks.append('label "{}" not present...'.format(keys[i])) - else: cols += [table.data[:,column]] - if remarks != []: damask.util.croak(remarks) - - ind = np.lexsort(cols) if cols != [] else np.arange(table.data.shape[0]) - if options.reverse: ind = ind[::-1] - -# ------------------------------------------ output result --------------------------------------- - - table.data = table.data[ind] - table.data_writeArray() - table.close() # close ASCII table + table.to_ASCII(sys.stdout if name is None else name) diff --git a/python/damask/table.py b/python/damask/table.py index b2be3ae20..3c867fcf5 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -1,3 +1,4 @@ +import random import re import pandas as pd @@ -104,7 +105,7 @@ class Table(): """ if re.match(r'[0-9]*?_',label): idx,key = label.split('_',1) - return self.data[key].to_numpy()[:,int(idx)-1] + return self.data[key].to_numpy()[:,int(idx)-1].reshape((-1,1)) else: return self.data[label].to_numpy().reshape((-1,)+self.shapes[label]) @@ -196,6 +197,32 @@ class Table(): self.shapes[label_new] = self.shapes.pop(label_old) + def sort_by(self,labels,ascending=True): + """ + Get column data. + + Parameters + ---------- + label : list of str or str + Column labels. + ascending : bool, optional + Set sort order. + + """ + _temp = [] + _labels = [] + for label in labels if isinstance(labels,list) else [labels]: + if re.match(r'[0-9]*?_',label): + _temp.append(str(random.getrandbits(128))) + self.add(_temp[-1],self.get(label)) + _labels.append(_temp[-1]) + else: + _labels.append(label) + + self.data.sort_values(_labels,axis=0,inplace=True,ascending=ascending) + for t in _temp: self.delete(t) + self.comments.append('sorted by [{}]'.format(', '.join(labels))) + def to_ASCII(self,fname): """ Store as plain text file. diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index 67bcab557..b17348881 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -19,13 +19,21 @@ def reference_dir(reference_dir_base): class TestTable: - def test_get_tensor(self,default): - d = default.get('F') - assert np.allclose(d,1.0) and d.shape[1:] == (3,3) + def test_get_scalar(self,default): + d = default.get('s') + assert np.allclose(d,1.0) and d.shape[1:] == (1,) def test_get_vector(self,default): d = default.get('v') assert np.allclose(d,1.0) and d.shape[1:] == (3,) + + def test_get_tensor(self,default): + d = default.get('F') + assert np.allclose(d,1.0) and d.shape[1:] == (3,3) + + def test_get_component(self,default): + d = default.get('5_F') + assert np.allclose(d,1.0) and d.shape[1:] == (1,) def test_write_read_str(self,default,tmpdir): default.to_ASCII(str(tmpdir.join('default.txt'))) @@ -87,3 +95,27 @@ class TestTable: def test_invalid_get(self,default): with pytest.raises(KeyError): default.get('n') + + def test_sort_scalar(self): + x = np.random.random((5,13)) + t = Table(x,{'F':(3,3),'v':(3,),'s':(1,)},['random test data']) + unsort = t.get('s') + t.sort_by('s') + sort = t.get('s') + assert np.all(np.sort(unsort,0)==sort) + + def test_sort_component(self): + x = np.random.random((5,12)) + t = Table(x,{'F':(3,3),'v':(3,)},['random test data']) + unsort = t.get('4_F') + t.sort_by('4_F') + sort = t.get('4_F') + assert np.all(np.sort(unsort,0)==sort) + + def test_sort_revert(self): + x = np.random.random((5,12)) + t = Table(x,{'F':(3,3),'v':(3,)},['random test data']) + t.sort_by('4_F',False) + sort = t.get('4_F') + assert np.all(np.sort(sort,0)==sort[::-1,:]) + From 90fdb24faaffa217be64f374657f4981fac91ccc Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 13:26:33 +0100 Subject: [PATCH 35/38] forgotten variable rename --- processing/post/scaleData.py | 2 +- processing/post/shiftData.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/post/scaleData.py b/processing/post/scaleData.py index 4484c5300..58f853251 100755 --- a/processing/post/scaleData.py +++ b/processing/post/scaleData.py @@ -36,7 +36,7 @@ parser.set_defaults(label = [], (options,filenames) = parser.parse_args() if filenames == []: filenames = [None] -if len(options.label) != len(options.factor): +if len(options.labels) != len(options.factor): parser.error('number of column labels and factors do not match.') for name in filenames: diff --git a/processing/post/shiftData.py b/processing/post/shiftData.py index fb2ba48f9..57b20fbd0 100755 --- a/processing/post/shiftData.py +++ b/processing/post/shiftData.py @@ -36,7 +36,7 @@ parser.set_defaults(label = [], (options,filenames) = parser.parse_args() if filenames == []: filenames = [None] -if len(options.label) != len(options.offset): +if len(options.labels) != len(options.offset): parser.error('number of column labels and offsets do not match.') for name in filenames: From 4bc037107bbc51db5bf71ccae870961b962109e2 Mon Sep 17 00:00:00 2001 From: Test User Date: Thu, 5 Dec 2019 13:35:23 +0100 Subject: [PATCH 36/38] [skip ci] updated version information after successful test of v2.0.3-1136-gcc67f0e1 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1fe12d401..63973c11b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.0.3-1133-gfede8225 +v2.0.3-1136-gcc67f0e1 From 3caf2c129695490425f18d59850d98f83062ac94 Mon Sep 17 00:00:00 2001 From: Philip Eisenlohr Date: Thu, 5 Dec 2019 09:05:50 -0500 Subject: [PATCH 37/38] condensing code --- python/damask/table.py | 89 +++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/python/damask/table.py b/python/damask/table.py index 3c867fcf5..9e7b57017 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -21,27 +21,27 @@ class Table(): Additional, human-readable information. """ + self.comments = [] if comments is None else [c for c in comments] self.data = pd.DataFrame(data=data) - - labels = {} - i = 0 - for label in shapes.keys(): - for components in range(np.prod(shapes[label])): - labels[i] = label - i+=1 - - if i != self.data.shape[1]: - raise IndexError('Shape mismatch between shapes and data') - - self.data.rename(columns=labels,inplace=True) - - if comments is None: - self.comments = [] - else: - self.comments = [c for c in comments] - self.shapes = shapes + labels = [] + for label,shape in self.shapes.items(): + labels += [label] * np.prod(shape) + + if len(labels) != self.data.shape[1]: + raise IndexError('Mismatch between shapes and data') + + self.data.rename(columns=dict(zip(range(len(labels)),labels)),inplace=True) + + + def __add_comment(self,label,shape,info): + if info is not None: + self.comments.append('{}{}: {}'.format(label, + ' '+str(shape) if np.prod(shape,dtype=int) > 1 else '', + info)) + + @staticmethod def from_ASCII(fname): """ @@ -67,7 +67,7 @@ class Table(): header = int(header) else: raise Exception - comments = [f.readline()[:-1] for i in range(header-1)] + comments = [f.readline()[:-1] for i in range(1,header)] labels = f.readline().split() shapes = {} @@ -81,10 +81,10 @@ class Table(): if vector_column: shapes[label.split('_',1)[1]] = (int(label.split('_',1)[0]),) else: - shapes[label]=(1,) - - data = pd.read_csv(f,names=[i for i in range(len(labels))],sep=r'\s+').to_numpy() + shapes[label] = (1,) + data = pd.read_csv(f,names=list(range(len(labels))),sep=r'\s+').to_numpy() + return Table(data,shapes,comments) @@ -107,7 +107,8 @@ class Table(): idx,key = label.split('_',1) return self.data[key].to_numpy()[:,int(idx)-1].reshape((-1,1)) else: - return self.data[label].to_numpy().reshape((-1,)+self.shapes[label]) + return self.data[label].to_numpy().reshape((-1,)+self.shapes[label]) # better return shape (N) instead of (N,1), i.e. no reshaping? + def set(self,label,data,info=None): """ @@ -123,11 +124,7 @@ class Table(): Human-readable information about the new data. """ - if info is not None: - if np.prod(data.shape[1:],dtype=int) == 1: - self.comments.append('{}: {}'.format(label,info)) - else: - self.comments.append('{} {}: {}'.format(label,data.shape[1:],info)) + self.__add_comment(label,data.shape[1:],info) if re.match(r'[0-9]*?_',label): idx,key = label.split('_',1) @@ -136,6 +133,7 @@ class Table(): else: self.data[label] = data.reshape(self.data[label].shape) + def add(self,label,data,info=None): """ Add column data. @@ -150,17 +148,15 @@ class Table(): Human-readable information about the modified data. """ - if info is not None: - if np.prod(data.shape[1:],dtype=int) == 1: - self.comments.append('{}: {}'.format(label,info)) - else: - self.comments.append('{} {}: {}'.format(label,data.shape[1:],info)) + self.__add_comment(label,data.shape[1:],info) self.shapes[label] = data.shape[1:] if len(data.shape) > 1 else (1,) size = np.prod(data.shape[1:],dtype=int) - new_data = pd.DataFrame(data=data.reshape(-1,size), - columns=[label for l in range(size)]) - self.data = pd.concat([self.data,new_data],axis=1) + self.data = pd.concat([self.data, + pd.DataFrame(data=data.reshape(-1,size), + columns=[label]*size)], + axis=1) + def delete(self,label): """ @@ -176,6 +172,7 @@ class Table(): del self.shapes[label] + def rename(self,label_old,label_new,info=None): """ Rename column data. @@ -190,9 +187,10 @@ class Table(): """ self.data.rename(columns={label_old:label_new},inplace=True) - comments = '{} => {}'.format(label_old,label_new) - comments += ': {}'.format(info) if info is not None else '' - self.comments.append(comments) + self.comments.append('{} => {}{}'.format(label_old, + label_new, + '' if info is None else ': {}'.format(info), + )) self.shapes[label_new] = self.shapes.pop(label_old) @@ -223,6 +221,7 @@ class Table(): for t in _temp: self.delete(t) self.comments.append('sorted by [{}]'.format(', '.join(labels))) + def to_ASCII(self,fname): """ Store as plain text file. @@ -238,14 +237,14 @@ class Table(): if(self.shapes[l] == (1,)): labels.append('{}'.format(l)) elif(len(self.shapes[l]) == 1): - labels+=['{}_{}'.format(i+1,l)\ - for i in range(self.shapes[l][0])] + labels += ['{}_{}'.format(i+1,l) \ + for i in range(self.shapes[l][0])] else: - labels+=['{}:{}_{}'.format('x'.join([str(d) for d in self.shapes[l]]),i+1,l)\ - for i in range(np.prod(self.shapes[l],dtype=int))] + labels += ['{}:{}_{}'.format('x'.join([str(d) for d in self.shapes[l]]),i+1,l) \ + for i in range(np.prod(self.shapes[l],dtype=int))] - header = ['{} header'.format(len(self.comments)+1)]\ - + self.comments\ + header = ['{} header'.format(len(self.comments)+1)] \ + + self.comments \ + [' '.join(labels)] try: From 2648a67dcdbf14db825ec310fb2c76b1733e0a8b Mon Sep 17 00:00:00 2001 From: Philip Eisenlohr Date: Thu, 5 Dec 2019 12:00:59 -0500 Subject: [PATCH 38/38] fixed sort_by to respect updated index (upon subsequent adding of new data) --- python/damask/table.py | 60 ++++++++++++++++++++------------------ python/tests/test_Table.py | 19 ++++++++---- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/python/damask/table.py b/python/damask/table.py index 9e7b57017..6181fdb1f 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -1,4 +1,3 @@ -import random import re import pandas as pd @@ -24,15 +23,24 @@ class Table(): self.comments = [] if comments is None else [c for c in comments] self.data = pd.DataFrame(data=data) self.shapes = shapes + self.__label_condensed() + + def __label_flat(self): + """Label data individually, e.g. v v v ==> 1_v 2_v 3_v.""" + labels = [] + for label,shape in self.shapes.items(): + size = np.prod(shape) + labels += ['{}{}'.format('' if size == 1 else '{}_'.format(i+1),label) for i in range(size)] + self.data.columns = labels + + + def __label_condensed(self): + """Label data condensed, e.g. 1_v 2_v 3_v ==> v v v.""" labels = [] for label,shape in self.shapes.items(): labels += [label] * np.prod(shape) - - if len(labels) != self.data.shape[1]: - raise IndexError('Mismatch between shapes and data') - - self.data.rename(columns=dict(zip(range(len(labels)),labels)),inplace=True) + self.data.columns = labels def __add_comment(self,label,shape,info): @@ -87,7 +95,7 @@ class Table(): return Table(data,shapes,comments) - + @property def labels(self): """Return the labels of all columns.""" return list(self.shapes.keys()) @@ -105,9 +113,11 @@ class Table(): """ if re.match(r'[0-9]*?_',label): idx,key = label.split('_',1) - return self.data[key].to_numpy()[:,int(idx)-1].reshape((-1,1)) + data = self.data[key].to_numpy()[:,int(idx)-1].reshape((-1,1)) else: - return self.data[label].to_numpy().reshape((-1,)+self.shapes[label]) # better return shape (N) instead of (N,1), i.e. no reshaping? + data = self.data[label].to_numpy().reshape((-1,)+self.shapes[label]) + + return data.astype(type(data.flatten()[0])) def set(self,label,data,info=None): @@ -149,13 +159,14 @@ class Table(): """ self.__add_comment(label,data.shape[1:],info) - + self.shapes[label] = data.shape[1:] if len(data.shape) > 1 else (1,) - size = np.prod(data.shape[1:],dtype=int) - self.data = pd.concat([self.data, - pd.DataFrame(data=data.reshape(-1,size), - columns=[label]*size)], - axis=1) + size = np.prod(data.shape[1:],dtype=int) + new = pd.DataFrame(data=data.reshape(-1,size), + columns=[label]*size, + ) + new.index = self.data.index + self.data = pd.concat([self.data,new],axis=1) def delete(self,label): @@ -201,24 +212,15 @@ class Table(): Parameters ---------- - label : list of str or str + label : str or list Column labels. - ascending : bool, optional + ascending : bool or list, optional Set sort order. """ - _temp = [] - _labels = [] - for label in labels if isinstance(labels,list) else [labels]: - if re.match(r'[0-9]*?_',label): - _temp.append(str(random.getrandbits(128))) - self.add(_temp[-1],self.get(label)) - _labels.append(_temp[-1]) - else: - _labels.append(label) - - self.data.sort_values(_labels,axis=0,inplace=True,ascending=ascending) - for t in _temp: self.delete(t) + self.__label_flat() + self.data.sort_values(labels,axis=0,inplace=True,ascending=ascending) + self.__label_condensed() self.comments.append('sorted by [{}]'.format(', '.join(labels))) diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index b17348881..a0dc31975 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -9,7 +9,7 @@ from damask import Table @pytest.fixture def default(): """Simple Table.""" - x = np.ones((5,13)) + x = np.ones((5,13),dtype=float) return Table(x,{'F':(3,3),'v':(3,),'s':(1,)},['test data','contains only ones']) @pytest.fixture @@ -58,7 +58,7 @@ class TestTable: assert np.allclose(d,0.0) and d.shape[1:] == (3,3) def test_labels(self,default): - assert default.labels() == ['F','v','s'] + assert default.labels == ['F','v','s'] def test_add(self,default): d = np.random.random((5,9)) @@ -82,9 +82,9 @@ class TestTable: default.get('v') - def test_invalid_initialization(self,default): - x = default.get('v') - with pytest.raises(IndexError): + def test_invalid_initialization(self): + x = np.random.random((5,10)) + with pytest.raises(ValueError): Table(x,{'F':(3,3)}) def test_invalid_set(self,default): @@ -115,7 +115,14 @@ class TestTable: def test_sort_revert(self): x = np.random.random((5,12)) t = Table(x,{'F':(3,3),'v':(3,)},['random test data']) - t.sort_by('4_F',False) + t.sort_by('4_F',ascending=False) sort = t.get('4_F') assert np.all(np.sort(sort,0)==sort[::-1,:]) + def test_sort(self): + t = Table(np.array([[0,1,],[2,1,]]), + {'v':(2,)}, + ['test data']) + t.add('s',np.array(['b','a'])) + t.sort_by('s') + assert np.all(t.get('1_v') == np.array([2,0]).reshape((2,1)))