From 07e97787985281a7394fcc88c75ad226c2880f72 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 4 Dec 2019 10:20:03 +0100 Subject: [PATCH 01/12] one more test --- python/tests/test_Table.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index 801df0392..c34f9ee3a 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -55,3 +55,7 @@ class TestTable: x = default.get_array('v') with pytest.raises(ValueError): default.set_array('F',x,'does not work') + + def test_invalid_get_array(self,default): + with pytest.raises(KeyError): + default.get_array('n') From 1c835725896b93fdd48c64a3aa87a5dccc3d240c Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 05:00:26 +0100 Subject: [PATCH 02/12] more descriptive names --- python/damask/table.py | 60 +++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/python/damask/table.py b/python/damask/table.py index 7d9e65961..bb195122d 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -6,40 +6,40 @@ import numpy as np class Table(): """Store spreadsheet-like data.""" - def __init__(self,array,headings,comments=None): + def __init__(self,data,shapes,comments=None): """ - New spreadsheet data. + New spreadsheet. Parameters ---------- - array : numpy.ndarray + data : numpy.ndarray Data. - headings : dict - Column headings. Labels as keys and shape as tuple. Example 'F':(3,3) for a deformation gradient. + shapes : dict with str:tuple pairs + Shapes of the columns. Example 'F':(3,3) for a deformation gradient. comments : iterable of str, optional - Additional, human-readable information + Additional, human-readable information. """ - self.data = pd.DataFrame(data=array) + self.data = pd.DataFrame(data=data) - d = {} + labels = {} i = 0 - for label in headings: - for components in range(np.prod(headings[label])): - d[i] = label + for label in shapes.keys(): + for components in range(np.prod(shapes[label])): + labels[i] = label i+=1 if i != self.data.shape[1]: - raise IndexError('Mismatch between array shape and headings') + raise IndexError('Shape mismatch between shapes and data') - self.data.rename(columns=d,inplace=True) + self.data.rename(columns=labels,inplace=True) if comments is None: self.comments = [] else: self.comments = [c for c in comments] - self.headings = headings + self.shapes = shapes @staticmethod def from_ASCII(fname): @@ -47,8 +47,8 @@ class Table(): Create table from ASCII file. The first line needs to indicate the number of subsequent header lines as 'n header'. - Vector data labels are indicated by '1_v, 2_v, ..., n_v'. - Tensor data labels are indicated by '3x3:1_T, 3x3:2_T, ..., 3x3:9_T'. + Vector data column labels are indicated by '1_v, 2_v, ..., n_v'. + Tensor data column labels are indicated by '3x3:1_T, 3x3:2_T, ..., 3x3:9_T'. Parameters ---------- @@ -69,20 +69,20 @@ class Table(): comments = [f.readline()[:-1] for i in range(header-1)] labels = f.readline().split() - headings = {} + shapes = {} for label in labels: tensor_column = re.search(r'[0-9,x]*?:[0-9]*?_',label) if tensor_column: my_shape = tensor_column.group().split(':',1)[0].split('x') - headings[label.split('_',1)[1]] = tuple([int(d) for d in my_shape]) + shapes[label.split('_',1)[1]] = tuple([int(d) for d in my_shape]) else: vector_column = re.match(r'[0-9]*?_',label) if vector_column: - headings[label.split('_',1)[1]] = (int(label.split('_',1)[0]),) + shapes[label.split('_',1)[1]] = (int(label.split('_',1)[0]),) else: - headings[label]=(1,) + shapes[label]=(1,) - return Table(np.loadtxt(f),headings,comments) + return Table(np.loadtxt(f),shapes,comments) def get_array(self,label): """ @@ -98,7 +98,7 @@ class Table(): idx,key = label.split('_',1) return self.data[key].to_numpy()[:,int(idx)-1] else: - return self.data[label].to_numpy().reshape((-1,)+self.headings[label]) + return self.data[label].to_numpy().reshape((-1,)+self.shapes[label]) def set_array(self,label,array,info): """ @@ -129,7 +129,7 @@ class Table(): def get_labels(self): """Return the labels of all columns.""" - return [label for label in self.headings] + return list(self.shapes.keys()) def add_array(self,label,array,info): """ @@ -150,7 +150,7 @@ class Table(): else: self.comments.append('{} {}: {}'.format(label,array.shape[1:],info)) - self.headings[label] = array.shape[1:] if len(array.shape) > 1 else (1,) + self.shapes[label] = array.shape[1:] if len(array.shape) > 1 else (1,) size = np.prod(array.shape[1:],dtype=int) new_data = pd.DataFrame(data=array.reshape(-1,size), columns=[label for l in range(size)]) @@ -167,15 +167,15 @@ class Table(): """ labels = [] - for l in self.headings: - if(self.headings[l] == (1,)): + for l in self.shapes: + if(self.shapes[l] == (1,)): labels.append('{}'.format(l)) - elif(len(self.headings[l]) == 1): + elif(len(self.shapes[l]) == 1): labels+=['{}_{}'.format(i+1,l)\ - for i in range(self.headings[l][0])] + for i in range(self.shapes[l][0])] else: - labels+=['{}:{}_{}'.format('x'.join([str(d) for d in self.headings[l]]),i+1,l)\ - for i in range(np.prod(self.headings[l],dtype=int))] + labels+=['{}:{}_{}'.format('x'.join([str(d) for d in self.shapes[l]]),i+1,l)\ + for i in range(np.prod(self.shapes[l],dtype=int))] header = ['{} header'.format(len(self.comments)+1)]\ + self.comments\ From 5fdf112f78de93711a5bd03fa11163c9d6536587 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 05:45:27 +0100 Subject: [PATCH 03/12] allow strings as data in ASCII file --- python/damask/table.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/damask/table.py b/python/damask/table.py index bb195122d..140665ab2 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -82,7 +82,10 @@ class Table(): else: shapes[label]=(1,) - return Table(np.loadtxt(f),shapes,comments) + data = pd.read_csv(f,names=[i for i in range(len(labels))],sep='\s+').to_numpy() + + return Table(data,shapes,comments) + def get_array(self,label): """ From 2172e92e7e2457633afc43fd9cd6740e14b9eafe Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 05:57:47 +0100 Subject: [PATCH 04/12] test exotic input files --- python/tests/reference/Table/datatype-mix.txt | 4 ++++ python/tests/reference/Table/whitespace-mix.txt | 6 ++++++ python/tests/test_Table.py | 12 ++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 python/tests/reference/Table/datatype-mix.txt create mode 100644 python/tests/reference/Table/whitespace-mix.txt diff --git a/python/tests/reference/Table/datatype-mix.txt b/python/tests/reference/Table/datatype-mix.txt new file mode 100644 index 000000000..2f6baa852 --- /dev/null +++ b/python/tests/reference/Table/datatype-mix.txt @@ -0,0 +1,4 @@ +1 header +a b +1.0 hallo +0.1 "hallo test" diff --git a/python/tests/reference/Table/whitespace-mix.txt b/python/tests/reference/Table/whitespace-mix.txt new file mode 100644 index 000000000..933a16e77 --- /dev/null +++ b/python/tests/reference/Table/whitespace-mix.txt @@ -0,0 +1,6 @@ +1 header +a b 1_c 2_c +1 2 3 4 +5 6 7 8 +9 10. 12. 12 + diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index c34f9ee3a..e7550984d 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -1,14 +1,21 @@ +import os + import pytest import numpy as np from damask import Table + @pytest.fixture def default(): """Simple Table.""" x = np.ones((5,13)) return Table(x,{'F':(3,3),'v':(3,),'s':(1,)},['test data','contains only ones']) +@pytest.fixture +def reference_dir(reference_dir_base): + """Directory containing reference results.""" + return os.path.join(reference_dir_base,'Table') class TestTable: @@ -31,6 +38,11 @@ class TestTable: with open(tmpdir.join('default.txt')) as f: new = Table.from_ASCII(f) assert all(default.data==new.data) + + @pytest.mark.parametrize('fname',['datatype-mix.txt','whitespace-mix.txt']) + def test_read_strange(self,reference_dir,fname): + with open(os.path.join(reference_dir,fname)) as f: + new = Table.from_ASCII(f) def test_set_array(self,default): default.set_array('F',np.zeros((5,3,3)),'set to zero') From e7a67262f809eacf6af7b8286740c861d5159bb2 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 06:00:49 +0100 Subject: [PATCH 05/12] clear enough --- python/damask/table.py | 2 +- python/tests/test_Table.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/damask/table.py b/python/damask/table.py index 140665ab2..56507847e 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -130,7 +130,7 @@ class Table(): self.data[label] = array.reshape(self.data[label].shape) - def get_labels(self): + def labels(self): """Return the labels of all columns.""" return list(self.shapes.keys()) diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index e7550984d..c8965d1bd 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -49,8 +49,8 @@ class TestTable: d=default.get_array('F') assert np.allclose(d,0.0) and d.shape[1:] == (3,3) - def test_get_labels(self,default): - assert default.get_labels() == ['F','v','s'] + def test_labels(self,default): + assert default.labels() == ['F','v','s'] def test_add_array(self,default): d = np.random.random((5,9)) From ee8e3386f4db6fea6b14926595a2f8c38a902750 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 06:10:27 +0100 Subject: [PATCH 06/12] shorter but still descriptive names --- processing/post/addCauchy.py | 8 ++-- processing/post/addDeterminant.py | 6 +-- processing/post/addDeviator.py | 12 +++--- processing/post/addMises.py | 12 +++--- processing/post/addPK2.py | 8 ++-- processing/post/scaleData.py | 5 ++- processing/post/shiftData.py | 5 ++- python/damask/table.py | 71 ++++++++++++++++--------------- python/tests/test_Table.py | 26 +++++------ 9 files changed, 79 insertions(+), 74 deletions(-) diff --git a/processing/post/addCauchy.py b/processing/post/addCauchy.py index ad29792a6..afc5a57be 100755 --- a/processing/post/addCauchy.py +++ b/processing/post/addCauchy.py @@ -41,9 +41,9 @@ for name in filenames: damask.util.report(scriptName,name) table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) - table.add_array('Cauchy', - damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3), - table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('Cauchy', + damask.mechanics.Cauchy(table.get(options.defgrad).reshape(-1,3,3), + table.get(options.stress ).reshape(-1,3,3)).reshape(-1,9), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addDeterminant.py b/processing/post/addDeterminant.py index 4c3f9d260..f2368559d 100755 --- a/processing/post/addDeterminant.py +++ b/processing/post/addDeterminant.py @@ -38,8 +38,8 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) for tensor in options.tensor: - table.add_array('det({})'.format(tensor), - np.linalg.det(table.get_array(tensor).reshape(-1,3,3)), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('det({})'.format(tensor), + np.linalg.det(table.get(tensor).reshape(-1,3,3)), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addDeviator.py b/processing/post/addDeviator.py index 7807c7f5d..ca06034b3 100755 --- a/processing/post/addDeviator.py +++ b/processing/post/addDeviator.py @@ -40,12 +40,12 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) for tensor in options.tensor: - table.add_array('dev({})'.format(tensor), - damask.mechanics.deviatoric_part(table.get_array(tensor).reshape(-1,3,3)).reshape((-1,9)), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('dev({})'.format(tensor), + damask.mechanics.deviatoric_part(table.get(tensor).reshape(-1,3,3)).reshape((-1,9)), + scriptID+' '+' '.join(sys.argv[1:])) if options.spherical: - table.add_array('sph({})'.format(tensor), - damask.mechanics.spherical_part(table.get_array(tensor).reshape(-1,3,3)), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('sph({})'.format(tensor), + damask.mechanics.spherical_part(table.get(tensor).reshape(-1,3,3)), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addMises.py b/processing/post/addMises.py index 9bc560f2d..0c2a6db50 100755 --- a/processing/post/addMises.py +++ b/processing/post/addMises.py @@ -45,12 +45,12 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) for strain in options.strain: - table.add_array('Mises({})'.format(strain), - damask.mechanics.Mises_strain(damask.mechanics.symmetric(table.get_array(strain).reshape(-1,3,3))), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('Mises({})'.format(strain), + damask.mechanics.Mises_strain(damask.mechanics.symmetric(table.get(strain).reshape(-1,3,3))), + scriptID+' '+' '.join(sys.argv[1:])) for stress in options.stress: - table.add_array('Mises({})'.format(stress), - damask.mechanics.Mises_stress(damask.mechanics.symmetric(table.get_array(stress).reshape(-1,3,3))), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('Mises({})'.format(stress), + damask.mechanics.Mises_stress(damask.mechanics.symmetric(table.get(stress).reshape(-1,3,3))), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/addPK2.py b/processing/post/addPK2.py index 45afdbcd1..185160d79 100755 --- a/processing/post/addPK2.py +++ b/processing/post/addPK2.py @@ -42,9 +42,9 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) - table.add_array('S', - damask.mechanics.PK2(table.get_array(options.defgrad).reshape(-1,3,3), - table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9), - scriptID+' '+' '.join(sys.argv[1:])) + table.add('S', + damask.mechanics.PK2(table.get(options.defgrad).reshape(-1,3,3), + table.get(options.stress ).reshape(-1,3,3)).reshape(-1,9), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/scaleData.py b/processing/post/scaleData.py index ad57f4b7e..60ffa41fa 100755 --- a/processing/post/scaleData.py +++ b/processing/post/scaleData.py @@ -44,7 +44,8 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) for i,label in enumerate(options.label): - table.set_array(label,table.get_array(label)*float(options.factor[i]), - scriptID+' '+' '.join(sys.argv[1:])) + table.set(label, + table.get(label)*float(options.factor[i]), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/shiftData.py b/processing/post/shiftData.py index 591e68c5d..37c72919e 100755 --- a/processing/post/shiftData.py +++ b/processing/post/shiftData.py @@ -44,7 +44,8 @@ for name in filenames: table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) for i,label in enumerate(options.label): - table.set_array(label,table.get_array(label)+float(options.offset[i]), - scriptID+' '+' '.join(sys.argv[1:])) + table.set(label, + table.get(label)+float(options.offset[i]), + scriptID+' '+' '.join(sys.argv[1:])) table.to_ASCII(sys.stdout if name is None else name) diff --git a/python/damask/table.py b/python/damask/table.py index 56507847e..171d15084 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -87,14 +87,19 @@ class Table(): return Table(data,shapes,comments) - def get_array(self,label): + def labels(self): + """Return the labels of all columns.""" + return list(self.shapes.keys()) + + + def get(self,label): """ - Return data as array. + Get column data. Parameters ---------- label : str - Label of the array. + Column label. """ if re.match(r'[0-9]*?_',label): @@ -103,62 +108,60 @@ class Table(): else: return self.data[label].to_numpy().reshape((-1,)+self.shapes[label]) - def set_array(self,label,array,info): + def set(self,label,data,info=None): """ - Modify data in the spreadsheet. + Set column data. Parameters ---------- label : str - Label for the new data. - array : np.ndarray + Column label. + data : np.ndarray New data. - info : str + info : str, optional Human-readable information about the new data. """ - if np.prod(array.shape[1:],dtype=int) == 1: - self.comments.append('{}: {}'.format(label,info)) - else: - self.comments.append('{} {}: {}'.format(label,array.shape[1:],info)) + if info is not None: + if np.prod(data.shape[1:],dtype=int) == 1: + self.comments.append('{}: {}'.format(label,info)) + else: + self.comments.append('{} {}: {}'.format(label,data.shape[1:],info)) if re.match(r'[0-9]*?_',label): idx,key = label.split('_',1) iloc = self.data.columns.get_loc(key).tolist().index(True) + int(idx) -1 - self.data.iloc[:,iloc] = array + self.data.iloc[:,iloc] = data else: - self.data[label] = array.reshape(self.data[label].shape) + self.data[label] = data.reshape(self.data[label].shape) - - def labels(self): - """Return the labels of all columns.""" - return list(self.shapes.keys()) - - def add_array(self,label,array,info): + def add(self,label,data,info=None): """ - Add data to the spreadsheet. + Add column data. Parameters ---------- label : str - Label for the new data. - array : np.ndarray - New data. - info : str - Human-readable information about the new data. + Column label. + data : np.ndarray + Modified data. + info : str, optional + Human-readable information about the modified data. """ - if np.prod(array.shape[1:],dtype=int) == 1: - self.comments.append('{}: {}'.format(label,info)) - else: - self.comments.append('{} {}: {}'.format(label,array.shape[1:],info)) + if info is not None: + if np.prod(data.shape[1:],dtype=int) == 1: + self.comments.append('{}: {}'.format(label,info)) + else: + self.comments.append('{} {}: {}'.format(label,data.shape[1:],info)) - self.shapes[label] = array.shape[1:] if len(array.shape) > 1 else (1,) - size = np.prod(array.shape[1:],dtype=int) - new_data = pd.DataFrame(data=array.reshape(-1,size), + self.shapes[label] = data.shape[1:] if len(data.shape) > 1 else (1,) + size = np.prod(data.shape[1:],dtype=int) + new_data = pd.DataFrame(data=data.reshape(-1,size), columns=[label for l in range(size)]) self.data = pd.concat([self.data,new_data],axis=1) - + + def to_ASCII(self,fname): """ Store as plain text file. diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index c8965d1bd..2fedfdc59 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -20,11 +20,11 @@ def reference_dir(reference_dir_base): class TestTable: def test_get_tensor(self,default): - d = default.get_array('F') + d = default.get('F') assert np.allclose(d,1.0) and d.shape[1:] == (3,3) def test_get_vector(self,default): - d = default.get_array('v') + d = default.get('v') assert np.allclose(d,1.0) and d.shape[1:] == (3,) def test_write_read_str(self,default,tmpdir): @@ -44,30 +44,30 @@ class TestTable: with open(os.path.join(reference_dir,fname)) as f: new = Table.from_ASCII(f) - def test_set_array(self,default): - default.set_array('F',np.zeros((5,3,3)),'set to zero') - d=default.get_array('F') + def test_set(self,default): + default.set('F',np.zeros((5,3,3)),'set to zero') + d=default.get('F') assert np.allclose(d,0.0) and d.shape[1:] == (3,3) def test_labels(self,default): assert default.labels() == ['F','v','s'] - def test_add_array(self,default): + def test_add(self,default): d = np.random.random((5,9)) - default.add_array('nine',d,'random data') - assert np.allclose(d,default.get_array('nine')) + default.add('nine',d,'random data') + assert np.allclose(d,default.get('nine')) def test_invalid_initialization(self,default): - x = default.get_array('v') + x = default.get('v') with pytest.raises(IndexError): Table(x,{'F':(3,3)}) def test_invalid_set(self,default): - x = default.get_array('v') + x = default.get('v') with pytest.raises(ValueError): - default.set_array('F',x,'does not work') + default.set('F',x,'does not work') - def test_invalid_get_array(self,default): + def test_invalid_get(self,default): with pytest.raises(KeyError): - default.get_array('n') + default.get('n') From 469d638afb87c2fa3dc6bc31c2be6df127e42596 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 06:50:06 +0100 Subject: [PATCH 07/12] making table class feature complete --- processing/post/addInfo.py | 29 +++-------------- processing/post/reLabel.py | 64 ++++++-------------------------------- python/damask/table.py | 34 ++++++++++++++++++++ python/tests/test_Table.py | 16 ++++++++++ 4 files changed, 65 insertions(+), 78 deletions(-) diff --git a/processing/post/addInfo.py b/processing/post/addInfo.py index 2d8192cc1..f7a54c3e2 100755 --- a/processing/post/addInfo.py +++ b/processing/post/addInfo.py @@ -24,35 +24,16 @@ parser.add_option('-i', dest = 'info', action = 'extend', metavar = '', help = 'items to add') - (options,filenames) = parser.parse_args() +if filenames == []: filenames = [None] if options.info is None: parser.error('no info specified.') -# --- loop over input files ------------------------------------------------------------------------ - -if filenames == []: filenames = [None] - for name in filenames: - try: table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ assemble header --------------------------------------- + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + table.comments += options.info - table.head_read() - table.info_append(options.info) - table.head_write() - -# ------------------------------------------ pass through data ------------------------------------- - - outputAlive = True - - while outputAlive and table.data_read(): # read next data line of ASCII table - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close ASCII tables + table.to_ASCII(sys.stdout if name is None else name) diff --git a/processing/post/reLabel.py b/processing/post/reLabel.py index e7ad1f1e9..4bd077586 100755 --- a/processing/post/reLabel.py +++ b/processing/post/reLabel.py @@ -35,62 +35,18 @@ parser.set_defaults(label = [], ) (options,filenames) = parser.parse_args() - -pattern = [re.compile('^()(.+)$'), # label pattern for scalar - re.compile('^(\d+_)?(.+)$'), # label pattern for multidimension - ] - -# --- loop over input files ------------------------------------------------------------------------- - if filenames == []: filenames = [None] +if len(options.label) != len(options.substitute): + parser.error('number of column labels and substitutes do not match.') + for name in filenames: - try: table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ read header ------------------------------------------ + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + for i,label in enumerate(options.label): + table.rename(label, + options.substitute[i], + scriptID+' '+' '.join(sys.argv[1:])) - table.head_read() - -# ------------------------------------------ process labels --------------------------------------- - - errors = [] - remarks = [] - - if len(options.label) == 0: - errors.append('no labels specified.') - elif len(options.label) != len(options.substitute): - errors.append('mismatch between number of labels ({}) and substitutes ({}).'.format(len(options.label), - len(options.substitute))) - else: - indices = table.label_index (options.label) - dimensions = table.label_dimension(options.label) - for i,index in enumerate(indices): - if index == -1: remarks.append('label "{}" not present...'.format(options.label[i])) - else: - m = pattern[int(dimensions[i]>1)].match(table.tags[index]) # isolate label name - for j in range(dimensions[i]): - table.tags[index+j] = table.tags[index+j].replace(m.group(2),options.substitute[i]) # replace name with substitute - - if remarks != []: damask.util.croak(remarks) - if errors != []: - damask.util.croak(errors) - table.close(dismiss = True) - continue - -# ------------------------------------------ assemble header --------------------------------------- - - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - table.head_write() - -# ------------------------------------------ process data ------------------------------------------ - - outputAlive = True - while outputAlive and table.data_read(): # read next data line of ASCII table - outputAlive = table.data_write() # output processed line - -# ------------------------------------------ output finalization ----------------------------------- - - table.close() # close ASCII tables + table.to_ASCII(sys.stdout if name is None else name) diff --git a/python/damask/table.py b/python/damask/table.py index 171d15084..7d413812d 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -161,6 +161,40 @@ class Table(): columns=[label for l in range(size)]) self.data = pd.concat([self.data,new_data],axis=1) + def delete(self,label): + """ + Delete column data. + + Parameters + ---------- + label : str + Column label. + + """ + self.data.drop(columns=label,inplace=True) + + del self.shapes[label] + + def rename(self,label_old,label_new,info=None): + """ + Rename column data. + + Parameters + ---------- + label_old : str + Old column label. + label_new : str + New column label. + + """ + self.data.rename(columns={label_old:label_new},inplace=True) + + comments = '{} => {}'.format(label_old,label_new) + comments += ': {}'.format(info) if info is not None else '' + self.comments.append(comments) + + self.shapes[label_new] = self.shapes.pop(label_old) + def to_ASCII(self,fname): """ diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index 2fedfdc59..b628abe9f 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -57,6 +57,22 @@ class TestTable: default.add('nine',d,'random data') assert np.allclose(d,default.get('nine')) + def test_rename_equivalent(self,default): + v = default.get('v') + default.rename('v','u') + u = default.get('u') + assert np.all(v == u) + + def test_rename_gone(self,default): + default.rename('v','V') + with pytest.raises(KeyError): + default.get('v') + + def test_delete(self,default): + default.delete('v') + with pytest.raises(KeyError): + default.get('v') + def test_invalid_initialization(self,default): x = default.get('v') From 3af696995ddcf65d6a8e8f44df248dc7beab3fa5 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 08:35:34 +0100 Subject: [PATCH 08/12] did not work when piping --- processing/post/addInfo.py | 2 ++ processing/post/reLabel.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/processing/post/addInfo.py b/processing/post/addInfo.py index f7a54c3e2..5e32510db 100755 --- a/processing/post/addInfo.py +++ b/processing/post/addInfo.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import os +import sys +from io import StringIO from optparse import OptionParser import damask diff --git a/processing/post/reLabel.py b/processing/post/reLabel.py index 4bd077586..85d16acea 100755 --- a/processing/post/reLabel.py +++ b/processing/post/reLabel.py @@ -2,8 +2,8 @@ import os import sys +from io import StringIO from optparse import OptionParser -import re import damask From 1bf4946e677858cceac2d7ac2f3032195e4dea5e Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 08:37:39 +0100 Subject: [PATCH 09/12] invalid escape sequence --- python/damask/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/damask/table.py b/python/damask/table.py index 7d413812d..b2be3ae20 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -82,7 +82,7 @@ class Table(): else: shapes[label]=(1,) - data = pd.read_csv(f,names=[i for i in range(len(labels))],sep='\s+').to_numpy() + data = pd.read_csv(f,names=[i for i in range(len(labels))],sep=r'\s+').to_numpy() return Table(data,shapes,comments) From f885748a589239a9e9d64607202423d07c29e315 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 08:43:14 +0100 Subject: [PATCH 10/12] variable was never used --- python/tests/test_Table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index b628abe9f..67bcab557 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -42,7 +42,7 @@ class TestTable: @pytest.mark.parametrize('fname',['datatype-mix.txt','whitespace-mix.txt']) def test_read_strange(self,reference_dir,fname): with open(os.path.join(reference_dir,fname)) as f: - new = Table.from_ASCII(f) + Table.from_ASCII(f) def test_set(self,default): default.set('F',np.zeros((5,3,3)),'set to zero') From 53a38e3a97a2b61d56c857fe3d3970f34f405745 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 10:37:13 +0100 Subject: [PATCH 11/12] polishing --- processing/post/scaleData.py | 6 +++--- processing/post/shiftData.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/processing/post/scaleData.py b/processing/post/scaleData.py index 60ffa41fa..4484c5300 100755 --- a/processing/post/scaleData.py +++ b/processing/post/scaleData.py @@ -22,7 +22,7 @@ Uniformly scale column values by given factor. """, version = scriptID) parser.add_option('-l','--label', - dest = 'label', + dest = 'labels', action = 'extend', metavar = '', help ='column(s) to scale') parser.add_option('-f','--factor', @@ -31,7 +31,7 @@ parser.add_option('-f','--factor', help = 'factor(s) per column') parser.set_defaults(label = [], - ) + factor = []) (options,filenames) = parser.parse_args() if filenames == []: filenames = [None] @@ -43,7 +43,7 @@ for name in filenames: damask.util.report(scriptName,name) table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) - for i,label in enumerate(options.label): + for i,label in enumerate(options.labels): table.set(label, table.get(label)*float(options.factor[i]), scriptID+' '+' '.join(sys.argv[1:])) diff --git a/processing/post/shiftData.py b/processing/post/shiftData.py index 37c72919e..fb2ba48f9 100755 --- a/processing/post/shiftData.py +++ b/processing/post/shiftData.py @@ -22,7 +22,7 @@ Uniformly shift column values by given offset. """, version = scriptID) parser.add_option('-l','--label', - dest = 'label', + dest = 'labels', action = 'extend', metavar = '', help ='column(s) to shift') parser.add_option('-o','--offset', @@ -31,7 +31,7 @@ parser.add_option('-o','--offset', help = 'offset(s) per column') parser.set_defaults(label = [], - ) + offset = []) (options,filenames) = parser.parse_args() if filenames == []: filenames = [None] @@ -43,7 +43,7 @@ for name in filenames: damask.util.report(scriptName,name) table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) - for i,label in enumerate(options.label): + for i,label in enumerate(options.labels): table.set(label, table.get(label)+float(options.offset[i]), scriptID+' '+' '.join(sys.argv[1:])) From 1018b892960692b26f30655c652822309a01a82e Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 5 Dec 2019 10:47:36 +0100 Subject: [PATCH 12/12] nice to have.. --- processing/post/sortTable.py | 45 +++++++----------------------------- python/damask/table.py | 29 ++++++++++++++++++++++- python/tests/test_Table.py | 38 +++++++++++++++++++++++++++--- 3 files changed, 71 insertions(+), 41 deletions(-) diff --git a/processing/post/sortTable.py b/processing/post/sortTable.py index 53a357226..3a3738d18 100755 --- a/processing/post/sortTable.py +++ b/processing/post/sortTable.py @@ -2,10 +2,9 @@ import os import sys +from io import StringIO from optparse import OptionParser -import numpy as np - import damask @@ -26,7 +25,7 @@ With coordinates in columns "x", "y", and "z"; sorting with x slowest and z fast parser.add_option('-l','--label', - dest = 'keys', + dest = 'labels', action = 'extend', metavar = '', help = 'list of column labels (a,b,c,...)') parser.add_option('-r','--reverse', @@ -38,42 +37,14 @@ parser.set_defaults(reverse = False, ) (options,filenames) = parser.parse_args() - - -# --- loop over input files ------------------------------------------------------------------------- - if filenames == []: filenames = [None] +if options.labels is None: + parser.error('no labels specified.') for name in filenames: - try: table = damask.ASCIItable(name = name, - buffered = False) - except: continue - damask.util.report(scriptName,name) + damask.util.report(scriptName,name) -# ------------------------------------------ assemble header --------------------------------------- + table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name) + table.sort_by(options.labels,not options.reverse) - table.head_read() - table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:])) - table.head_write() - -# ------------------------------------------ process data --------------------------------------- - - table.data_readArray() - - keys = table.labels(raw = True)[::-1] if options.keys is None else options.keys[::-1] # numpy sorts with most significant column as last - - cols = [] - remarks = [] - for i,column in enumerate(table.label_index(keys)): - if column < 0: remarks.append('label "{}" not present...'.format(keys[i])) - else: cols += [table.data[:,column]] - if remarks != []: damask.util.croak(remarks) - - ind = np.lexsort(cols) if cols != [] else np.arange(table.data.shape[0]) - if options.reverse: ind = ind[::-1] - -# ------------------------------------------ output result --------------------------------------- - - table.data = table.data[ind] - table.data_writeArray() - table.close() # close ASCII table + table.to_ASCII(sys.stdout if name is None else name) diff --git a/python/damask/table.py b/python/damask/table.py index b2be3ae20..3c867fcf5 100644 --- a/python/damask/table.py +++ b/python/damask/table.py @@ -1,3 +1,4 @@ +import random import re import pandas as pd @@ -104,7 +105,7 @@ class Table(): """ if re.match(r'[0-9]*?_',label): idx,key = label.split('_',1) - return self.data[key].to_numpy()[:,int(idx)-1] + return self.data[key].to_numpy()[:,int(idx)-1].reshape((-1,1)) else: return self.data[label].to_numpy().reshape((-1,)+self.shapes[label]) @@ -196,6 +197,32 @@ class Table(): self.shapes[label_new] = self.shapes.pop(label_old) + def sort_by(self,labels,ascending=True): + """ + Get column data. + + Parameters + ---------- + label : list of str or str + Column labels. + ascending : bool, optional + Set sort order. + + """ + _temp = [] + _labels = [] + for label in labels if isinstance(labels,list) else [labels]: + if re.match(r'[0-9]*?_',label): + _temp.append(str(random.getrandbits(128))) + self.add(_temp[-1],self.get(label)) + _labels.append(_temp[-1]) + else: + _labels.append(label) + + self.data.sort_values(_labels,axis=0,inplace=True,ascending=ascending) + for t in _temp: self.delete(t) + self.comments.append('sorted by [{}]'.format(', '.join(labels))) + def to_ASCII(self,fname): """ Store as plain text file. diff --git a/python/tests/test_Table.py b/python/tests/test_Table.py index 67bcab557..b17348881 100644 --- a/python/tests/test_Table.py +++ b/python/tests/test_Table.py @@ -19,13 +19,21 @@ def reference_dir(reference_dir_base): class TestTable: - def test_get_tensor(self,default): - d = default.get('F') - assert np.allclose(d,1.0) and d.shape[1:] == (3,3) + def test_get_scalar(self,default): + d = default.get('s') + assert np.allclose(d,1.0) and d.shape[1:] == (1,) def test_get_vector(self,default): d = default.get('v') assert np.allclose(d,1.0) and d.shape[1:] == (3,) + + def test_get_tensor(self,default): + d = default.get('F') + assert np.allclose(d,1.0) and d.shape[1:] == (3,3) + + def test_get_component(self,default): + d = default.get('5_F') + assert np.allclose(d,1.0) and d.shape[1:] == (1,) def test_write_read_str(self,default,tmpdir): default.to_ASCII(str(tmpdir.join('default.txt'))) @@ -87,3 +95,27 @@ class TestTable: def test_invalid_get(self,default): with pytest.raises(KeyError): default.get('n') + + def test_sort_scalar(self): + x = np.random.random((5,13)) + t = Table(x,{'F':(3,3),'v':(3,),'s':(1,)},['random test data']) + unsort = t.get('s') + t.sort_by('s') + sort = t.get('s') + assert np.all(np.sort(unsort,0)==sort) + + def test_sort_component(self): + x = np.random.random((5,12)) + t = Table(x,{'F':(3,3),'v':(3,)},['random test data']) + unsort = t.get('4_F') + t.sort_by('4_F') + sort = t.get('4_F') + assert np.all(np.sort(unsort,0)==sort) + + def test_sort_revert(self): + x = np.random.random((5,12)) + t = Table(x,{'F':(3,3),'v':(3,)},['random test data']) + t.sort_by('4_F',False) + sort = t.get('4_F') + assert np.all(np.sort(sort,0)==sort[::-1,:]) +