From fb286af354ab03dbe1ad53585de3d22266a2dc51 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Thu, 31 Oct 2019 10:45:34 +0100
Subject: [PATCH 01/17] [skip sc] first draft

---
 processing/post/addCauchy.py | 57 +++-----------------------
 python/damask/__init__.py    |  1 +
 python/damask/table.py       | 77 ++++++++++++++++++++++++++++++++++++
 3 files changed, 83 insertions(+), 52 deletions(-)
 create mode 100644 python/damask/table.py

diff --git a/processing/post/addCauchy.py b/processing/post/addCauchy.py
index 18c4ec215..9037567f8 100755
--- a/processing/post/addCauchy.py
+++ b/processing/post/addCauchy.py
@@ -1,11 +1,8 @@
 #!/usr/bin/env python3
 
 import os
-import sys
 from optparse import OptionParser
 
-import numpy as np
-
 import damask
 
 
@@ -37,53 +34,9 @@ parser.set_defaults(defgrad = 'f',
 
 (options,filenames) = parser.parse_args()
 
-# --- loop over input files -------------------------------------------------------------------------
-
-if filenames == []: filenames = [None]
-
 for name in filenames:
-  try:
-    table = damask.ASCIItable(name = name, buffered = False)
-  except:
-    continue
-  damask.util.report(scriptName,name)
-
-# ------------------------------------------ read header ------------------------------------------
-
-  table.head_read()
-
-# ------------------------------------------ sanity checks ----------------------------------------
-
-  errors = []
-  column = {}
-  
-  for tensor in [options.defgrad,options.stress]:
-    dim = table.label_dimension(tensor)
-    if   dim <  0: errors.append('column {} not found.'.format(tensor))
-    elif dim != 9: errors.append('column {} is not a tensor.'.format(tensor))
-    else:
-      column[tensor] = table.label_index(tensor)
-
-  if errors != []:
-    damask.util.croak(errors)
-    table.close(dismiss = True)
-    continue
-
-# ------------------------------------------ assemble header --------------------------------------
-
-  table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))
-  table.labels_append(['{}_Cauchy'.format(i+1) for i in range(9)])                                  # extend ASCII header with new labels
-  table.head_write()
-
-# ------------------------------------------ process data ------------------------------------------
-
-  outputAlive = True
-  while outputAlive and table.data_read():                                                          # read next data line of ASCII table
-    F = np.array(list(map(float,table.data[column[options.defgrad]:column[options.defgrad]+9])),'d').reshape(3,3)
-    P = np.array(list(map(float,table.data[column[options.stress ]:column[options.stress ]+9])),'d').reshape(3,3)
-    table.data_append(list(1.0/np.linalg.det(F)*np.dot(P,F.T).reshape(9)))                          # [Cauchy] = (1/det(F)) * [P].[F_transpose]
-    outputAlive = table.data_write()                                                                # output processed line
-
-# ------------------------------------------ output finalization -----------------------------------
-
-  table.close()                                                                                     # close input ASCII table (works for stdin)
+    table = damask.Table(name)
+    table.add_array('Cauchy',damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3),
+                                                     table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9),
+                    scriptID)
+    table.to_ASCII()
diff --git a/python/damask/__init__.py b/python/damask/__init__.py
index f876d1417..da699be29 100644
--- a/python/damask/__init__.py
+++ b/python/damask/__init__.py
@@ -9,6 +9,7 @@ name = 'damask'
 # classes
 from .environment import Environment      # noqa
 from .asciitable  import ASCIItable       # noqa
+from .table       import Table            # noqa
     
 from .config      import Material         # noqa
 from .colormaps   import Colormap, Color  # noqa
diff --git a/python/damask/table.py b/python/damask/table.py
new file mode 100644
index 000000000..81901c252
--- /dev/null
+++ b/python/damask/table.py
@@ -0,0 +1,77 @@
+import re
+
+import pandas as pd
+import numpy as np
+
+class Table():
+    """Read and write to ASCII tables"""
+    
+    def __init__(self,name):
+        self.name = name
+        with open(self.name) as f:
+            header,keyword = f.readline().split()
+            if keyword == 'header':
+                header = int(header)
+            else:
+                raise Exception
+            self.comments = [f.readline()[:-1] for i in range(header-1)]
+            labels_raw    = f.readline().split()
+            self.data     = pd.read_csv(f,delim_whitespace=True,header=None)
+            
+            labels_repeated = [l.split('_',1)[1] if '_' in l else l for l in labels_raw]
+            self.data.rename(columns=dict(zip([l for l in self.data.columns],labels_repeated)),inplace=True)
+            
+            self.shape = {}
+            for l in labels_raw:
+                tensor_column = re.search(':.*?_',l)
+                if tensor_column:
+                    my_shape = tensor_column.group()[1:-1].split('x')
+                    self.shape[l.split('_',1)[1]] = tuple([int(d) for d in my_shape])
+                else:
+                    vector_column = re.match('.*?_',l)
+                    if vector_column:
+                        self.shape[l.split('_',1)[1]] = (int(l.split('_',1)[0]),)
+                    else:
+                        self.shape[l]=(1,)
+            
+            self.labels = list(dict.fromkeys(labels_repeated))
+
+
+    def get_array(self,label):
+        return self.data[label].to_numpy().reshape((-1,)+self.shape[label])
+
+
+    def add_array(self,label,array,info):
+        if np.product(array.shape[1:],dtype=int) == 1: 
+            self.comments.append('{}: {}'.format(label,info))
+
+        else:
+            self.comments.append('{} {}: {}'.format(label,array.shape[1:],info))
+        
+        self.shape[label] = array.shape[1:]
+        self.labels.append(label)
+        size = np.product(array.shape[1:]) 
+        new_data = pd.DataFrame(data=array.reshape(-1,size),
+                                columns=[label for l in range(size)])
+        self.data = pd.concat([self.data,new_data],axis=1)
+        
+
+    def to_ASCII(self,name=None):
+        labels = []
+        for l in self.labels:
+            if(self.shape[l] == (1,)):
+                labels.append('{}'.format(l))
+            elif(len(self.shape[l]) == 1):
+                labels+=['{}_{}'.format(i+1,l)\
+                         for i in range(self.shape[l][0])]
+            else:
+                labels+=['{}:{}_{}'.format(i+1,'x'.join([str(d) for d in self.shape[l]]),l)\
+                               for i in range(np.product(self.shape[l]))]
+
+        header = ['{} header'.format(len(self.comments)+1)]\
+               + self.comments\
+               + [' '.join(labels)]
+
+        with open(name if name is not None else self.name,'w') as f:
+            for line in header: f.write(line+'\n')
+            self.data.to_csv(f,sep=' ',index=False,header=False)

From 81c739192a499ea40b60f18e355b3e9922bab4d0 Mon Sep 17 00:00:00 2001
From: Test User <damask_user@mpie.de>
Date: Tue, 26 Nov 2019 02:14:29 +0100
Subject: [PATCH 02/17] [skip ci] updated version information after successful
 test of v2.0.3-1097-ga7fca4df

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index a5ac7d281..1a1650364 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-v2.0.3-1073-g6f3cb071
+v2.0.3-1097-ga7fca4df

From 845cfc34ecc09fdb4e78c2a9c6378157b9cf0fe9 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Tue, 26 Nov 2019 14:26:25 +0100
Subject: [PATCH 03/17] similar logic as in geom class

- filename is not part of the object
- transparent handling of files, strings, and path-like objects for file
IO
---
 processing/post/addCauchy.py |  2 +-
 python/damask/table.py       | 70 +++++++++++++++++++-----------------
 2 files changed, 39 insertions(+), 33 deletions(-)

diff --git a/processing/post/addCauchy.py b/processing/post/addCauchy.py
index 9037567f8..3a0e14da7 100755
--- a/processing/post/addCauchy.py
+++ b/processing/post/addCauchy.py
@@ -39,4 +39,4 @@ for name in filenames:
     table.add_array('Cauchy',damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3),
                                                      table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9),
                     scriptID)
-    table.to_ASCII()
+    table.to_ASCII(name)
diff --git a/python/damask/table.py b/python/damask/table.py
index 81901c252..e77ae2b9c 100644
--- a/python/damask/table.py
+++ b/python/damask/table.py
@@ -5,36 +5,39 @@ import numpy as np
 
 class Table():
     """Read and write to ASCII tables"""
-    
-    def __init__(self,name):
-        self.name = name
-        with open(self.name) as f:
-            header,keyword = f.readline().split()
-            if keyword == 'header':
-                header = int(header)
+   
+    def __init__(self,fname):
+        try:
+            f = open(fname)
+        except TypeError:
+            f = fname
+
+        header,keyword = f.readline().split()
+        if keyword == 'header':
+            header = int(header)
+        else:
+            raise Exception
+        self.comments = [f.readline()[:-1] for i in range(header-1)]
+        labels_raw    = f.readline().split()
+        self.data     = pd.read_csv(f,delim_whitespace=True,header=None)
+        
+        labels_repeated = [l.split('_',1)[1] if '_' in l else l for l in labels_raw]
+        self.data.rename(columns=dict(zip([l for l in self.data.columns],labels_repeated)),inplace=True)
+        
+        self.shape = {}
+        for l in labels_raw:
+            tensor_column = re.search(':.*?_',l)
+            if tensor_column:
+                my_shape = tensor_column.group()[1:-1].split('x')
+                self.shape[l.split('_',1)[1]] = tuple([int(d) for d in my_shape])
             else:
-                raise Exception
-            self.comments = [f.readline()[:-1] for i in range(header-1)]
-            labels_raw    = f.readline().split()
-            self.data     = pd.read_csv(f,delim_whitespace=True,header=None)
-            
-            labels_repeated = [l.split('_',1)[1] if '_' in l else l for l in labels_raw]
-            self.data.rename(columns=dict(zip([l for l in self.data.columns],labels_repeated)),inplace=True)
-            
-            self.shape = {}
-            for l in labels_raw:
-                tensor_column = re.search(':.*?_',l)
-                if tensor_column:
-                    my_shape = tensor_column.group()[1:-1].split('x')
-                    self.shape[l.split('_',1)[1]] = tuple([int(d) for d in my_shape])
+                vector_column = re.match('.*?_',l)
+                if vector_column:
+                    self.shape[l.split('_',1)[1]] = (int(l.split('_',1)[0]),)
                 else:
-                    vector_column = re.match('.*?_',l)
-                    if vector_column:
-                        self.shape[l.split('_',1)[1]] = (int(l.split('_',1)[0]),)
-                    else:
-                        self.shape[l]=(1,)
-            
-            self.labels = list(dict.fromkeys(labels_repeated))
+                    self.shape[l]=(1,)
+        
+        self.labels = list(dict.fromkeys(labels_repeated))
 
 
     def get_array(self,label):
@@ -56,7 +59,7 @@ class Table():
         self.data = pd.concat([self.data,new_data],axis=1)
         
 
-    def to_ASCII(self,name=None):
+    def to_ASCII(self,fname):
         labels = []
         for l in self.labels:
             if(self.shape[l] == (1,)):
@@ -72,6 +75,9 @@ class Table():
                + self.comments\
                + [' '.join(labels)]
 
-        with open(name if name is not None else self.name,'w') as f:
-            for line in header: f.write(line+'\n')
-            self.data.to_csv(f,sep=' ',index=False,header=False)
+        try:
+            f = open(fname,'w')
+        except TypeError:
+            f = fname
+        for line in header: f.write(line+'\n')
+        self.data.to_csv(f,sep=' ',index=False,header=False)

From 925a4f73d649eee0276e102e0c95da2c119d18fa Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Tue, 26 Nov 2019 20:32:54 +0100
Subject: [PATCH 04/17] staticmethod better suited than class method

a classmethod changes the class, i.e. it assigns attributes and gives
them specific values.
a staticmethod does not alter the class

https://www.geeksforgeeks.org/class-method-vs-static-method-python
---
 python/damask/geom.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/damask/geom.py b/python/damask/geom.py
index 1c9e10cd1..32ea2ed89 100644
--- a/python/damask/geom.py
+++ b/python/damask/geom.py
@@ -239,8 +239,8 @@ class Geom():
         header.append('homogenization {}'.format(self.get_homogenization()))
         return header
         
-    @classmethod
-    def from_file(cls,fname):
+    @staticmethod
+    def from_file(fname):
         """
         Reads a geom file.
 
@@ -300,7 +300,7 @@ class Geom():
         if not np.any(np.mod(microstructure.flatten(),1) != 0.0):                                   # no float present
             microstructure = microstructure.astype('int')
         
-        return cls(microstructure.reshape(grid),size,origin,homogenization,comments)
+        return Geom(microstructure.reshape(grid),size,origin,homogenization,comments)
 
 
     def to_file(self,fname,pack=None):

From 5661f60552e25ffe19ce2bb505e9b2a8404bad76 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Tue, 26 Nov 2019 21:36:24 +0100
Subject: [PATCH 05/17] fname seems to be the common name

---
 python/damask/dadf5.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/python/damask/dadf5.py b/python/damask/dadf5.py
index d879946eb..5ecc8e619 100644
--- a/python/damask/dadf5.py
+++ b/python/damask/dadf5.py
@@ -18,17 +18,17 @@ class DADF5():
   """
   
 # ------------------------------------------------------------------
-  def __init__(self,filename):
+  def __init__(self,fname):
     """
     Opens an existing DADF5 file.
     
     Parameters
     ----------
-    filename : str
+    fname : str
         name of the DADF5 file to be openend.
     
     """
-    with h5py.File(filename,'r') as f:
+    with h5py.File(fname,'r') as f:
       
       if f.attrs['DADF5-major'] != 0 or not 2 <= f.attrs['DADF5-minor'] <= 3:
         raise TypeError('Unsupported DADF5 version {} '.format(f.attrs['DADF5-version']))
@@ -64,7 +64,7 @@ class DADF5():
                    'con_physics':    self.con_physics,
                    'mat_physics':    self.mat_physics}
                   
-    self.filename   = filename
+    self.fname = fname
 
 
   def __manage_visible(self,datasets,what,action):
@@ -298,7 +298,7 @@ class DADF5():
 
     groups = []
     
-    with h5py.File(self.filename,'r') as f:
+    with h5py.File(self.fname,'r') as f:
       for i in self.iter_visible('increments'):
         for o,p in zip(['constituents','materialpoints'],['con_physics','mat_physics']):
           for oo in self.iter_visible(o):
@@ -315,7 +315,7 @@ class DADF5():
   def list_data(self):
     """Return information on all active datasets in the file."""
     message = ''
-    with h5py.File(self.filename,'r') as f:
+    with h5py.File(self.fname,'r') as f:
       for s,i in enumerate(self.iter_visible('increments')):
         message+='\n{} ({}s)\n'.format(i,self.times[s])
         for o,p in zip(['constituents','materialpoints'],['con_physics','mat_physics']):
@@ -336,7 +336,7 @@ class DADF5():
   def get_dataset_location(self,label):
     """Return the location of all active datasets with given label."""
     path = []
-    with h5py.File(self.filename,'r') as f:
+    with h5py.File(self.fname,'r') as f:
       for i in self.iter_visible('increments'):
         k = '/'.join([i,'geometry',label])
         try:
@@ -358,14 +358,14 @@ class DADF5():
     
   def get_constituent_ID(self,c=0):
     """Pointwise constituent ID."""
-    with h5py.File(self.filename,'r') as f:
+    with h5py.File(self.fname,'r') as f:
       names = f['/mapping/cellResults/constituent']['Name'][:,c].astype('str')
     return np.array([int(n.split('_')[0]) for n in names.tolist()],dtype=np.int32)
 
 
   def get_crystal_structure(self):                                                                  # ToDo: extension to multi constituents/phase
     """Info about the crystal structure."""
-    with h5py.File(self.filename,'r') as f:
+    with h5py.File(self.fname,'r') as f:
       return f[self.get_dataset_location('orientation')[0]].attrs['Lattice'].astype('str')          # np.bytes_ to string
 
 
@@ -375,7 +375,7 @@ class DADF5():
     
     If more than one path is given, the dataset is composed of the individual contributions.
     """
-    with h5py.File(self.filename,'r') as f:
+    with h5py.File(self.fname,'r') as f:
       shape = (self.Nmaterialpoints,) + np.shape(f[path[0]])[1:]
       if len(shape) == 1: shape = shape +(1,)
       dataset = np.full(shape,np.nan,dtype=np.dtype(f[path[0]]))
@@ -418,7 +418,7 @@ class DADF5():
                            )
       return np.concatenate((x[:,:,:,None],y[:,:,:,None],y[:,:,:,None]),axis = 3).reshape([np.product(self.grid),3])
     else:
-      with h5py.File(self.filename,'r') as f:
+      with h5py.File(self.fname,'r') as f:
         return f['geometry/x_c'][()]
 
 
@@ -798,7 +798,7 @@ class DADF5():
     todo = []
     # ToDo: It would be more memory efficient to read only from file when required, i.e. do to it in pool.add_task
     for group in self.groups_with_datasets([d['label'] for d in datasets_requested]):
-      with h5py.File(self.filename,'r') as f:
+      with h5py.File(self.fname,'r') as f:
         datasets_in = {}
         for d in datasets_requested:
           loc  = f[group+'/'+d['label']]
@@ -813,7 +813,7 @@ class DADF5():
     N_not_calculated = len(todo)
     while N_not_calculated > 0:    
       result = results.get()
-      with h5py.File(self.filename,'a') as f:                                                       # write to file
+      with h5py.File(self.fname,'a') as f:                                                       # write to file
         dataset_out = f[result['group']].create_dataset(result['label'],data=result['data'])
         for k in result['meta'].keys():
           dataset_out.attrs[k] = result['meta'][k].encode()

From 2d96136a0d9fb305a0bd43ce11e00077c9d900d1 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Tue, 26 Nov 2019 22:53:46 +0100
Subject: [PATCH 06/17] more general constructor for Table

reading from file is just one case (now handled by static method).
General constructor needs data and header information as dictionary.
Works only with python 3.7 where dict keeps the insertion order. Earlier
python versions/other implementations might fail.
---
 processing/post/addCauchy.py | 15 +++++--
 python/damask/table.py       | 82 ++++++++++++++++++++++++------------
 2 files changed, 66 insertions(+), 31 deletions(-)

diff --git a/processing/post/addCauchy.py b/processing/post/addCauchy.py
index 3a0e14da7..788f1b580 100755
--- a/processing/post/addCauchy.py
+++ b/processing/post/addCauchy.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 
 import os
+import sys
+from io import StringIO
 from optparse import OptionParser
 
 import damask
@@ -34,9 +36,14 @@ parser.set_defaults(defgrad = 'f',
 
 (options,filenames) = parser.parse_args()
 
+if filenames == []: filenames = [None]
+
 for name in filenames:
-    table = damask.Table(name)
-    table.add_array('Cauchy',damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3),
-                                                     table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9),
+    damask.util.report(scriptName,name)
+
+    table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name)
+    table.add_array('Cauchy',
+                    damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3),
+                                            table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9),
                     scriptID)
-    table.to_ASCII(name)
+    table.to_ASCII(sys.stdout if name is None else name)
diff --git a/python/damask/table.py b/python/damask/table.py
index e77ae2b9c..6c5103bc9 100644
--- a/python/damask/table.py
+++ b/python/damask/table.py
@@ -4,9 +4,42 @@ import pandas as pd
 import numpy as np
 
 class Table():
-    """Read and write to ASCII tables"""
-   
-    def __init__(self,fname):
+    """Store spreadsheet-like data."""
+  
+    def __init__(self,array,headings,comments=None):
+        """
+        New spreadsheet data.
+        
+        Parameters
+        ----------
+        array : numpy.ndarray
+            Data.
+        headings : dict
+            Column headings. Labels as keys and shape as tuple. Example 'F':(3,3) for a deformation gradient. 
+        comments : iterable of str, optional
+            Additional, human-readable information
+        
+        """
+        self.data = pd.DataFrame(data=array)
+
+        d = {}
+        i = 0
+        for label in headings:
+            for components in range(np.prod(headings[label])):
+                d[i] = label
+                i+=1
+
+        self.data.rename(columns=d,inplace=True)
+
+        if comments is None:
+            self.comments = []
+        else:
+            self.comments = [c for c in comments]
+
+        self.headings  = headings
+
+    @staticmethod
+    def from_ASCII(fname):
         try:
             f = open(fname)
         except TypeError:
@@ -17,43 +50,38 @@ class Table():
             header = int(header)
         else:
             raise Exception
-        self.comments = [f.readline()[:-1] for i in range(header-1)]
-        labels_raw    = f.readline().split()
-        self.data     = pd.read_csv(f,delim_whitespace=True,header=None)
-        
-        labels_repeated = [l.split('_',1)[1] if '_' in l else l for l in labels_raw]
-        self.data.rename(columns=dict(zip([l for l in self.data.columns],labels_repeated)),inplace=True)
-        
-        self.shape = {}
+        comments   = [f.readline()[:-1] for i in range(header-1)]
+        labels_raw = f.readline().split()
+        labels     = [l.split('_',1)[1] if '_' in l else l for l in labels_raw]
+       
+        headings = {}
         for l in labels_raw:
             tensor_column = re.search(':.*?_',l)
             if tensor_column:
                 my_shape = tensor_column.group()[1:-1].split('x')
-                self.shape[l.split('_',1)[1]] = tuple([int(d) for d in my_shape])
+                headings[l.split('_',1)[1]] = tuple([int(d) for d in my_shape])
             else:
                 vector_column = re.match('.*?_',l)
                 if vector_column:
-                    self.shape[l.split('_',1)[1]] = (int(l.split('_',1)[0]),)
+                    headings[l.split('_',1)[1]] = (int(l.split('_',1)[0]),)
                 else:
-                    self.shape[l]=(1,)
+                    headings[l]=(1,)
         
-        self.labels = list(dict.fromkeys(labels_repeated))
+        return Table(np.loadtxt(f),headings,comments)
 
 
     def get_array(self,label):
-        return self.data[label].to_numpy().reshape((-1,)+self.shape[label])
+        return self.data[label].to_numpy().reshape((-1,)+self.headings[label])
 
 
     def add_array(self,label,array,info):
-        if np.product(array.shape[1:],dtype=int) == 1: 
+        if np.prod(array.shape[1:],dtype=int) == 1: 
             self.comments.append('{}: {}'.format(label,info))
-
         else:
             self.comments.append('{} {}: {}'.format(label,array.shape[1:],info))
         
-        self.shape[label] = array.shape[1:]
-        self.labels.append(label)
-        size = np.product(array.shape[1:]) 
+        self.headings[label] = array.shape[1:] if len(array.shape) > 1 else (1,)
+        size = np.prod(array.shape[1:],dtype=int) 
         new_data = pd.DataFrame(data=array.reshape(-1,size),
                                 columns=[label for l in range(size)])
         self.data = pd.concat([self.data,new_data],axis=1)
@@ -61,15 +89,15 @@ class Table():
 
     def to_ASCII(self,fname):
         labels = []
-        for l in self.labels:
-            if(self.shape[l] == (1,)):
+        for l in self.headings:
+            if(self.headings[l] == (1,)):
                 labels.append('{}'.format(l))
-            elif(len(self.shape[l]) == 1):
+            elif(len(self.headings[l]) == 1):
                 labels+=['{}_{}'.format(i+1,l)\
-                         for i in range(self.shape[l][0])]
+                         for i in range(self.headings[l][0])]
             else:
-                labels+=['{}:{}_{}'.format(i+1,'x'.join([str(d) for d in self.shape[l]]),l)\
-                               for i in range(np.product(self.shape[l]))]
+                labels+=['{}:{}_{}'.format(i+1,'x'.join([str(d) for d in self.headings[l]]),l)\
+                               for i in range(np.prod(self.headings[l],dtype=int))]
 
         header = ['{} header'.format(len(self.comments)+1)]\
                + self.comments\

From 31d3958ca6d43b63270c6755def366fbe4824d02 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Tue, 26 Nov 2019 22:55:38 +0100
Subject: [PATCH 07/17] using fast new Table class

more a proof-of-concept since shell scripts are deprecated. The detailed
error handling of the former scripts is not implemented, i.e. the user
need to ensure that the files exist and the data has the correct shape
---
 processing/post/addDeterminant.py | 66 ++++----------------------
 processing/post/addDeviator.py    | 79 +++++--------------------------
 2 files changed, 22 insertions(+), 123 deletions(-)

diff --git a/processing/post/addDeterminant.py b/processing/post/addDeterminant.py
index 14f0321be..090b9224a 100755
--- a/processing/post/addDeterminant.py
+++ b/processing/post/addDeterminant.py
@@ -4,20 +4,13 @@ import os
 import sys
 from optparse import OptionParser
 
-import damask
+import numpy as np
 
+import damask
 
 scriptName = os.path.splitext(os.path.basename(__file__))[0]
 scriptID   = ' '.join([scriptName,damask.version])
 
-def determinant(m):
-  return  +m[0]*m[4]*m[8] \
-          +m[1]*m[5]*m[6] \
-          +m[2]*m[3]*m[7] \
-          -m[2]*m[4]*m[6] \
-          -m[1]*m[3]*m[8] \
-          -m[0]*m[5]*m[7]
-
 
 # --------------------------------------------------------------------
 #                                MAIN
@@ -43,52 +36,11 @@ if options.tensor is None:
 if filenames == []: filenames = [None]
 
 for name in filenames:
-  try:
-    table = damask.ASCIItable(name = name,
-                              buffered = False)
-  except: continue
-  damask.util.report(scriptName,name)
+    damask.util.report(scriptName,name)
 
-# ------------------------------------------ read header ------------------------------------------
-
-  table.head_read()
-
-# ------------------------------------------ sanity checks ----------------------------------------
-
-  items = {
-            'tensor': {'dim': 9, 'shape': [3,3], 'labels':options.tensor, 'column': []},
-          }
-  errors  = []
-  remarks = []
-  
-  for type, data in items.items():
-    for what in data['labels']:
-      dim = table.label_dimension(what)
-      if dim != data['dim']: remarks.append('column {} is not a {}...'.format(what,type))
-      else:
-        items[type]['column'].append(table.label_index(what))
-        table.labels_append('det({})'.format(what))                                                 # extend ASCII header with new labels
-
-  if remarks != []: damask.util.croak(remarks)
-  if errors  != []:
-    damask.util.croak(errors)
-    table.close(dismiss = True)
-    continue
-
-# ------------------------------------------ assemble header --------------------------------------
-
-  table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))
-  table.head_write()
-
-# ------------------------------------------ process data ------------------------------------------
-
-  outputAlive = True
-  while outputAlive and table.data_read():                                                          # read next data line of ASCII table
-    for type, data in items.items():
-      for column in data['column']:
-        table.data_append(determinant(list(map(float,table.data[column: column+data['dim']]))))
-    outputAlive = table.data_write()                                                                # output processed line
-
-# ------------------------------------------ output finalization -----------------------------------
-
-  table.close()                                                                                     # close input ASCII table (works for stdin)
+    table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name)
+    for tensor in options.tensor:
+         table.add_array('det({})'.format(tensor),
+                         np.linalg.det(table.get_array(tensor).reshape(-1,3,3)),
+                         scriptID)
+    table.to_ASCII(sys.stdout if name is None else name)
diff --git a/processing/post/addDeviator.py b/processing/post/addDeviator.py
index c9aeaacfd..c0bb77e6b 100755
--- a/processing/post/addDeviator.py
+++ b/processing/post/addDeviator.py
@@ -2,6 +2,7 @@
 
 import os
 import sys
+from io import StringIO
 from optparse import OptionParser
 
 import damask
@@ -9,17 +10,6 @@ import damask
 scriptName = os.path.splitext(os.path.basename(__file__))[0]
 scriptID   = ' '.join([scriptName,damask.version])
 
-oneThird = 1.0/3.0
-
-def deviator(m,spherical = False):                                                                  # Careful, do not change the value of m, its intent(inout)!
-  sph = oneThird*(m[0]+m[4]+m[8])
-  dev = [
-           m[0]-sph,  m[1],     m[2],
-           m[3],      m[4]-sph, m[5],
-           m[6],      m[7],     m[8]-sph,
-        ]
-  return dev,sph if spherical else dev
-
 
 # --------------------------------------------------------------------
 #                                MAIN
@@ -49,58 +39,15 @@ if options.tensor is None:
 if filenames == []: filenames = [None]
 
 for name in filenames:
-  try:
-    table = damask.ASCIItable(name = name, buffered = False)
-  except:
-    continue
-  damask.util.report(scriptName,name)
-
-# ------------------------------------------ read header ------------------------------------------
-
-  table.head_read()
-
-# ------------------------------------------ sanity checks ----------------------------------------
-
-  items = {
-            'tensor': {'dim': 9, 'shape': [3,3], 'labels':options.tensor, 'active':[], 'column': []},
-          }
-  errors  = []
-  remarks = []
-  column = {}
-  
-  for type, data in items.items():
-    for what in data['labels']:
-      dim = table.label_dimension(what)
-      if dim != data['dim']: remarks.append('column {} is not a {}.'.format(what,type))
-      else:
-        items[type]['active'].append(what)
-        items[type]['column'].append(table.label_index(what))
-
-  if remarks != []: damask.util.croak(remarks)
-  if errors  != []:
-    damask.util.croak(errors)
-    table.close(dismiss = True)
-    continue
-
-# ------------------------------------------ assemble header --------------------------------------
-
-  table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))
-  for type, data in items.items():
-    for label in data['active']:
-      table.labels_append(['{}_dev({})'.format(i+1,label) for i in range(data['dim'])] + \
-                         (['sph({})'.format(label)] if options.spherical else []))                  # extend ASCII header with new labels
-  table.head_write()
-
-# ------------------------------------------ process data ------------------------------------------
-
-  outputAlive = True
-  while outputAlive and table.data_read():                                                          # read next data line of ASCII table
-    for type, data in items.items():
-      for column in data['column']:
-        table.data_append(deviator(list(map(float,table.data[column:
-                                                        column+data['dim']])),options.spherical))
-    outputAlive = table.data_write()                                                                # output processed line
-
-# ------------------------------------------ output finalization -----------------------------------
-
-  table.close()                                                                                     # close input ASCII table (works for stdin)
+    damask.util.report(scriptName,name)
+    
+    table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name)
+    for tensor in options.tensor:
+         table.add_array('dev({})'.format(tensor),
+                         damask.mechanics.deviatoric_part(table.get_array(tensor).reshape(-1,3,3)).reshape((-1,9)),
+                         scriptID)
+         if options.spherical:
+             table.add_array('sph({})'.format(tensor),
+                             damask.mechanics.spherical_part(table.get_array(tensor).reshape(-1,3,3)),
+                             scriptID)
+    table.to_ASCII(sys.stdout if name is None else name)

From 882a11c5f874b2491c88753c80b8cba69970f2f1 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Wed, 27 Nov 2019 08:43:20 +0100
Subject: [PATCH 08/17] get labels without shape information

---
 python/damask/table.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/damask/table.py b/python/damask/table.py
index 6c5103bc9..7df642d02 100644
--- a/python/damask/table.py
+++ b/python/damask/table.py
@@ -69,10 +69,11 @@ class Table():
         
         return Table(np.loadtxt(f),headings,comments)
 
-
     def get_array(self,label):
         return self.data[label].to_numpy().reshape((-1,)+self.headings[label])
 
+    def get_labels(self):
+        return [label for label in self.headings]
 
     def add_array(self,label,array,info):
         if np.prod(array.shape[1:],dtype=int) == 1: 
@@ -86,7 +87,6 @@ class Table():
                                 columns=[label for l in range(size)])
         self.data = pd.concat([self.data,new_data],axis=1)
         
-
     def to_ASCII(self,fname):
         labels = []
         for l in self.headings:

From eb033e11b29522ccf9ad634ed6bbd52c8452eaf1 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Wed, 27 Nov 2019 08:44:38 +0100
Subject: [PATCH 09/17] polishing

---
 processing/post/addCauchy.py      |  4 ++--
 processing/post/addDeterminant.py |  8 +++-----
 processing/post/addDeviator.py    | 12 +++++-------
 3 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/processing/post/addCauchy.py b/processing/post/addCauchy.py
index 788f1b580..ad29792a6 100755
--- a/processing/post/addCauchy.py
+++ b/processing/post/addCauchy.py
@@ -35,7 +35,6 @@ parser.set_defaults(defgrad = 'f',
                    )
 
 (options,filenames) = parser.parse_args()
-
 if filenames == []: filenames = [None]
 
 for name in filenames:
@@ -45,5 +44,6 @@ for name in filenames:
     table.add_array('Cauchy',
                     damask.mechanics.Cauchy(table.get_array(options.defgrad).reshape(-1,3,3),
                                             table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9),
-                    scriptID)
+                    scriptID+' '+' '.join(sys.argv[1:]))
+
     table.to_ASCII(sys.stdout if name is None else name)
diff --git a/processing/post/addDeterminant.py b/processing/post/addDeterminant.py
index 090b9224a..e38261bf6 100755
--- a/processing/post/addDeterminant.py
+++ b/processing/post/addDeterminant.py
@@ -27,14 +27,11 @@ parser.add_option('-t','--tensor',
                   help = 'heading of columns containing tensor field values')
 
 (options,filenames) = parser.parse_args()
+if filenames == []: filenames = [None]
 
 if options.tensor is None:
   parser.error('no data column specified.')
 
-# --- loop over input files -------------------------------------------------------------------------
-
-if filenames == []: filenames = [None]
-
 for name in filenames:
     damask.util.report(scriptName,name)
 
@@ -42,5 +39,6 @@ for name in filenames:
     for tensor in options.tensor:
          table.add_array('det({})'.format(tensor),
                          np.linalg.det(table.get_array(tensor).reshape(-1,3,3)),
-                         scriptID)
+                         scriptID+' '+' '.join(sys.argv[1:]))
+
     table.to_ASCII(sys.stdout if name is None else name)
diff --git a/processing/post/addDeviator.py b/processing/post/addDeviator.py
index c0bb77e6b..7807c7f5d 100755
--- a/processing/post/addDeviator.py
+++ b/processing/post/addDeviator.py
@@ -30,13 +30,10 @@ parser.add_option('-s','--spherical',
                   help = 'report spherical part of tensor (hydrostatic component, pressure)')
 
 (options,filenames) = parser.parse_args()
+if filenames == []: filenames = [None]
 
 if options.tensor is None:
-  parser.error('no data column specified...')
-
-# --- loop over input files -------------------------------------------------------------------------
-
-if filenames == []: filenames = [None]
+    parser.error('no data column specified...')
 
 for name in filenames:
     damask.util.report(scriptName,name)
@@ -45,9 +42,10 @@ for name in filenames:
     for tensor in options.tensor:
          table.add_array('dev({})'.format(tensor),
                          damask.mechanics.deviatoric_part(table.get_array(tensor).reshape(-1,3,3)).reshape((-1,9)),
-                         scriptID)
+                         scriptID+' '+' '.join(sys.argv[1:]))
          if options.spherical:
              table.add_array('sph({})'.format(tensor),
                              damask.mechanics.spherical_part(table.get_array(tensor).reshape(-1,3,3)),
-                             scriptID)
+                             scriptID+' '+' '.join(sys.argv[1:]))
+
     table.to_ASCII(sys.stdout if name is None else name)

From ab83dc2ebc91a44589f03040775ce11e38fe3952 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Wed, 27 Nov 2019 08:45:01 +0100
Subject: [PATCH 10/17] use central facilities

---
 processing/post/addMises.py | 80 +++++++------------------------------
 1 file changed, 14 insertions(+), 66 deletions(-)

diff --git a/processing/post/addMises.py b/processing/post/addMises.py
index be11b0f1c..9bc560f2d 100755
--- a/processing/post/addMises.py
+++ b/processing/post/addMises.py
@@ -2,10 +2,8 @@
 
 import os
 import sys
+from io import StringIO
 from optparse import OptionParser
-from collections import OrderedDict
-
-import numpy as np
 
 import damask
 
@@ -13,15 +11,6 @@ import damask
 scriptName = os.path.splitext(os.path.basename(__file__))[0]
 scriptID   = ' '.join([scriptName,damask.version])
 
-def Mises(what,tensor):
-
-  dev = tensor - np.trace(tensor)/3.0*np.eye(3)
-  symdev = 0.5*(dev+dev.T)
-  return np.sqrt(np.sum(symdev*symdev.T)*
-        {
-         'stress': 3.0/2.0,
-         'strain': 2.0/3.0,
-         }[what.lower()])
 
 # --------------------------------------------------------------------
 #                                MAIN
@@ -47,62 +36,21 @@ parser.set_defaults(strain = [],
 (options,filenames) = parser.parse_args()
 
 if options.stress is [] and options.strain is []:
-  parser.error('no data column specified...')
-
-# --- loop over input files -------------------------------------------------------------------------
+    parser.error('no data column specified...')
 
 if filenames == []: filenames = [None]
 
 for name in filenames:
-  try:
-    table = damask.ASCIItable(name = name,
-                              buffered = False)
-  except: continue
-  damask.util.report(scriptName,name)
+    damask.util.report(scriptName,name)
 
-# ------------------------------------------ read header ------------------------------------------
-
-  table.head_read()
-
-# ------------------------------------------ sanity checks ----------------------------------------
-
-  items = OrderedDict([
-            ('strain', {'dim': 9, 'shape': [3,3], 'labels':options.strain, 'active':[], 'column': []}),
-            ('stress', {'dim': 9, 'shape': [3,3], 'labels':options.stress, 'active':[], 'column': []})
-          ])
-  errors  = []
-  remarks = []
-  
-  for type, data in items.items():
-    for what in data['labels']:
-      dim = table.label_dimension(what)
-      if dim != data['dim']: remarks.append('column {} is not a {}...'.format(what,type))
-      else:
-        items[type]['active'].append(what)
-        items[type]['column'].append(table.label_index(what))
-        table.labels_append('Mises({})'.format(what))                                               # extend ASCII header with new labels
-
-  if remarks != []: damask.util.croak(remarks)
-  if errors  != []:
-    damask.util.croak(errors)
-    table.close(dismiss = True)
-    continue
-
-# ------------------------------------------ assemble header --------------------------------------
-
-  table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))
-  table.head_write()
-
-# ------------------------------------------ process data ------------------------------------------
-
-  outputAlive = True
-  while outputAlive and table.data_read():                                                          # read next data line of ASCII table
-    for type, data in items.items():
-      for column in data['column']:
-        table.data_append(Mises(type,
-                                np.array(table.data[column:column+data['dim']],'d').reshape(data['shape'])))
-    outputAlive = table.data_write()                                                                # output processed line
-
-# ------------------------------------------ output finalization -----------------------------------
-
-  table.close()                                                                                     # close input ASCII table (works for stdin)
+    table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name)
+    for strain in options.strain:
+        table.add_array('Mises({})'.format(strain),
+                        damask.mechanics.Mises_strain(damask.mechanics.symmetric(table.get_array(strain).reshape(-1,3,3))),
+                        scriptID+' '+' '.join(sys.argv[1:]))
+    for stress in options.stress:
+        table.add_array('Mises({})'.format(stress),
+                        damask.mechanics.Mises_stress(damask.mechanics.symmetric(table.get_array(stress).reshape(-1,3,3))),
+                        scriptID+' '+' '.join(sys.argv[1:]))
+    
+    table.to_ASCII(sys.stdout if name is None else name)

From 9ad74745c15a43fe5cb0c884ac82ca74cde9047c Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Wed, 27 Nov 2019 08:48:41 +0100
Subject: [PATCH 11/17] missing import detected by prospector

---
 processing/post/addDeterminant.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/processing/post/addDeterminant.py b/processing/post/addDeterminant.py
index e38261bf6..4c3f9d260 100755
--- a/processing/post/addDeterminant.py
+++ b/processing/post/addDeterminant.py
@@ -2,6 +2,7 @@
 
 import os
 import sys
+from io import StringIO
 from optparse import OptionParser
 
 import numpy as np

From 96714089b19b3f38e247d06c79d075521c5b7d2e Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Wed, 27 Nov 2019 09:58:58 +0100
Subject: [PATCH 12/17] documented and tested handling of multi-dimensional
 data

more precise regex expressions. get_array can handle individual
components
---
 python/damask/table.py | 34 +++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/python/damask/table.py b/python/damask/table.py
index 7df642d02..bad2a60c3 100644
--- a/python/damask/table.py
+++ b/python/damask/table.py
@@ -40,6 +40,13 @@ class Table():
 
     @staticmethod
     def from_ASCII(fname):
+        """
+        Create table from ASCII file.
+
+        The first line needs to indicate the number of subsequent header lines as 'n header'.
+        Vector data labels are indicated by '1_x, 2_x, ..., n_x'.
+        Tensor data labels are indicated by '3x3:1_x, 3x3:2_x, ..., 3x3:9_x'.
+        """
         try:
             f = open(fname)
         except TypeError:
@@ -50,29 +57,34 @@ class Table():
             header = int(header)
         else:
             raise Exception
-        comments   = [f.readline()[:-1] for i in range(header-1)]
-        labels_raw = f.readline().split()
-        labels     = [l.split('_',1)[1] if '_' in l else l for l in labels_raw]
+        comments = [f.readline()[:-1] for i in range(header-1)]
+        labels   = f.readline().split()
        
         headings = {}
-        for l in labels_raw:
-            tensor_column = re.search(':.*?_',l)
+        for label in labels:
+            tensor_column = re.search(r'[0-9,x]*?:[0-9]*?_',label)
             if tensor_column:
-                my_shape = tensor_column.group()[1:-1].split('x')
-                headings[l.split('_',1)[1]] = tuple([int(d) for d in my_shape])
+                my_shape = tensor_column.group().split(':',1)[0].split('x')
+                headings[label.split('_',1)[1]] = tuple([int(d) for d in my_shape])
             else:
-                vector_column = re.match('.*?_',l)
+                vector_column = re.match(r'[0-9]*?_',label)
                 if vector_column:
-                    headings[l.split('_',1)[1]] = (int(l.split('_',1)[0]),)
+                    headings[label.split('_',1)[1]] = (int(label.split('_',1)[0]),)
                 else:
-                    headings[l]=(1,)
+                    headings[label]=(1,)
         
         return Table(np.loadtxt(f),headings,comments)
 
     def get_array(self,label):
-        return self.data[label].to_numpy().reshape((-1,)+self.headings[label])
+        """Return data as array."""
+        if re.match(r'[0-9]*?_',label):
+            idx,key = label.split('_',1)
+            return self.data[key].to_numpy()[:,int(idx)-1]
+        else: 
+            return self.data[label].to_numpy().reshape((-1,)+self.headings[label])
 
     def get_labels(self):
+        """Return the labels of all columns."""
         return [label for label in self.headings]
 
     def add_array(self,label,array,info):

From 39734ef53cc81438ff69f84cfc87e89ec270fa0c Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Wed, 27 Nov 2019 10:56:29 +0100
Subject: [PATCH 13/17] alter data

allows to access multiple columns ('f') and individual components
('1_f')
---
 python/damask/table.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/python/damask/table.py b/python/damask/table.py
index bad2a60c3..2e426d6fb 100644
--- a/python/damask/table.py
+++ b/python/damask/table.py
@@ -83,6 +83,16 @@ class Table():
         else: 
             return self.data[label].to_numpy().reshape((-1,)+self.headings[label])
 
+    def set_array(self,label,array):
+        """Set data."""
+        if re.match(r'[0-9]*?_',label):
+            idx,key = label.split('_',1)
+            iloc = self.data.columns.get_loc(key).tolist().index(True) + int(idx) -1
+            self.data.iloc[:,iloc] = array
+        else: 
+            self.data[label]       = array
+
+
     def get_labels(self):
         """Return the labels of all columns."""
         return [label for label in self.headings]

From 76c357737854fac8824fe798067fa8807d514ed9 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Wed, 27 Nov 2019 12:03:35 +0100
Subject: [PATCH 14/17] pandas.DataFrame needs to be a 2nd order array

---
 python/damask/table.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/damask/table.py b/python/damask/table.py
index 2e426d6fb..3cd6ed330 100644
--- a/python/damask/table.py
+++ b/python/damask/table.py
@@ -90,7 +90,7 @@ class Table():
             iloc = self.data.columns.get_loc(key).tolist().index(True) + int(idx) -1
             self.data.iloc[:,iloc] = array
         else: 
-            self.data[label]       = array
+            self.data[label]       = array.reshape(self.data[label].shape)
 
 
     def get_labels(self):

From 7d4b982c736f63f532d26bd7b67a1dcc155ff5b9 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Wed, 27 Nov 2019 12:08:23 +0100
Subject: [PATCH 15/17] make information on modifications mandatory

---
 processing/post/scaleData.py | 59 ++++++------------------------------
 python/damask/table.py       |  7 ++++-
 2 files changed, 15 insertions(+), 51 deletions(-)

diff --git a/processing/post/scaleData.py b/processing/post/scaleData.py
index 5b03f8e07..90eaeb0cc 100755
--- a/processing/post/scaleData.py
+++ b/processing/post/scaleData.py
@@ -35,58 +35,17 @@ parser.set_defaults(label  = [],
                    )
 
 (options,filenames) = parser.parse_args()
-
-if len(options.label) != len(options.factor):
-  parser.error('number of column labels and factors do not match.')
-
-# --- loop over input files -------------------------------------------------------------------------
-
 if filenames == []: filenames = [None]
 
+if len(options.label) != len(options.factor):
+    parser.error('number of column labels and factors do not match.')
+
 for name in filenames:
-  try:
-    table = damask.ASCIItable(name = name,
-                              buffered = False)
-  except: continue
-  damask.util.report(scriptName,name)
+    damask.util.report(scriptName,name)
 
-# ------------------------------------------ read header ------------------------------------------
+    table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name)
+    for i,label in enumerate(options.label):
+        table.set_array(label,table.get_array(label)*float(options.factor[i]),
+                        scriptID+' '+' '.join(sys.argv[1:]))
 
-  table.head_read()
-
-  errors  = []
-  remarks = []
-  columns  = []
-  dims     = []
-  factors  = []
-
-  for what,factor in zip(options.label,options.factor):
-    col = table.label_index(what)
-    if col < 0: remarks.append('column {} not found...'.format(what,type))
-    else:
-      columns.append(col)
-      factors.append(float(factor))
-      dims.append(table.label_dimension(what))
-
-  if remarks != []: damask.util.croak(remarks)
-  if errors  != []:
-    damask.util.croak(errors)
-    table.close(dismiss = True)
-    continue
-       
-# ------------------------------------------ assemble header ---------------------------------------  
-
-  table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))
-  table.head_write()
-
-# ------------------------------------------ process data ------------------------------------------
-
-  outputAlive = True
-  while outputAlive and table.data_read():                                                          # read next data line of ASCII table
-    for col,dim,factor in zip(columns,dims,factors):                                                # loop over items
-      table.data[col:col+dim] = factor * np.array(table.data[col:col+dim],'d')
-    outputAlive = table.data_write()                                                                # output processed line
-
-# ------------------------------------------ output finalization -----------------------------------  
-
-  table.close()                                                                                     # close ASCII tables
+    table.to_ASCII(sys.stdout if name is None else name)
diff --git a/python/damask/table.py b/python/damask/table.py
index 3cd6ed330..0c9f3ac40 100644
--- a/python/damask/table.py
+++ b/python/damask/table.py
@@ -83,8 +83,13 @@ class Table():
         else: 
             return self.data[label].to_numpy().reshape((-1,)+self.headings[label])
 
-    def set_array(self,label,array):
+    def set_array(self,label,array,info):
         """Set data."""
+        if np.prod(array.shape[1:],dtype=int) == 1: 
+            self.comments.append('{}: {}'.format(label,info))
+        else:
+            self.comments.append('{} {}: {}'.format(label,array.shape[1:],info))
+
         if re.match(r'[0-9]*?_',label):
             idx,key = label.split('_',1)
             iloc = self.data.columns.get_loc(key).tolist().index(True) + int(idx) -1

From cee095b58ed19a2b45fdcd76b7b138a659b94b34 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Wed, 27 Nov 2019 12:19:37 +0100
Subject: [PATCH 16/17] better use centralized code

---
 processing/post/addPK2.py    | 54 ++++++------------------------------
 processing/post/scaleData.py |  3 +-
 python/damask/mechanics.py   | 21 +++++++++++++-
 3 files changed, 29 insertions(+), 49 deletions(-)

diff --git a/processing/post/addPK2.py b/processing/post/addPK2.py
index f38753619..45afdbcd1 100755
--- a/processing/post/addPK2.py
+++ b/processing/post/addPK2.py
@@ -2,10 +2,9 @@
 
 import os
 import sys
+from io import StringIO
 from optparse import OptionParser
 
-import numpy as np
-
 import damask
 
 
@@ -36,53 +35,16 @@ parser.set_defaults(defgrad = 'f',
                    )
 
 (options,filenames) = parser.parse_args()
-
-# --- loop over input files -------------------------------------------------------------------------
-
 if filenames == []: filenames = [None]
 
 for name in filenames:
-  try:
-    table = damask.ASCIItable(name = name,
-                              buffered = False)
-  except: continue
-  damask.util.report(scriptName,name)
+    damask.util.report(scriptName,name)
 
-# ------------------------------------------ read header ------------------------------------------
+    table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name)
 
-  table.head_read()
+    table.add_array('S',
+                    damask.mechanics.PK2(table.get_array(options.defgrad).reshape(-1,3,3),
+                                         table.get_array(options.stress).reshape(-1,3,3)).reshape(-1,9),
+                    scriptID+' '+' '.join(sys.argv[1:]))
 
-# ------------------------------------------ sanity checks ----------------------------------------
-
-  errors = []
-  column = {}
-  
-  for tensor in [options.defgrad,options.stress]:
-    dim = table.label_dimension(tensor)
-    if   dim <  0: errors.append('column {} not found.'.format(tensor))
-    elif dim != 9: errors.append('column {} is not a tensor.'.format(tensor))
-    else:
-      column[tensor] = table.label_index(tensor)
-
-  if errors != []:
-    damask.util.croak(errors)
-    table.close(dismiss = True)
-    continue
-
-# ------------------------------------------ assemble header --------------------------------------
-
-  table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))
-  table.labels_append(['{}_S'.format(i+1) for i in range(9)])                                       # extend ASCII header with new labels
-  table.head_write()
-
-# ------------------------------------------ process data ------------------------------------------
-  outputAlive = True
-  while outputAlive and table.data_read():                                                          # read next data line of ASCII table
-    F = np.array(list(map(float,table.data[column[options.defgrad]:column[options.defgrad]+9])),'d').reshape(3,3)
-    P = np.array(list(map(float,table.data[column[options.stress ]:column[options.stress ]+9])),'d').reshape(3,3)
-    table.data_append(list(np.dot(np.linalg.inv(F),P).reshape(9)))                                  # [S] =[P].[F-1]
-    outputAlive = table.data_write()                                                                # output processed line
-
-# ------------------------------------------ output finalization -----------------------------------
-
-  table.close()                                                                                     # close input ASCII table (works for stdin)
+    table.to_ASCII(sys.stdout if name is None else name)
diff --git a/processing/post/scaleData.py b/processing/post/scaleData.py
index 90eaeb0cc..ad57f4b7e 100755
--- a/processing/post/scaleData.py
+++ b/processing/post/scaleData.py
@@ -2,10 +2,9 @@
 
 import os
 import sys
+from io import StringIO
 from optparse import OptionParser
 
-import numpy as np
-
 import damask
 
 
diff --git a/python/damask/mechanics.py b/python/damask/mechanics.py
index 476682380..5503d7048 100644
--- a/python/damask/mechanics.py
+++ b/python/damask/mechanics.py
@@ -19,7 +19,26 @@ def Cauchy(F,P):
     else:
         sigma = np.einsum('i,ijk,ilk->ijl',1.0/np.linalg.det(F),P,F)
     return symmetric(sigma)
-  
+
+
+def PK2(F,P):
+    """
+    Return 2. Piola-Kirchhoff stress calculated from 1. Piola-Kirchhoff stress and deformation gradient.
+      
+    Parameters
+    ----------
+    F : numpy.array of shape (:,3,3) or (3,3)
+      Deformation gradient.
+    P : numpy.array of shape (:,3,3) or (3,3)
+      1. Piola-Kirchhoff stress.
+
+    """
+    if np.shape(F) == np.shape(P) == (3,3):
+        S = np.dot(np.linalg.inv(F),P)
+    else:
+        S = np.einsum('ijk,ikl->ijl',np.linalg.inv(F),P)
+    return S
+
 
 def strain_tensor(F,t,m):
     """

From a8016d64bbce73cb3e44e08bc3e6ec2768506855 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Wed, 27 Nov 2019 12:31:32 +0100
Subject: [PATCH 17/17] simplified

---
 processing/post/shiftData.py | 62 ++++++------------------------------
 1 file changed, 10 insertions(+), 52 deletions(-)

diff --git a/processing/post/shiftData.py b/processing/post/shiftData.py
index 69a9696fa..591e68c5d 100755
--- a/processing/post/shiftData.py
+++ b/processing/post/shiftData.py
@@ -2,10 +2,9 @@
 
 import os
 import sys
+from io import StringIO
 from optparse import OptionParser
 
-import numpy as np
-
 import damask
 
 
@@ -35,58 +34,17 @@ parser.set_defaults(label  = [],
                    )
 
 (options,filenames) = parser.parse_args()
-
-if len(options.label) != len(options.offset):
-  parser.error('number of column labels and offsets do not match.')
-
-# --- loop over input files -------------------------------------------------------------------------
-
 if filenames == []: filenames = [None]
 
+if len(options.label) != len(options.offset):
+    parser.error('number of column labels and offsets do not match.')
+
 for name in filenames:
-  try:
-    table = damask.ASCIItable(name = name,
-                              buffered = False)
-  except: continue
-  damask.util.report(scriptName,name)
+    damask.util.report(scriptName,name)
 
-# ------------------------------------------ read header ------------------------------------------
+    table = damask.Table.from_ASCII(StringIO(''.join(sys.stdin.read())) if name is None else name)
+    for i,label in enumerate(options.label):
+        table.set_array(label,table.get_array(label)+float(options.offset[i]),
+                        scriptID+' '+' '.join(sys.argv[1:]))
 
-  table.head_read()
-
-  errors  = []
-  remarks = []
-  columns  = []
-  dims     = []
-  offsets  = []
-
-  for what,offset in zip(options.label,options.offset):
-    col = table.label_index(what)
-    if col < 0: remarks.append('column {} not found...'.format(what,type))
-    else:
-      columns.append(col)
-      offsets.append(float(offset))
-      dims.append(table.label_dimension(what))
-
-  if remarks != []: damask.util.croak(remarks)
-  if errors  != []:
-    damask.util.croak(errors)
-    table.close(dismiss = True)
-    continue
-       
-# ------------------------------------------ assemble header ---------------------------------------  
-
-  table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))
-  table.head_write()
-
-# ------------------------------------------ process data ------------------------------------------
-
-  outputAlive = True
-  while outputAlive and table.data_read():                                                          # read next data line of ASCII table
-    for col,dim,offset in zip(columns,dims,offsets):                                                # loop over items
-      table.data[col:col+dim] = offset + np.array(table.data[col:col+dim],'d')
-    outputAlive = table.data_write()                                                                # output processed line
-
-# ------------------------------------------ output finalization -----------------------------------  
-
-  table.close()                                                                                     # close ASCII tables
+    table.to_ASCII(sys.stdout if name is None else name)