parallelize addition of datasets

threads does not work, muliprocessing shows good performance: Overhead
is small compared to the performance gain.
Especially useful for long running functions of the orientation class

python/damask/dadf5.py

@@ -1,7 +1,8 @@
-from queue     import Queue
+import multiprocessing
 import re
 import glob
 import os
+from functools import partial
 
 import vtk
 from vtk.util import numpy_support
@@ -443,6 +444,17 @@ class DADF5():
                 return f['geometry/x_c'][()]
 
 
+    @staticmethod
+    def _add_absolute(x):
+        return {
+                'data':  np.abs(x['data']),
+                'label': '|{}|'.format(x['label']),
+                'meta':  {
+                          'Unit':        x['meta']['Unit'],
+                          'Description': 'Absolute value of {} ({})'.format(x['label'],x['meta']['Description']),
+                          'Creator':     'dadf5.py:add_abs v{}'.format(version)
+                          }
+                 }
     def add_absolute(self,x):
         """
         Add absolute value.
@@ -453,21 +465,24 @@ class DADF5():
           Label of scalar, vector, or tensor dataset to take absolute value of.
 
         """
-        def _add_absolute(x):
+        self.__add_generic_pointwise(self._add_absolute,{'x':x})
+
+
+    @staticmethod
+    def _add_calculation(**kwargs):
+        formula = kwargs['formula']
+        for d in re.findall(r'#(.*?)#',formula):
+            formula = formula.replace('#{}#'.format(d),"kwargs['{}']['data']".format(d))
 
         return {
-                  'data':  np.abs(x['data']),
+                'data':  eval(formula),
-                  'label': '|{}|'.format(x['label']),
+                'label': kwargs['label'],
                 'meta':  {
-                            'Unit':        x['meta']['Unit'],
+                          'Unit':        kwargs['unit'],
-                            'Description': 'Absolute value of {} ({})'.format(x['label'],x['meta']['Description']),
+                          'Description': '{} (formula: {})'.format(kwargs['description'],kwargs['formula']),
-                            'Creator':     'dadf5.py:add_abs v{}'.format(version)
+                          'Creator':     'dadf5.py:add_calculation v{}'.format(version)
                           }
                  }
-
-        self.__add_generic_pointwise(_add_absolute,{'x':x})
-
-
     def add_calculation(self,label,formula,unit='n/a',description=None,vectorized=True):
         """
         Add result of a general formula.
@@ -489,28 +504,24 @@ class DADF5():
         if not vectorized:
           raise NotImplementedError
 
-        def _add_calculation(**kwargs):
-
-          formula = kwargs['formula']
-          for d in re.findall(r'#(.*?)#',formula):
-            formula = formula.replace('#{}#'.format(d),"kwargs['{}']['data']".format(d))
-
-          return {
-                  'data':  eval(formula),
-                  'label': kwargs['label'],
-                  'meta':  {
-                            'Unit':        kwargs['unit'],
-                            'Description': '{} (formula: {})'.format(kwargs['description'],kwargs['formula']),
-                            'Creator':     'dadf5.py:add_calculation v{}'.format(version)
-                            }
-                   }
-
         dataset_mapping  = {d:d for d in set(re.findall(r'#(.*?)#',formula))}                       # datasets used in the formula
         args             = {'formula':formula,'label':label,'unit':unit,'description':description}
-
+        self.__add_generic_pointwise(self._add_calculation,dataset_mapping,args)
-        self.__add_generic_pointwise(_add_calculation,dataset_mapping,args)
 
 
+    @staticmethod
+    def _add_Cauchy(P,F):
+        return {
+                'data':  mechanics.Cauchy(P['data'],F['data']),
+                'label': 'sigma',
+                'meta':  {
+                          'Unit':        P['meta']['Unit'],
+                          'Description': 'Cauchy stress calculated from {} ({}) '.format(P['label'],
+                                                                                         P['meta']['Description'])+\
+                                         'and {} ({})'.format(F['label'],F['meta']['Description']),
+                          'Creator':     'dadf5.py:add_Cauchy v{}'.format(version)
+                          }
+                }
     def add_Cauchy(self,P='P',F='F'):
         """
         Add Cauchy stress calculated from first Piola-Kirchhoff stress and deformation gradient.
@@ -523,23 +534,20 @@ class DADF5():
           Label of the dataset containing the deformation gradient. Defaults to ‘F’.
 
         """
-        def _add_Cauchy(P,F):
+        self.__add_generic_pointwise(self._add_Cauchy,{'P':P,'F':F})
 
+
+    @staticmethod
+    def _add_determinant(T):
         return {
-                  'data':  mechanics.Cauchy(P['data'],F['data']),
+                'data':  np.linalg.det(T['data']),
-                  'label': 'sigma',
+                'label': 'det({})'.format(T['label']),
                 'meta':  {
-                            'Unit':        P['meta']['Unit'],
+                          'Unit':        T['meta']['Unit'],
-                            'Description': 'Cauchy stress calculated from {} ({}) '.format(P['label'],
+                          'Description': 'Determinant of tensor {} ({})'.format(T['label'],T['meta']['Description']),
-                                                                                           P['meta']['Description'])+\
+                          'Creator':     'dadf5.py:add_determinant v{}'.format(version)
-                                           'and {} ({})'.format(F['label'],F['meta']['Description']),
-                            'Creator':     'dadf5.py:add_Cauchy v{}'.format(version)
                           }
                 }
-
-        self.__add_generic_pointwise(_add_Cauchy,{'P':P,'F':F})
-
-
     def add_determinant(self,T):
         """
         Add the determinant of a tensor.
@@ -550,33 +558,11 @@ class DADF5():
           Label of tensor dataset.
 
         """
-        def _add_determinant(T):
+        self.__add_generic_pointwise(self._add_determinant,{'T':T})
-
-          return {
-                  'data':  np.linalg.det(T['data']),
-                  'label': 'det({})'.format(T['label']),
-                  'meta':  {
-                            'Unit':        T['meta']['Unit'],
-                            'Description': 'Determinant of tensor {} ({})'.format(T['label'],T['meta']['Description']),
-                            'Creator':     'dadf5.py:add_determinant v{}'.format(version)
-                            }
-                  }
-
-        self.__add_generic_pointwise(_add_determinant,{'T':T})
 
 
-    def add_deviator(self,T):
+    @staticmethod
-        """
-        Add the deviatoric part of a tensor.
-
-        Parameters
-        ----------
-        T : str
-          Label of tensor dataset.
-
-        """
     def _add_deviator(T):
-
         if not np.all(np.array(T['data'].shape[1:]) == np.array([3,3])):
             raise ValueError
 
@@ -589,10 +575,30 @@ class DADF5():
                           'Creator':     'dadf5.py:add_deviator v{}'.format(version)
                           }
                  }
+    def add_deviator(self,T):
+        """
+        Add the deviatoric part of a tensor.
 
-        self.__add_generic_pointwise(_add_deviator,{'T':T})
+        Parameters
+        ----------
+        T : str
+          Label of tensor dataset.
+
+        """
+        self.__add_generic_pointwise(self._add_deviator,{'T':T})
 
 
+    @staticmethod
+    def _add_eigenvalue(S):
+        return {
+                'data': mechanics.eigenvalues(S['data']),
+                'label': 'lambda({})'.format(S['label']),
+                'meta' : {
+                          'Unit':         S['meta']['Unit'],
+                          'Description': 'Eigenvalues of {} ({})'.format(S['label'],S['meta']['Description']),
+                          'Creator':     'dadf5.py:add_eigenvalues v{}'.format(version)
+                         }
+                }
     def add_eigenvalues(self,S):
         """
         Add eigenvalues of symmetric tensor.
@@ -603,21 +609,20 @@ class DADF5():
           Label of symmetric tensor dataset.
 
         """
-        def _add_eigenvalue(S):
+        self.__add_generic_pointwise(self._add_eigenvalue,{'S':S})
 
+
+    @staticmethod
+    def _add_eigenvector(S):
         return {
-                  'data': mechanics.eigenvalues(S['data']),
+                'data': mechanics.eigenvectors(S['data']),
-                  'label': 'lambda({})'.format(S['label']),
+                'label': 'v({})'.format(S['label']),
                 'meta' : {
-                            'Unit':         S['meta']['Unit'],
+                          'Unit':        '1',
-                            'Description': 'Eigenvalues of {} ({})'.format(S['label'],S['meta']['Description']),
+                          'Description': 'Eigenvectors of {} ({})'.format(S['label'],S['meta']['Description']),
-                            'Creator':     'dadf5.py:add_eigenvalues v{}'.format(version)
+                          'Creator':     'dadf5.py:add_eigenvectors v{}'.format(version)
                          }
                 }
-
-        self.__add_generic_pointwise(_add_eigenvalue,{'S':S})
-
-
     def add_eigenvectors(self,S):
         """
         Add eigenvectors of symmetric tensor.
@@ -628,35 +633,11 @@ class DADF5():
           Label of symmetric tensor dataset.
 
         """
-        def _add_eigenvector(S):
+        self.__add_generic_pointwise(self._add_eigenvector,{'S':S})
-
-          return {
-                  'data': mechanics.eigenvectors(S['data']),
-                  'label': 'v({})'.format(S['label']),
-                  'meta' : {
-                            'Unit':        '1',
-                            'Description': 'Eigenvectors of {} ({})'.format(S['label'],S['meta']['Description']),
-                            'Creator':     'dadf5.py:add_eigenvectors v{}'.format(version)
-                           }
-                  }
-
-        self.__add_generic_pointwise(_add_eigenvector,{'S':S})
 
 
-    def add_IPFcolor(self,q,l):
+    @staticmethod
-        """
-        Add RGB color tuple of inverse pole figure (IPF) color.
-
-        Parameters
-        ----------
-        q : str
-          Label of the dataset containing the crystallographic orientation as quaternions.
-        l : numpy.array of shape (3)
-          Lab frame direction for inverse pole figure.
-
-        """
     def _add_IPFcolor(q,l):
-
         d      = np.array(l)
         d_unit = d/np.linalg.norm(d)
         m      = util.scale_to_coprime(d)
@@ -678,10 +659,32 @@ class DADF5():
                           'Creator':     'dadf5.py:add_IPFcolor v{}'.format(version)
                          }
                }
+    def add_IPFcolor(self,q,l):
+        """
+        Add RGB color tuple of inverse pole figure (IPF) color.
 
-        self.__add_generic_pointwise(_add_IPFcolor,{'q':q},{'l':l})
+        Parameters
+        ----------
+        q : str
+          Label of the dataset containing the crystallographic orientation as quaternions.
+        l : numpy.array of shape (3)
+          Lab frame direction for inverse pole figure.
+
+        """
+        self.__add_generic_pointwise(self._add_IPFcolor,{'q':q},{'l':l})
 
 
+    @staticmethod
+    def _add_maximum_shear(S):
+        return {
+                'data':  mechanics.maximum_shear(S['data']),
+                'label': 'max_shear({})'.format(S['label']),
+                'meta':  {
+                          'Unit':        S['meta']['Unit'],
+                          'Description': 'Maximum shear component of {} ({})'.format(S['label'],S['meta']['Description']),
+                          'Creator':     'dadf5.py:add_maximum_shear v{}'.format(version)
+                          }
+                 }
     def add_maximum_shear(self,S):
         """
         Add maximum shear components of symmetric tensor.
@@ -692,21 +695,23 @@ class DADF5():
           Label of symmetric tensor dataset.
 
         """
-        def _add_maximum_shear(S):
+        self.__add_generic_pointwise(self._add_maximum_shear,{'S':S})
+
+
+    @staticmethod
+    def _add_Mises(S):
+        t = 'strain' if S['meta']['Unit'] == '1' else \
+            'stress'
 
         return {
-                  'data':  mechanics.maximum_shear(S['data']),
+                'data':  mechanics.Mises_strain(S['data']) if t=='strain' else mechanics.Mises_stress(S['data']),
-                  'label': 'max_shear({})'.format(S['label']),
+                'label': '{}_vM'.format(S['label']),
                 'meta':  {
                           'Unit':        S['meta']['Unit'],
-                            'Description': 'Maximum shear component of {} ({})'.format(S['label'],S['meta']['Description']),
+                          'Description': 'Mises equivalent {} of {} ({})'.format(t,S['label'],S['meta']['Description']),
-                            'Creator':     'dadf5.py:add_maximum_shear v{}'.format(version)
+                          'Creator':     'dadf5.py:add_Mises v{}'.format(version)
                           }
                 }
-
-        self.__add_generic_pointwise(_add_maximum_shear,{'S':S})
-
-
     def add_Mises(self,S):
         """
         Add the equivalent Mises stress or strain of a symmetric tensor.
@@ -717,37 +722,11 @@ class DADF5():
           Label of symmetric tensorial stress or strain dataset.
 
         """
-        def _add_Mises(S):
+        self.__add_generic_pointwise(self._add_Mises,{'S':S})
-
-          t = 'strain' if S['meta']['Unit'] == '1' else \
-              'stress'
-          return {
-                  'data':  mechanics.Mises_strain(S['data']) if t=='strain' else mechanics.Mises_stress(S['data']),
-                  'label': '{}_vM'.format(S['label']),
-                  'meta':  {
-                            'Unit':        S['meta']['Unit'],
-                            'Description': 'Mises equivalent {} of {} ({})'.format(t,S['label'],S['meta']['Description']),
-                            'Creator':     'dadf5.py:add_Mises v{}'.format(version)
-                            }
-                  }
-
-        self.__add_generic_pointwise(_add_Mises,{'S':S})
 
 
-    def add_norm(self,x,ord=None):
+    @staticmethod
-        """
-        Add the norm of vector or tensor.
-
-        Parameters
-        ----------
-        x : str
-          Label of vector or tensor dataset.
-        ord : {non-zero int, inf, -inf, ‘fro’, ‘nuc’}, optional
-          Order of the norm. inf means NumPy’s inf object. For details refer to numpy.linalg.norm.
-
-        """
     def _add_norm(x,ord):
-
         o = ord
         if len(x['data'].shape) == 2:
             axis = 1
@@ -769,24 +748,23 @@ class DADF5():
                           'Creator':     'dadf5.py:add_norm v{}'.format(version)
                           }
                  }
-
+    def add_norm(self,x,ord=None):
-        self.__add_generic_pointwise(_add_norm,{'x':x},{'ord':ord})
-
-
-    def add_PK2(self,P='P',F='F'):
         """
-        Add 2. Piola-Kirchhoff calculated from first Piola-Kirchhoff stress and deformation gradient.
+        Add the norm of vector or tensor.
 
         Parameters
         ----------
-        P : str, optional
+        x : str
-          Label first  Piola-Kirchhoff stress dataset. Defaults to ‘P’.
+          Label of vector or tensor dataset.
-        F : str, optional
+        ord : {non-zero int, inf, -inf, ‘fro’, ‘nuc’}, optional
-          Label of deformation gradient dataset. Defaults to ‘F’.
+          Order of the norm. inf means NumPy’s inf object. For details refer to numpy.linalg.norm.
 
         """
-        def _add_PK2(P,F):
+        self.__add_generic_pointwise(self._add_norm,{'x':x},{'ord':ord})
 
+
+    @staticmethod
+    def _add_PK2(P,F):
         return {
                 'data':  mechanics.PK2(P['data'],F['data']),
                 'label': 'S',
@@ -798,26 +776,23 @@ class DADF5():
                           'Creator':     'dadf5.py:add_PK2 v{}'.format(version)
                           }
                 }
-
+    def add_PK2(self,P='P',F='F'):
-        self.__add_generic_pointwise(_add_PK2,{'P':P,'F':F})
-
-
-    def add_pole(self,q,p,polar=False):
         """
-        Add coordinates of stereographic projection of given pole in crystal frame.
+        Add second Piola-Kirchhoff calculated from first Piola-Kirchhoff stress and deformation gradient.
 
         Parameters
         ----------
-        q : str
+        P : str, optional
-          Label of the dataset containing the crystallographic orientation as quaternions.
+          Label first  Piola-Kirchhoff stress dataset. Defaults to ‘P’.
-        p : numpy.array of shape (3)
+        F : str, optional
-          Crystallographic direction or plane.
+          Label of deformation gradient dataset. Defaults to ‘F’.
-        polar : bool, optional
-          Give pole in polar coordinates. Defaults to False.
 
         """
-        def _add_pole(q,p,polar):
+        self.__add_generic_pointwise(self._add_PK2,{'P':P,'F':F})
 
+
+    @staticmethod
+    def _add_pole(q,p,polar):
         pole      = np.array(p)
         unit_pole = pole/np.linalg.norm(pole)
         m         = util.scale_to_coprime(pole)
@@ -839,10 +814,36 @@ class DADF5():
                           'Creator' : 'dadf5.py:add_pole v{}'.format(version)
                          }
                }
+    def add_pole(self,q,p,polar=False):
+        """
+        Add coordinates of stereographic projection of given pole in crystal frame.
 
-        self.__add_generic_pointwise(_add_pole,{'q':q},{'p':p,'polar':polar})
+        Parameters
+        ----------
+        q : str
+          Label of the dataset containing the crystallographic orientation as quaternions.
+        p : numpy.array of shape (3)
+          Crystallographic direction or plane.
+        polar : bool, optional
+          Give pole in polar coordinates. Defaults to False.
+
+        """
+        self.__add_generic_pointwise(self._add_pole,{'q':q},{'p':p,'polar':polar})
 
 
+    @staticmethod
+    def _add_rotational_part(F):
+        if not np.all(np.array(F['data'].shape[1:]) == np.array([3,3])):
+            raise ValueError
+        return {
+                'data':  mechanics.rotational_part(F['data']),
+                'label': 'R({})'.format(F['label']),
+                'meta':  {
+                          'Unit':        F['meta']['Unit'],
+                          'Description': 'Rotational part of {} ({})'.format(F['label'],F['meta']['Description']),
+                          'Creator':     'dadf5.py:add_rotational_part v{}'.format(version)
+                          }
+                 }
     def add_rotational_part(self,F):
         """
         Add rotational part of a deformation gradient.
@@ -853,33 +854,12 @@ class DADF5():
           Label of deformation gradient dataset.
 
         """
-        def _add_rotational_part(F):
 
-          return {
+        self.__add_generic_pointwise(self._add_rotational_part,{'F':F})
-                  'data':  mechanics.rotational_part(F['data']),
-                  'label': 'R({})'.format(F['label']),
-                  'meta':  {
-                            'Unit':        F['meta']['Unit'],
-                            'Description': 'Rotational part of {} ({})'.format(F['label'],F['meta']['Description']),
-                            'Creator':     'dadf5.py:add_rotational_part v{}'.format(version)
-                            }
-                   }
-
-        self.__add_generic_pointwise(_add_rotational_part,{'F':F})
 
 
-    def add_spherical(self,T):
+    @staticmethod
-        """
-        Add the spherical (hydrostatic) part of a tensor.
-
-        Parameters
-        ----------
-        T : str
-          Label of tensor dataset.
-
-        """
     def _add_spherical(T):
-
         if not np.all(np.array(T['data'].shape[1:]) == np.array([3,3])):
             raise ValueError
 
@@ -892,10 +872,32 @@ class DADF5():
                           'Creator':     'dadf5.py:add_spherical v{}'.format(version)
                           }
                  }
+    def add_spherical(self,T):
+        """
+        Add the spherical (hydrostatic) part of a tensor.
 
-        self.__add_generic_pointwise(_add_spherical,{'T':T})
+        Parameters
+        ----------
+        T : str
+          Label of tensor dataset.
+
+        """
+        self.__add_generic_pointwise(self._add_spherical,{'T':T})
 
 
+    @staticmethod
+    def _add_strain_tensor(F,t,m):
+        if not np.all(np.array(F['data'].shape[1:]) == np.array([3,3])):
+            raise ValueError
+        return {
+                'data':  mechanics.strain_tensor(F['data'],t,m),
+                'label': 'epsilon_{}^{}({})'.format(t,m,F['label']),
+                'meta':  {
+                          'Unit':        F['meta']['Unit'],
+                          'Description': 'Strain tensor of {} ({})'.format(F['label'],F['meta']['Description']),
+                          'Creator':     'dadf5.py:add_strain_tensor v{}'.format(version)
+                          }
+                 }
     def add_strain_tensor(self,F='F',t='V',m=0.0):
         """
         Add strain tensor of a deformation gradient.
@@ -913,21 +915,24 @@ class DADF5():
           Order of the strain calculation. Defaults to ‘0.0’.
 
         """
-        def _add_strain_tensor(F,t,m):
+        self.__add_generic_pointwise(self._add_strain_tensor,{'F':F},{'t':t,'m':m})
+
+
+    @staticmethod
+    def _add_stretch_tensor(F,t):
+        if not np.all(np.array(F['data'].shape[1:]) == np.array([3,3])):
+            raise ValueError
 
         return {
-                  'data':  mechanics.strain_tensor(F['data'],t,m),
+                'data':  mechanics.left_stretch(F['data']) if t == 'V' else mechanics.right_stretch(F['data']),
-                  'label': 'epsilon_{}^{}({})'.format(t,m,F['label']),
+                'label': '{}({})'.format(t,F['label']),
                 'meta':  {
                           'Unit':        F['meta']['Unit'],
-                            'Description': 'Strain tensor of {} ({})'.format(F['label'],F['meta']['Description']),
+                          'Description': '{} stretch tensor of {} ({})'.format('Left' if t == 'V' else 'Right',
-                            'Creator':     'dadf5.py:add_strain_tensor v{}'.format(version)
+                                                                               F['label'],F['meta']['Description']),
+                          'Creator':     'dadf5.py:add_stretch_tensor v{}'.format(version)
                           }
                  }
-
-        self.__add_generic_pointwise(_add_strain_tensor,{'F':F},{'t':t,'m':m})
-
-
     def add_stretch_tensor(self,F='F',t='V'):
         """
         Add stretch tensor of a deformation gradient.
@@ -941,77 +946,54 @@ class DADF5():
           Defaults to ‘V’.
 
         """
-        def _add_stretch_tensor(F,t):
+        self.__add_generic_pointwise(self._add_stretch_tensor,{'F':F},{'t':t})
-
-          return {
-                  'data':  mechanics.left_stretch(F['data']) if t == 'V' else mechanics.right_stretch(F['data']),
-                  'label': '{}({})'.format(t,F['label']),
-                  'meta':  {
-                            'Unit':        F['meta']['Unit'],
-                            'Description': '{} stretch tensor of {} ({})'.format('Left' if t == 'V' else 'Right',
-                                                                                 F['label'],F['meta']['Description']),
-                            'Creator':     'dadf5.py:add_stretch_tensor v{}'.format(version)
-                            }
-                   }
-
-        self.__add_generic_pointwise(_add_stretch_tensor,{'F':F},{'t':t})
 
 
-    def __add_generic_pointwise(self,func,dataset_mapping,args={}):
+    def job(self,group,func,datasets,args,lock):
-        """
+        try:
-        General function to add pointwise data.
+            d = self._read(group,datasets,lock)
+            r = func(**d,**args)
+            return [group,r]
+        except Exception as err:
+            print('Error during calculation: {}.'.format(err))
+            return None
 
-        Parameters
-        ----------
-        func : function
-          Function that calculates a new dataset from one or more datasets per HDF5 group.
-        dataset_mapping : dictionary
-          Mapping HDF5 data label to callback function argument
-        extra_args : dictionary, optional
-          Any extra arguments parsed to func.
 
-        """
+    def _read(self,group,datasets,lock):
-        def job(args):
+      datasets_in = {}
-          """Call function with input data + extra arguments, returns results + group."""
+      lock.acquire()
-          args['results'].put({**args['func'](**args['in']),'group':args['group']})
+      with h5py.File(self.fname,'r') as f:
+        for k,v in datasets.items():
+          loc  = f[group+'/'+v]
+          datasets_in[k]={'data':loc[()],
+                          'label':v,
+                          'meta':{k2:v2.decode() for k2,v2 in loc.attrs.items()}}
+      lock.release()
+      return datasets_in
+
+    def __add_generic_pointwise(self,func,datasets,args={}):
 
       env = Environment()
       N_threads = int(env.options['DAMASK_NUM_THREADS'])
-        N_threads //=N_threads # disable for the moment
+      pool = multiprocessing.Pool(N_threads)
+      m = multiprocessing.Manager()
+      lock = m.Lock()
 
-        results = Queue(N_threads)
+      groups = self.groups_with_datasets(datasets.values())
-        pool    = util.ThreadPool(N_threads)
+      default_arg = partial(self.job,func=func,datasets=datasets,args=args,lock=lock)
-        N_added = N_threads + 1
+      for result in pool.imap_unordered(default_arg,groups):
-
+          if not result: continue
-        todo = []
+          lock.acquire()
-        # ToDo: It would be more memory efficient to read only from file when required, i.e. do to it in pool.add_task
+          with h5py.File(self.fname, 'a') as f:
-        for group in self.groups_with_datasets(dataset_mapping.values()):
+              try:
-          with h5py.File(self.fname,'r') as f:
+                  dataset = f[result[0]].create_dataset(result[1]['label'],data=result[1]['data'])
-            datasets_in = {}
+                  for l,v in result[1]['meta'].items():
-            for arg,label in dataset_mapping.items():
+                      dataset.attrs[l]=v.encode()
-              loc  = f[group+'/'+label]
+              except OSError as err:
-              data = loc[()]
+                  print('Could not add dataset: {}.'.format(err))
-              meta = {k:loc.attrs[k].decode() for k in loc.attrs.keys()}
+          lock.release()
-              datasets_in[arg] = {'data': data, 'meta': meta, 'label': label}
+      pool.close()
-
+      pool.join()
-          todo.append({'in':{**datasets_in,**args},'func':func,'group':group,'results':results})
-
-        pool.map(job, todo[:N_added])                                                                   # initialize
-
-        N_not_calculated = len(todo)
-        while N_not_calculated > 0:
-          result = results.get()
-          with h5py.File(self.fname,'a') as f:                                                          # write to file
-            dataset_out = f[result['group']].create_dataset(result['label'],data=result['data'])
-            for k in result['meta'].keys():
-              dataset_out.attrs[k] = result['meta'][k].encode()
-            N_not_calculated-=1
-
-          if N_added < len(todo):                                                                       # add more jobs
-            pool.add_task(job,todo[N_added])
-            N_added +=1
-
-        pool.wait_completion()
 
 
     def to_vtk(self,labels,mode='cell'):

python/damask/util.py

@@ -201,57 +201,3 @@ class return_message():
   def __repr__(self):
     """Return message suitable for interactive shells."""
     return srepr(self.message)
-
-
-class ThreadPool:
-  """Pool of threads consuming tasks from a queue."""
-
-  class Worker(Thread):
-    """Thread executing tasks from a given tasks queue."""
-
-    def __init__(self, tasks):
-      """Worker for tasks."""
-      Thread.__init__(self)
-      self.tasks = tasks
-      self.daemon = True
-      self.start()
-
-    def run(self):
-      while True:
-        func, args, kargs = self.tasks.get()
-        try:
-          func(*args, **kargs)
-        except Exception as e:
-          # An exception happened in this thread
-          print(e)
-        finally:
-          # Mark this task as done, whether an exception happened or not
-          self.tasks.task_done()
-
-
-  def __init__(self, num_threads):
-    """
-    Thread pool.
-
-    Parameters
-    ----------
-    num_threads : int
-      number of threads
-
-    """
-    self.tasks = Queue(num_threads)
-    for _ in range(num_threads):
-      self.Worker(self.tasks)
-
-  def add_task(self, func, *args, **kargs):
-    """Add a task to the queue."""
-    self.tasks.put((func, args, kargs))
-
-  def map(self, func, args_list):
-    """Add a list of tasks to the queue."""
-    for args in args_list:
-      self.add_task(func, args)
-
-  def wait_completion(self):
-    """Wait for completion of all the tasks in the queue."""
-    self.tasks.join()