Merge branch 'migrate-to-pathlib' into 'development'

Migrate to pathlib

See merge request damask/DAMASK!177

installation/symlink_Processing.py

@@ -1,56 +1,30 @@
 #!/usr/bin/env python3
 
 # Makes postprocessing routines accessible from everywhere.
-import os
 import sys
+from pathlib import Path
 
 import damask
 
-damaskEnv = damask.Environment()
+env = damask.Environment()
-baseDir = damaskEnv.relPath('processing/')
+bin_dir = env.root_dir/Path('bin')
-binDir  = damaskEnv.relPath('bin/')
 
-if not os.path.isdir(binDir):
+if not bin_dir.exists():
-  os.mkdir(binDir)
+    bin_dir.mkdir()
 
-#define ToDo list
-processing_subDirs    = ['pre',
-                         'post',
-                        ]
 
 sys.stdout.write('\nsymbolic linking...\n')
+for sub_dir in ['pre','post']:
+    the_dir = env.root_dir/Path('processing')/Path(sub_dir)
 
-for subDir in processing_subDirs:
+    for the_file in the_dir.glob('*.py'):
-  theDir = os.path.abspath(os.path.join(baseDir,subDir))
+        src = the_dir/the_file
-
+        dst = bin_dir/Path(the_file.with_suffix('').name)
-  sys.stdout.write('\n'+binDir+' ->\n'+theDir+damask.util.deemph(' ...')+'\n')
+        if dst.is_file(): dst.unlink() # dst.unlink(True) for Python >3.8
-
+        dst.symlink_to(src)
-  for theFile in os.listdir(theDir):
-    theName,theExt = os.path.splitext(theFile)
-    if theExt in ['.py']:
-
-      src      = os.path.abspath(os.path.join(theDir,theFile))
-      sym_link = os.path.abspath(os.path.join(binDir,theName))
-
-      if os.path.lexists(sym_link):
-        os.remove(sym_link)
-        output = theName+damask.util.deemph(theExt)
-      else:
-        output = damask.util.emph(theName)+damask.util.deemph(theExt)
-
-      sys.stdout.write(damask.util.deemph('... ')+output+'\n')
-      os.symlink(src,sym_link)
 
 
 sys.stdout.write('\npruning broken links...\n')
-
+for filename in bin_dir.glob('*'):
-brokenLinks = 0
+    if not filename.is_file():
-
+        filename.unlink
-for filename in os.listdir(binDir):
-  path = os.path.join(binDir,filename)
-  if os.path.islink(path) and not os.path.exists(path):
-    sys.stdout.write(' '+damask.util.delete(path)+'\n')
-    os.remove(path)
-    brokenLinks += 1
-
-sys.stdout.write(('none.' if brokenLinks == 0 else '')+'\n')

processing/pre/mentat_pbcOnBoxMesh.py

@@ -6,7 +6,7 @@ import numpy as np
 from optparse import OptionParser
 import damask
 
-sys.path.append(damask.solver.Marc().libraryPath())
+sys.path.append(str(damask.solver.Marc().library_path))
 
 scriptName = os.path.splitext(os.path.basename(__file__))[0]
 scriptID   = ' '.join([scriptName,damask.version])
@@ -197,14 +197,15 @@ def add_servoLinks(mfd_data,active=[True,True,True]):  # directions on which to
 
   if mfd_data[i]['uid'] == 1705: del mfd_data[i]
   mfd_data.insert(i, links)
-  
+
+
 #--------------------------------------------------------------------------------------------------
 #                                MAIN
 #-------------------------------------------------------------------------------------------------- 
+
 parser = OptionParser(option_class=damask.extendableOption, usage='%prog options [file[s]]', description = """
 Set up servo linking to achieve periodic boundary conditions for a regular hexahedral mesh.
 Use *py_connection to operate on model presently opened in MSC.Mentat.
-
 """, version = scriptID)
 
 parser.add_option('-p', '--port',
@@ -229,10 +230,7 @@ if remote and filenames != []:
 if filenames == []: filenames = [None]
 
 if remote:
-  try:    import py_mentat
+  import py_mentat
-  except:
-    damask.util.croak('no valid Mentat release found.')
-    sys.exit(-1)
 
   damask.util.report(scriptName, 'waiting to connect...')
   filenames = [os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names()) + '.mfd')]
@@ -240,14 +238,14 @@ if remote:
     py_mentat.py_connect('',options.port)
     py_mentat.py_send('*set_save_formatted on')
     py_mentat.py_send('*save_as_model "{}" yes'.format(filenames[0]))
-    py_mentat.py_get_int("nnodes()")                                    # hopefully blocks until file is written
+    py_mentat.py_get_int("nnodes()")
-  except:
+  except py_mentat.InputError as err:
-    damask.util.croak('failed. try setting Tools/Python/"Run as Separate Process" & "Initiate".')
+    damask.util.croak('{}. Try Tools/Python/"Run as Separate Process" & "Initiate".'.format(err))
-    sys.exit()
+    sys.exit(-1)
   damask.util.croak( 'connected...')
 
 for name in filenames:
-  while remote and not os.path.exists(name): time.sleep(0.5)           # wait for Mentat to write MFD file
+  while remote and not os.path.exists(name): time.sleep(0.5)
   with  open( name,'r') if name is not None else sys.stdin as fileIn:
     damask.util.report(scriptName, name)
     mfd = parseMFD(fileIn)
@@ -257,5 +255,4 @@ for name in filenames:
     fileOut.write(asMFD(mfd))
 
 if remote:
-  try:    py_mentat.py_send('*open_model "{}"'.format(filenames[0]))
+  py_mentat.py_send('*open_model "{}"'.format(filenames[0]))
-  except: damask.util.croak('lost connection on sending open command for "{}".'.format(filenames[0]))

processing/pre/mentat_spectralBox.py

@@ -9,7 +9,7 @@ import damask
 
 scriptName = os.path.splitext(os.path.basename(__file__))[0]
 scriptID   = ' '.join([scriptName,damask.version])
-sys.path.append(damask.solver.Marc().libraryPath())
+sys.path.append(str(damask.solver.Marc().library_path))
 
 #-------------------------------------------------------------------------------------------------
 def outMentat(cmd,locals):

python/damask/__init__.py

@@ -1,9 +1,9 @@
 """Tools for pre and post processing of DAMASK simulations."""
-import os as _os
+from pathlib import Path as _Path
 import re as _re
 
 name = 'damask'
-with open(_os.path.join(_os.path.dirname(__file__),'VERSION')) as _f:
+with open(_Path(__file__).parent/_Path('VERSION')) as _f:
     version = _re.sub(r'^v','',_f.readline().strip())
 
 # make classes directly accessible as damask.Class

python/damask/_environment.py

@@ -1,11 +1,11 @@
 import os
+from pathlib import Path
 import tkinter
 
 class Environment:
 
     def __init__(self):
         """Read and provide values of DAMASK configuration."""
-        self.options = self._get_options()
         try:
             tk = tkinter.Tk()
             self.screen_width  = tk.winfo_screenwidth()
@@ -15,17 +15,8 @@ class Environment:
             self.screen_width  = 1024
             self.screen_height =  768
 
-    def relPath(self,relative = '.'):
+    @property
-        """Return absolute path from path relative to DAMASK root."""
+    def options(self):
-        return os.path.join(self.rootDir(),relative)
-
-
-    def rootDir(self):
-        """Return DAMASK root path."""
-        return os.path.normpath(os.path.join(os.path.realpath(__file__),'../../../'))
-
-
-    def _get_options(self):
         options = {}
         for item in ['DAMASK_NUM_THREADS',
                      'MSC_ROOT',
@@ -34,3 +25,13 @@ class Environment:
             options[item] = os.environ[item] if item in os.environ else None
 
         return options
+
+    @property
+    def root_dir(self):
+        """Return DAMASK root path."""
+        return Path(__file__).parents[2]
+
+
+    # for compatibility
+    def rootDir(self):
+        return str(self.root_dir)

python/damask/_result.py

@@ -6,6 +6,7 @@ import os
 import datetime
 import xml.etree.ElementTree as ET
 import xml.dom.minidom
+from pathlib import Path
 from functools import partial
 
 import h5py
@@ -88,7 +89,7 @@ class Result:
                           'con_physics':    self.con_physics, 'mat_physics':    self.mat_physics
                          }
 
-        self.fname = os.path.abspath(fname)
+        self.fname = Path(fname).absolute()
 
         self._allow_modification = False
 
@@ -1056,14 +1057,17 @@ class Result:
         Parameters
         ----------
         func : function
-            Callback function that calculates a new dataset from one or more datasets per HDF5 group.
+            Callback function that calculates a new dataset from one or
+            more datasets per HDF5 group.
         datasets : dictionary
-            Details of the datasets to be used: label (in HDF5 file) and arg (argument to which the data is parsed in func).
+            Details of the datasets to be used: label (in HDF5 file) and
+            arg (argument to which the data is parsed in func).
         args : dictionary, optional
             Arguments parsed to func.
 
         """
-        pool = multiprocessing.Pool(int(Environment().options['DAMASK_NUM_THREADS']))
+        num_threads = Environment().options['DAMASK_NUM_THREADS']
+        pool = multiprocessing.Pool(int(num_threads) if num_threads is not None else None)
         lock = multiprocessing.Manager().Lock()
 
         groups = self.groups_with_datasets(datasets.values())
@@ -1190,7 +1194,7 @@ class Result:
                                                        'Dimensions': '{} {} {} {}'.format(*self.grid,np.prod(shape))}
                                 data_items[-1].text='{}:{}'.format(os.path.split(self.fname)[1],name)
 
-        with open(os.path.splitext(self.fname)[0]+'.xdmf','w') as f:
+        with open(self.fname.with_suffix('.xdmf').name,'w') as f:
             f.write(xml.dom.minidom.parseString(ET.tostring(xdmf).decode()).toprettyxml())
 
 
@@ -1266,7 +1270,4 @@ class Result:
             u = self.read_dataset(self.get_dataset_location('u_n' if mode.lower() == 'cell' else 'u_p'))
             v.add(u,'u')
 
-            file_out = '{}_inc{}'.format(os.path.splitext(os.path.basename(self.fname))[0],
+            v.write('{}_inc{}'.format(self.fname.stem,inc[3:].zfill(N_digits)))
-                                            inc[3:].zfill(N_digits))
-
-            v.write(file_out)

python/damask/_vtk.py

@@ -1,4 +1,4 @@
-import os
+from pathlib import Path
 
 import pandas as pd
 import numpy as np
@@ -126,8 +126,8 @@ class VTK:
             vtkUnstructuredGrid, and vtkPolyData.
 
         """
-        ext = os.path.splitext(fname)[1]
+        ext = Path(fname).suffix
-        if ext == '.vtk':
+        if ext == '.vtk' or dataset_type:
             reader = vtk.vtkGenericDataObjectReader()
             reader.SetFileName(fname)
             reader.Update()
@@ -176,10 +176,10 @@ class VTK:
             writer = vtk.vtkXMLPolyDataWriter()
 
         default_ext = writer.GetDefaultFileExtension()
-        name, ext = os.path.splitext(fname)
+        ext = Path(fname).suffix
         if ext and ext != '.'+default_ext:
-            raise ValueError('Given extension {} is not .{}'.format(ext,default_ext))
+            raise ValueError('Given extension {} does not match default .{}'.format(ext,default_ext))
-        writer.SetFileName('{}.{}'.format(name,default_ext))
+        writer.SetFileName(str(Path(fname).with_suffix('.'+default_ext)))
         writer.SetCompressorTypeToZLib()
         writer.SetDataModeToBinary()
         writer.SetInputData(self.geom)

python/damask/solver/_marc.py

@@ -1,7 +1,7 @@
-import os
 import subprocess
 import shlex
 import string
+from pathlib import Path
 
 from .._environment import Environment
 
@@ -19,28 +19,24 @@ class Marc:
 
         """
         self.solver  = 'Marc'
-        try:
+        self.version = version
-            self.version = int(version)
-        except TypeError:
-            self.version = -1
 
-
+    @property
-#--------------------------
+    def library_path(self):
-    def libraryPath(self):
 
         path_MSC = Environment().options['MSC_ROOT']
-        path_lib = '{}/mentat{}/shlib/linux64'.format(path_MSC,self.version)
+        path_lib = Path('{}/mentat{}/shlib/linux64'.format(path_MSC,self.version))
 
-        return path_lib if os.path.exists(path_lib) else ''
+        return path_lib if path_lib.is_dir() else None
 
 
-#--------------------------
+    @property
-    def toolsPath(self):
+    def tools_path(self):
 
         path_MSC   = Environment().options['MSC_ROOT']
-        path_tools = '{}/marc{}/tools'.format(path_MSC,self.version)
+        path_tools = Path('{}/marc{}/tools'.format(path_MSC,self.version))
 
-        return path_tools if os.path.exists(path_tools) else ''
+        return path_tools if path_tools.is_dir() else None
 
 
 #--------------------------
@@ -53,21 +49,21 @@ class Marc:
                   ):
 
 
-        damaskEnv = Environment()
+        env = Environment()
 
-        user = os.path.join(damaskEnv.relPath('src'),'DAMASK_marc{}.{}'.format(self.version,'f90' if compile else 'marc'))
+        user = env.root_dir/Path('src/DAMASK_marc{}'.format(self.version)).with_suffix('.f90' if compile else '.marc')
-        if not os.path.isfile(user):
+        if not user.is_file():
             raise FileNotFoundError("DAMASK4Marc ({}) '{}' not found".format(('source' if compile else 'binary'),user))
 
         # Define options [see Marc Installation and Operation Guide, pp 23]
         script = 'run_damask_{}mp'.format(optimization)
 
-        cmd = os.path.join(self.toolsPath(),script) + \
+        cmd = str(self.tools_path/Path(script)) + \
               ' -jid ' + model + '_' + job + \
               ' -nprocd 1  -autorst 0 -ci n  -cr n  -dcoup 0 -b no -v no'
 
-        if compile: cmd += ' -u ' + user + ' -save y'
+        if compile: cmd += ' -u ' + str(user) + ' -save y'
-        else:       cmd += ' -prog ' + os.path.splitext(user)[0]
+        else:       cmd += ' -prog ' + str(user.with_suffix(''))
 
         print('job submission {} compilation: {}'.format('with' if compile else 'without',user))
         if logfile: log = open(logfile, 'w')

python/tests/test_VTK.py

@@ -17,18 +17,24 @@ class TestVTK:
         size   = np.random.random(3) + 1.0
         origin = np.random.random(3)
         v = VTK.from_rectilinearGrid(grid,size,origin)
-        s = v.__repr__()
+        string = v.__repr__()
         v.write(os.path.join(tmp_path,'rectilinearGrid'))
-        v = VTK.from_file(os.path.join(tmp_path,'rectilinearGrid.vtr'))
+        vtr = VTK.from_file(os.path.join(tmp_path,'rectilinearGrid.vtr'))
-        assert(s == v.__repr__())
+        with open(os.path.join(tmp_path,'rectilinearGrid.vtk'),'w') as f:
+            f.write(string)
+        vtk = VTK.from_file(os.path.join(tmp_path,'rectilinearGrid.vtk'),'VTK_rectilinearGrid')
+        assert(string == vtr.__repr__() == vtk.__repr__())
 
     def test_polyData(self,tmp_path):
         points = np.random.rand(3,100)
         v = VTK.from_polyData(points)
-        s = v.__repr__()
+        string = v.__repr__()
         v.write(os.path.join(tmp_path,'polyData'))
-        v = VTK.from_file(os.path.join(tmp_path,'polyData.vtp'))
+        vtp = VTK.from_file(os.path.join(tmp_path,'polyData.vtp'))
-        assert(s == v.__repr__())
+        with open(os.path.join(tmp_path,'polyData.vtk'),'w') as f:
+            f.write(string)
+        vtk = VTK.from_file(os.path.join(tmp_path,'polyData.vtk'),'polyData')
+        assert(string == vtp.__repr__() == vtk.__repr__())
 
     @pytest.mark.parametrize('cell_type,n',[
                                             ('VTK_hexahedron',8),
@@ -41,7 +47,17 @@ class TestVTK:
         nodes = np.random.rand(n,3)
         connectivity = np.random.choice(np.arange(n),n,False).reshape(-1,n)
         v = VTK.from_unstructuredGrid(nodes,connectivity,cell_type)
-        s = v.__repr__()
+        string = v.__repr__()
         v.write(os.path.join(tmp_path,'unstructuredGrid'))
-        v = VTK.from_file(os.path.join(tmp_path,'unstructuredGrid.vtu'))
+        vtu = VTK.from_file(os.path.join(tmp_path,'unstructuredGrid.vtu'))
-        assert(s == v.__repr__())
+        with open(os.path.join(tmp_path,'unstructuredGrid.vtk'),'w') as f:
+            f.write(string)
+        vtk = VTK.from_file(os.path.join(tmp_path,'unstructuredGrid.vtk'),'unstructuredgrid')
+        assert(string == vtu.__repr__() == vtk.__repr__())
+
+    @pytest.mark.parametrize('name,dataset_type',[('this_file_does_not_exist.vtk',None),
+                                                  ('this_file_does_not_exist.vtk','vtk'),
+                                                  ('this_file_does_not_exist.vtx', None)])
+    def test_invalid_dataset_type(self,dataset_type,name):
+        with pytest.raises(TypeError):
+            VTK.from_file('this_file_does_not_exist.vtk',dataset_type)