From 11b2b04b392ab68814c07f19d4760d4ff913b0f9 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 28 Apr 2021 08:47:35 +0200 Subject: [PATCH 1/5] MSC_ROOT and MSC_VERSION have no meaning anymore DAMASK_grid/mesh use 4 threads per default (hard coded in parallelization.f90). It's the user's responsibility to use a different value for OMP_NUM_THREADS --- env/DAMASK.sh | 7 ++----- env/DAMASK.zsh | 9 ++------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/env/DAMASK.sh b/env/DAMASK.sh index edf019921..335fb4a6b 100644 --- a/env/DAMASK.sh +++ b/env/DAMASK.sh @@ -30,7 +30,6 @@ PATH=${DAMASK_ROOT}/bin:$PATH SOLVER=$(type -p DAMASK_grid || true 2>/dev/null) [ "x$SOLVER" == "x" ] && SOLVER=$(blink 'Not found!') -[ "x$OMP_NUM_THREADS" == "x" ] && OMP_NUM_THREADS=1 # currently, there is no information that unlimited stack size causes problems # still, http://software.intel.com/en-us/forums/topic/501500 suggest to fix it @@ -55,10 +54,8 @@ if [ ! -z "$PS1" ]; then [[ $(canonicalPath "$PETSC_DIR") == $PETSC_DIR ]] \ || echo " ~~> "$(canonicalPath "$PETSC_DIR") fi - echo -n "MSC.Marc/Mentat " - [ -d $MSC_ROOT ] && echo $MSC_ROOT || blink $MSC_ROOT - echo - echo "Multithreading OMP_NUM_THREADS=$OMP_NUM_THREADS" + [ "x$PETSC_ARCH" != "x" ] && echo "PETSc architecture $PETSC_ARCH" + [ "x$OMP_NUM_THREADS" != "x" ] && echo "Multithreading OMP_NUM_THREADS=$OMP_NUM_THREADS" echo -n "heap size " [[ "$(ulimit -d)" == "unlimited" ]] \ && echo "unlimited" \ diff --git a/env/DAMASK.zsh b/env/DAMASK.zsh index 07064eb7a..c4c7b705c 100644 --- a/env/DAMASK.zsh +++ b/env/DAMASK.zsh @@ -20,7 +20,6 @@ PATH=${DAMASK_ROOT}/bin:$PATH SOLVER=$(which DAMASK_grid || true 2>/dev/null) [[ "x$SOLVER" == "x" ]] && SOLVER=$(blink 'Not found!') -[[ "x$OMP_NUM_THREADS" == "x" ]] && OMP_NUM_THREADS=1 # currently, there is no information that unlimited stack size causes problems # still, http://software.intel.com/en-us/forums/topic/501500 suggest to fix it @@ -45,12 +44,8 @@ if [ ! -z "$PS1" ]; then [[ $(canonicalPath "$PETSC_DIR") == $PETSC_DIR ]] \ || echo " ~~> "$(canonicalPath "$PETSC_DIR") fi - [[ "x$PETSC_ARCH" == "x" ]] \ - || echo "PETSc architecture $PETSC_ARCH" - echo -n "MSC.Marc/Mentat " - [ -d $MSC_ROOT ] && echo $MSC_ROOT || blink $MSC_ROOT - echo - echo "Multithreading OMP_NUM_THREADS=$OMP_NUM_THREADS" + [[ "x$PETSC_ARCH" != "x" ]] && echo "PETSc architecture $PETSC_ARCH" + [[ "x$OMP_NUM_THREADS" != "x" ]] && echo "Multithreading OMP_NUM_THREADS=$OMP_NUM_THREADS" echo -n "heap size " [[ "$(ulimit -d)" == "unlimited" ]] \ && echo "unlimited" \ From f860b26664777f8ba22d40913be17fdbd077df63 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 28 Apr 2021 08:53:40 +0200 Subject: [PATCH 2/5] correct type hints --- installation/mods_MarcMentat/apply_DAMASK_modifications.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installation/mods_MarcMentat/apply_DAMASK_modifications.py b/installation/mods_MarcMentat/apply_DAMASK_modifications.py index 5b6b7a2ce..4c6568466 100755 --- a/installation/mods_MarcMentat/apply_DAMASK_modifications.py +++ b/installation/mods_MarcMentat/apply_DAMASK_modifications.py @@ -17,10 +17,10 @@ parser.add_argument('--editor', dest='editor', metavar='string', default='vi', parser.add_argument('--msc-root', dest='msc_root', metavar='string', default=damask.solver._marc._msc_root, help='MSC.Marc/Mentat root directory') -parser.add_argument('--msc-version', dest='msc_version', metavar='string', +parser.add_argument('--msc-version', dest='msc_version', type=float, metavar='float', default=damask.solver._marc._msc_version, help='MSC.Marc/Mentat version') -parser.add_argument('--damask-root', dest='damask_root', type=float, metavar = 'float', +parser.add_argument('--damask-root', dest='damask_root', metavar = 'string', default=damask.solver._marc._damask_root, help='DAMASK root directory') From f89f07e5cf86dcd8efbfdd50aee15d5563881ccc Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 28 Apr 2021 19:04:54 +0200 Subject: [PATCH 3/5] avoid use of global variables in functions --- .../apply_DAMASK_modifications.py | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/installation/mods_MarcMentat/apply_DAMASK_modifications.py b/installation/mods_MarcMentat/apply_DAMASK_modifications.py index 4c6568466..38323b3f7 100755 --- a/installation/mods_MarcMentat/apply_DAMASK_modifications.py +++ b/installation/mods_MarcMentat/apply_DAMASK_modifications.py @@ -7,6 +7,11 @@ from pathlib import Path import damask +def copy_and_replace(in_file,dst,msc_root,editor): + with open(in_file) as f_in, open(dst/Path(in_file).name,'w') as f_out: + f_out.write(f_in.read().replace('%INSTALLDIR%',msc_root).replace('%EDITOR%',editor)) + + parser = argparse.ArgumentParser( description='Apply DAMASK modification to MSC.Marc/Mentat', prog = Path(__file__).name, @@ -24,27 +29,18 @@ parser.add_argument('--damask-root', dest='damask_root', metavar = 'string', default=damask.solver._marc._damask_root, help='DAMASK root directory') - args = parser.parse_args() msc_root = Path(args.msc_root) damask_root = Path(args.damask_root) msc_version = args.msc_version -def copy_and_replace(in_file,dst): - with open(in_file) as f: - content = f.read() - content = content.replace('%INSTALLDIR%',str(msc_root)) - content = content.replace('%EDITOR%', args.editor) - with open(dst/Path(in_file).name,'w') as f: - f.write(content) - print('adapting Marc tools...\n') src = damask_root/f'installation/mods_MarcMentat/{msc_version}/Marc_tools' dst = msc_root/f'marc{msc_version}/tools' for in_file in glob.glob(str(src/'*damask*')) + [str(src/'include_linux64')]: - copy_and_replace(in_file,dst) + copy_and_replace(in_file,dst,args.msc_root,args.editor) print('adapting Mentat scripts and menus...\n') @@ -52,12 +48,12 @@ print('adapting Mentat scripts and menus...\n') src = damask_root/f'installation/mods_MarcMentat/{msc_version}/Mentat_bin' dst = msc_root/f'mentat{msc_version}/bin' for in_file in glob.glob(str(src/'*[!.original]')): - copy_and_replace(in_file,dst) + copy_and_replace(in_file,dst,args.msc_root,args.editor) src = damask_root/f'installation/mods_MarcMentat/{msc_version}/Mentat_menus' dst = msc_root/f'mentat{msc_version}/menus' for in_file in glob.glob(str(src/'job_run.ms')): - copy_and_replace(in_file,dst) + copy_and_replace(in_file,dst,args.msc_root,args.editor) print('compiling Mentat menu binaries...') From 00bc6a90314dace3d300e9c4e8d421cbc6a80d3d Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 28 Apr 2021 20:26:25 +0200 Subject: [PATCH 4/5] 4 threads as default. Reasonable for modern computers hardcoded + and as shell variable --- env/DAMASK.sh | 3 ++- env/DAMASK.zsh | 3 ++- python/damask/_grid.py | 4 ++-- python/damask/_result.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/env/DAMASK.sh b/env/DAMASK.sh index 335fb4a6b..c74c447ed 100644 --- a/env/DAMASK.sh +++ b/env/DAMASK.sh @@ -55,7 +55,8 @@ if [ ! -z "$PS1" ]; then || echo " ~~> "$(canonicalPath "$PETSC_DIR") fi [ "x$PETSC_ARCH" != "x" ] && echo "PETSc architecture $PETSC_ARCH" - [ "x$OMP_NUM_THREADS" != "x" ] && echo "Multithreading OMP_NUM_THREADS=$OMP_NUM_THREADS" + [ "x$OMP_NUM_THREADS" == "x" ] && export OMP_NUM_THREADS=4 + echo "Multithreading OMP_NUM_THREADS=$OMP_NUM_THREADS" echo -n "heap size " [[ "$(ulimit -d)" == "unlimited" ]] \ && echo "unlimited" \ diff --git a/env/DAMASK.zsh b/env/DAMASK.zsh index c4c7b705c..5813d110a 100644 --- a/env/DAMASK.zsh +++ b/env/DAMASK.zsh @@ -45,7 +45,8 @@ if [ ! -z "$PS1" ]; then || echo " ~~> "$(canonicalPath "$PETSC_DIR") fi [[ "x$PETSC_ARCH" != "x" ]] && echo "PETSc architecture $PETSC_ARCH" - [[ "x$OMP_NUM_THREADS" != "x" ]] && echo "Multithreading OMP_NUM_THREADS=$OMP_NUM_THREADS" + [[ "x$OMP_NUM_THREADS" == "x" ]] && export OMP_NUM_THREADS=4 + echo "Multithreading OMP_NUM_THREADS=$OMP_NUM_THREADS" echo -n "heap size " [[ "$(ulimit -d)" == "unlimited" ]] \ && echo "unlimited" \ diff --git a/python/damask/_grid.py b/python/damask/_grid.py index 1b58870e4..9e30ba5f9 100644 --- a/python/damask/_grid.py +++ b/python/damask/_grid.py @@ -392,7 +392,7 @@ class Grid: seeds_p = seeds coords = grid_filters.coordinates0_point(cells,size).reshape(-1,3) - pool = mp.Pool(int(os.environ.get('OMP_NUM_THREADS',1))) + pool = mp.Pool(int(os.environ.get('OMP_NUM_THREADS',4))) result = pool.map_async(partial(Grid._find_closest_seed,seeds_p,weights_p), [coord for coord in coords]) pool.close() pool.join() @@ -437,7 +437,7 @@ class Grid: """ coords = grid_filters.coordinates0_point(cells,size).reshape(-1,3) KDTree = spatial.cKDTree(seeds,boxsize=size) if periodic else spatial.cKDTree(seeds) - devNull,material_ = KDTree.query(coords) + devNull,material_ = KDTree.query(coords, workers = int(os.environ.get('OMP_NUM_THREADS',4))) return Grid(material = (material_ if material is None else material[material_]).reshape(cells), size = size, diff --git a/python/damask/_result.py b/python/damask/_result.py index 0520f08f2..8dd915d1b 100644 --- a/python/damask/_result.py +++ b/python/damask/_result.py @@ -1179,7 +1179,7 @@ class Result: """ chunk_size = 1024**2//8 - pool = mp.Pool(int(os.environ.get('OMP_NUM_THREADS',1))) + pool = mp.Pool(int(os.environ.get('OMP_NUM_THREADS',4))) lock = mp.Manager().Lock() groups = [] From b061b4911ddb6f217b661418dd4851f44303539b Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 29 Apr 2021 08:56:40 +0200 Subject: [PATCH 5/5] compatible with older scipy versions --- python/damask/_grid.py | 8 ++++++-- python/damask/seeds.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/damask/_grid.py b/python/damask/_grid.py index 9e30ba5f9..237d56ba1 100644 --- a/python/damask/_grid.py +++ b/python/damask/_grid.py @@ -436,8 +436,12 @@ class Grid: """ coords = grid_filters.coordinates0_point(cells,size).reshape(-1,3) - KDTree = spatial.cKDTree(seeds,boxsize=size) if periodic else spatial.cKDTree(seeds) - devNull,material_ = KDTree.query(coords, workers = int(os.environ.get('OMP_NUM_THREADS',4))) + tree = spatial.cKDTree(seeds,boxsize=size) if periodic else \ + spatial.cKDTree(seeds) + try: + material_ = tree.query(coords, workers = int(os.environ.get('OMP_NUM_THREADS',4)))[1] + except TypeError: + material_ = tree.query(coords, n_jobs = int(os.environ.get('OMP_NUM_THREADS',4)))[1] # scipy <1.6 return Grid(material = (material_ if material is None else material[material_]).reshape(cells), size = size, diff --git a/python/damask/seeds.py b/python/damask/seeds.py index 02a050e66..126f138aa 100644 --- a/python/damask/seeds.py +++ b/python/damask/seeds.py @@ -78,7 +78,7 @@ def from_Poisson_disc(size,N_seeds,N_candidates,distance,periodic=True,rng_seed= candidates = rng.random((N_candidates,3))*_np.broadcast_to(size,(N_candidates,3)) tree = _spatial.cKDTree(coords[:s],boxsize=size) if periodic else \ _spatial.cKDTree(coords[:s]) - distances, dev_null = tree.query(candidates) + distances = tree.query(candidates)[0] best = distances.argmax() if distances[best] > distance: # require minimum separation coords[s] = candidates[best] # maximum separation to existing point cloud