223031012
/
DAMASK_EICMD
mirror of https://github.com/achalhp/DAMASK_EICMD.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

handling of MPI in 'parallelization'

This commit is contained in:
Martin Diehl 2020-09-13 10:19:38 +02:00
parent 126ef8be9f
commit da0e16520c
20 changed files with 84 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/CPFEM.f90
View File

@ -48,7 +48,7 @@ module CPFEM
end type tNumerics
type(tNumerics), private :: num
type, private :: tDebugOptions
logical :: &
basic, &
@ -58,9 +58,9 @@ module CPFEM
element, &
ip
end type tDebugOptions
type(tDebugOptions), private :: debugCPFEM
public :: &
CPFEM_general, &
CPFEM_initAll, &
@ -74,7 +74,9 @@ contains
!--------------------------------------------------------------------------------------------------
subroutine CPFEM_initAll
call parallelization_init
call DAMASK_interface_init
call prec_init
call IO_init
call config_init
call math_init
@ -112,13 +114,13 @@ subroutine CPFEM_init
allocate(CPFEM_dcsdE_knownGood(6,6,discretization_nIP,discretization_nElem), source= 0.0_pReal)
!------------------------------------------------------------------------------
! read numerical parameters and do sanity check
! read numerical parameters and do sanity check
num_commercialFEM => numerics_root%get('commercialFEM',defaultVal=emptyDict)
num%iJacoStiffness = num_commercialFEM%get_asInt('ijacostiffness',defaultVal=1)
if (num%iJacoStiffness < 1) call IO_error(301,ext_msg='iJacoStiffness')
!------------------------------------------------------------------------------
! read debug options
! read debug options
debug_CPFEM => debug_root%get('cpfem',defaultVal=emptyList)
debugCPFEM%basic = debug_CPFEM%contains('basic')

2
src/CPFEM2.f90
View File

@ -40,7 +40,9 @@ contains
!--------------------------------------------------------------------------------------------------
subroutine CPFEM_initAll
call parallelization_init
call DAMASK_interface_init ! Spectral and FEM interface to commandline
call prec_init
call IO_init
call base64_init
#ifdef Mesh

63
src/DAMASK_interface.f90
View File

@ -73,13 +73,7 @@ subroutine DAMASK_interface_init
userName !< name of user calling the executable
integer :: &
stat, &
i, &
#ifdef _OPENMP
threadLevel, &
#endif
worldrank = 0, &
worldsize = 0, &
typeSize
i
integer, dimension(8) :: &
dateAndTime
integer :: err
@ -87,44 +81,10 @@ subroutine DAMASK_interface_init
external :: &
quit
open(6, encoding='UTF-8') ! for special characters in output
!--------------------------------------------------------------------------------------------------
! PETSc Init
#ifdef _OPENMP
! If openMP is enabled, check if the MPI libary supports it and initialize accordingly.
! Otherwise, the first call to PETSc will do the initialization.
call MPI_Init_Thread(MPI_THREAD_FUNNELED,threadLevel,err)
if (err /= 0) call quit(1)
if (threadLevel<MPI_THREAD_FUNNELED) then
write(6,'(/,a)') ' ERROR: MPI library does not support OpenMP'
call quit(1)
endif
#endif
call PETScInitializeNoArguments(petsc_err) ! according to PETSc manual, that should be the first line in the code
CHKERRQ(petsc_err) ! this is a macro definition, it is case sensitive
call MPI_Comm_rank(PETSC_COMM_WORLD,worldrank,err)
if (err /= 0) call quit(1)
call MPI_Comm_size(PETSC_COMM_WORLD,worldsize,err)
if (err /= 0) call quit(1)
mainProcess: if (worldrank == 0) then
if (output_unit /= 6) then
write(output_unit,'(/,a)') ' ERROR: STDOUT != 6'
call quit(1)
endif
if (error_unit /= 0) then
write(output_unit,'(/,a)') ' ERROR: STDERR != 0'
call quit(1)
endif
else mainProcess
close(6) ! disable output for non-master processes (open 6 to rank specific file for debug)
open(6,file='/dev/null',status='replace') ! close(6) alone will leave some temp files in cwd
endif mainProcess
write(6,'(/,a)') ' <<<+- DAMASK_interface init -+>>>'
open(6, encoding='UTF-8') ! for special characters in output
! http://patorjk.com/software/taag/#p=display&f=Lean&t=DAMASK%203
#ifdef DEBUG
write(6,*) achar(27)//'[31m'
@ -162,20 +122,6 @@ subroutine DAMASK_interface_init
write(6,'(/,a,2(i2.2,a),i4.4)') ' Date: ',dateAndTime(3),'/',dateAndTime(2),'/', dateAndTime(1)
write(6,'(a,2(i2.2,a),i2.2)') ' Time: ',dateAndTime(5),':', dateAndTime(6),':', dateAndTime(7)
call MPI_Type_size(MPI_INTEGER,typeSize,err)
if (err /= 0) call quit(1)
if (typeSize*8 /= bit_size(0)) then
write(6,'(a)') ' Mismatch between MPI and DAMASK integer'
call quit(1)
endif
call MPI_Type_size(MPI_DOUBLE,typeSize,err)
if (err /= 0) call quit(1)
if (typeSize*8 /= storage_size(0.0_pReal)) then
write(6,'(a)') ' Mismatch between MPI and DAMASK real'
call quit(1)
endif
do i = 1, command_argument_count()
call get_command_argument(i,arg,status=err)
if (err /= 0) call quit(1)
@ -269,9 +215,6 @@ subroutine DAMASK_interface_init
call setSIGUSR1(.false.)
call setSIGUSR2(.false.)
call prec_init
call parallelization_init
end subroutine DAMASK_interface_init

5
src/DAMASK_marc.f90
View File

@ -26,7 +26,6 @@
#define PASTE(x,y) x ## y
#include "prec.f90"
#include "parallelization.f90"
module DAMASK_interface
use prec
@ -44,7 +43,6 @@ module DAMASK_interface
logical, protected, public :: symmetricSolver
character(len=*), parameter, public :: INPUTFILEEXTENSION = '.dat'
public :: &
DAMASK_interface_init, &
getSolverJobName
@ -91,9 +89,6 @@ subroutine DAMASK_interface_init
endif
symmetricSolver = solverIsSymmetric()
call prec_init
call parallelization_init
end subroutine DAMASK_interface_init

2
src/HDF5_utilities.f90
View File

@ -11,9 +11,9 @@ module HDF5_utilities
#endif
use prec
use parallelization
use IO
use rotations
use config
implicit none
public

1
src/commercialFEM_fileList.f90
View File

@ -3,6 +3,7 @@
!> @brief all DAMASK files without solver
!> @details List of files needed by MSC.Marc
!--------------------------------------------------------------------------------------------------
#include "parallelization.f90"
#include "IO.f90"
#include "YAML_types.f90"
#include "YAML_parse.f90"

20
src/config.f90
View File

@ -2,7 +2,7 @@
!> @author Martin Diehl, Max-Planck-Institut für Eisenforschung GmbH
!> @brief Reads in the material, numerics & debug configuration from their respective file
!> @details Reads the material configuration file, where solverJobName.yaml takes
!! precedence over material.yaml.
!! precedence over material.yaml.
!--------------------------------------------------------------------------------------------------
module config
use prec
@ -24,10 +24,6 @@ module config
numerics_root, &
debug_root
integer, protected, public :: &
worldrank = 0, & !< MPI worldrank (/=0 for MPI simulations only)
worldsize = 1 !< MPI worldsize (/=1 for MPI simulations only)
public :: &
config_init, &
config_deallocate
@ -40,7 +36,7 @@ contains
subroutine config_init
write(6,'(/,a)') ' <<<+- config init -+>>>'; flush(6)
call parse_material
call parse_numerics
call parse_debug
@ -76,12 +72,6 @@ end subroutine parse_material
subroutine parse_numerics
logical :: fexist
integer :: ierr
#ifdef PETSc
call MPI_Comm_rank(PETSC_COMM_WORLD,worldrank,ierr);CHKERRQ(ierr)
call MPI_Comm_size(PETSC_COMM_WORLD,worldsize,ierr);CHKERRQ(ierr)
#endif
numerics_root => emptyDict
inquire(file='numerics.yaml', exist=fexist)
@ -90,7 +80,6 @@ subroutine parse_numerics
numerics_root => parse_flow(to_flow(IO_read('numerics.yaml')))
endif
end subroutine parse_numerics
@ -99,7 +88,7 @@ end subroutine parse_numerics
!--------------------------------------------------------------------------------------------------
subroutine parse_debug
logical :: fexist
logical :: fexist
debug_root => emptyDict
inquire(file='debug.yaml', exist=fexist)
@ -113,10 +102,11 @@ end subroutine parse_debug
!--------------------------------------------------------------------------------------------------
!> @brief deallocates material.yaml structure
!ToDo: deallocation of numerics debug (optional)
!--------------------------------------------------------------------------------------------------
subroutine config_deallocate
deallocate(material_root) !ToDo: deallocation of numerics debug (optional)
deallocate(material_root)
end subroutine config_deallocate

1
src/crystallite.f90
View File

@ -10,6 +10,7 @@
module crystallite
use prec
use parallelization
use IO
use HDF5_utilities
use DAMASK_interface

1
src/grid/DAMASK_grid.f90
View File

@ -10,6 +10,7 @@ program DAMASK_grid
#include <petsc/finclude/petscsys.h>
use PETScsys
use prec
use parallelization
use DAMASK_interface
use IO
use config

1
src/grid/discretization_grid.f90
View File

@ -9,6 +9,7 @@ module discretization_grid
use PETScsys
use prec
use parallelization
use system_routines
use base64
use zlib

3
src/grid/grid_damage_spectral.f90
View File

@ -11,12 +11,13 @@ module grid_damage_spectral
use PETScsnes
use prec
use parallelization
use IO
use spectral_utilities
use discretization_grid
use damage_nonlocal
use config
use YAML_types
use config
implicit none
private

1
src/grid/grid_mech_FEM.f90
View File

@ -11,6 +11,7 @@ module grid_mech_FEM
use PETScsnes
use prec
use parallelization
use DAMASK_interface
use HDF5_utilities
use math

1
src/grid/grid_mech_spectral_basic.f90
View File

@ -11,6 +11,7 @@ module grid_mech_spectral_basic
use PETScsnes
use prec
use parallelization
use DAMASK_interface
use HDF5_utilities
use math

1
src/grid/grid_mech_spectral_polarisation.f90
View File

@ -11,6 +11,7 @@ module grid_mech_spectral_polarisation
use PETScsnes
use prec
use parallelization
use DAMASK_interface
use HDF5_utilities
use math

1
src/grid/grid_thermal_spectral.f90
View File

@ -11,6 +11,7 @@ module grid_thermal_spectral
use PETScsnes
use prec
use parallelization
use IO
use spectral_utilities
use discretization_grid

2
src/grid/spectral_utilities.f90
View File

@ -10,11 +10,11 @@ module spectral_utilities
use prec
use DAMASK_interface
use parallelization
use math
use rotations
use IO
use discretization_grid
use config
use discretization
use homogenization

1
src/mesh/DAMASK_mesh.f90
View File

@ -11,6 +11,7 @@ program DAMASK_mesh
use PetscDM
use prec
use DAMASK_interface
use parallelization
use IO
use math
use CPFEM2

1
src/mesh/discretization_mesh.f90
View File

@ -13,6 +13,7 @@ module discretization_mesh
use PETScis
use DAMASK_interface
use parallelization
use IO
use config
use discretization

56
src/parallelization.f90
View File

@ -4,6 +4,7 @@
!--------------------------------------------------------------------------------------------------
module parallelization
use prec
use, intrinsic :: iso_fortran_env
#ifdef PETSc
#include <petsc/finclude/petscsys.h>
@ -14,6 +15,10 @@ module parallelization
implicit none
private
integer, protected, public :: &
worldrank = 0, & !< MPI worldrank (/=0 for MPI simulations only)
worldsize = 1 !< MPI worldsize (/=1 for MPI simulations only)
public :: &
parallelization_init
@ -24,13 +29,56 @@ contains
!--------------------------------------------------------------------------------------------------
subroutine parallelization_init
!$ integer :: got_env, DAMASK_NUM_THREADS
integer :: err, typeSize
!$ integer :: got_env, DAMASK_NUM_THREADS, threadLevel
!$ character(len=6) NumThreadsString
#ifdef PETSc
PetscErrorCode :: petsc_err
#else
write(6,'(/,a)') ' <<<+- parallelization init -+>>>'; flush(6)
#endif
!$ call get_environment_variable(name='DAMASK_NUM_THREADS',value=NumThreadsString,STATUS=got_env) ! get environment variable DAMASK_NUM_THREADS...
!$ if(got_env /= 0) then ! could not get number of threads, set it to 1
#ifdef PETSc
#ifdef _OPENMP
! If openMP is enabled, check if the MPI libary supports it and initialize accordingly.
! Otherwise, the first call to PETSc will do the initialization.
call MPI_Init_Thread(MPI_THREAD_FUNNELED,threadLevel,err)
if (err /= 0) error stop 'MPI init failed'
if (threadLevel<MPI_THREAD_FUNNELED) error stop 'MPI library does not support OpenMP'
#endif
call PETScInitializeNoArguments(petsc_err) ! first line in the code according to PETSc manual
CHKERRQ(petsc_err)
call MPI_Comm_rank(PETSC_COMM_WORLD,worldrank,err)
if (err /= 0) error stop 'Could not determine worldrank'
if (worldrank == 1) write(6,'(/,a)') ' <<<+- parallelization init -+>>>'; flush(6)
call MPI_Comm_size(PETSC_COMM_WORLD,worldsize,err)
if (err /= 0) error stop 'Could not determine worldsize'
call MPI_Type_size(MPI_INTEGER,typeSize,err)
if (err /= 0) error stop 'Could not determine MPI integer size'
if (typeSize*8 /= bit_size(0)) error stop 'Mismatch between MPI and DAMASK integer'
call MPI_Type_size(MPI_DOUBLE,typeSize,err)
if (err /= 0) error stop 'Could not determine MPI real size'
if (typeSize*8 /= storage_size(0.0_pReal)) error stop 'Mismatch between MPI and DAMASK real'
#endif
mainProcess: if (worldrank == 0) then
if (output_unit /= 6) error stop 'STDOUT != 6'
if (error_unit /= 0) error stop 'STDERR != 0'
else mainProcess
close(6) ! disable output for non-master processes (open 6 to rank specific file for debug)
open(6,file='/dev/null',status='replace') ! close(6) alone will leave some temp files in cwd
endif mainProcess
!$ call get_environment_variable(name='DAMASK_NUM_THREADS',value=NumThreadsString,STATUS=got_env)
!$ if(got_env /= 0) then
!$ write(6,*) 'Could not determine value of $DAMASK_NUM_THREADS'
!$ DAMASK_NUM_THREADS = 1_pI32
!$ else
@ -42,7 +90,7 @@ subroutine parallelization_init
!$ endif
!$ write(6,'(a,i8,/)') ' DAMASK_NUM_THREADS: ',DAMASK_NUM_THREADS
!$ call omp_set_num_threads(DAMASK_NUM_THREADS)
end subroutine parallelization_init
end module parallelization

2
src/results.f90
View File

@ -6,8 +6,8 @@
!--------------------------------------------------------------------------------------------------
module results
use DAMASK_interface
use parallelization
use rotations
use config
use HDF5_utilities
#ifdef PETSc
use PETSC