diff --git a/src/CPFEM.f90 b/src/CPFEM.f90 index b2fccca26..8549046bd 100644 --- a/src/CPFEM.f90 +++ b/src/CPFEM.f90 @@ -48,7 +48,7 @@ module CPFEM end type tNumerics type(tNumerics), private :: num - + type, private :: tDebugOptions logical :: & basic, & @@ -58,9 +58,9 @@ module CPFEM element, & ip end type tDebugOptions - + type(tDebugOptions), private :: debugCPFEM - + public :: & CPFEM_general, & CPFEM_initAll, & @@ -74,7 +74,9 @@ contains !-------------------------------------------------------------------------------------------------- subroutine CPFEM_initAll + call parallelization_init call DAMASK_interface_init + call prec_init call IO_init call config_init call math_init @@ -112,13 +114,13 @@ subroutine CPFEM_init allocate(CPFEM_dcsdE_knownGood(6,6,discretization_nIP,discretization_nElem), source= 0.0_pReal) !------------------------------------------------------------------------------ -! read numerical parameters and do sanity check +! read numerical parameters and do sanity check num_commercialFEM => numerics_root%get('commercialFEM',defaultVal=emptyDict) num%iJacoStiffness = num_commercialFEM%get_asInt('ijacostiffness',defaultVal=1) if (num%iJacoStiffness < 1) call IO_error(301,ext_msg='iJacoStiffness') !------------------------------------------------------------------------------ -! read debug options +! read debug options debug_CPFEM => debug_root%get('cpfem',defaultVal=emptyList) debugCPFEM%basic = debug_CPFEM%contains('basic') diff --git a/src/CPFEM2.f90 b/src/CPFEM2.f90 index b2bdebcc6..645fcf59d 100644 --- a/src/CPFEM2.f90 +++ b/src/CPFEM2.f90 @@ -40,7 +40,9 @@ contains !-------------------------------------------------------------------------------------------------- subroutine CPFEM_initAll + call parallelization_init call DAMASK_interface_init ! Spectral and FEM interface to commandline + call prec_init call IO_init call base64_init #ifdef Mesh diff --git a/src/DAMASK_interface.f90 b/src/DAMASK_interface.f90 index f08e06f31..4a9761c48 100644 --- a/src/DAMASK_interface.f90 +++ b/src/DAMASK_interface.f90 @@ -73,13 +73,7 @@ subroutine DAMASK_interface_init userName !< name of user calling the executable integer :: & stat, & - i, & -#ifdef _OPENMP - threadLevel, & -#endif - worldrank = 0, & - worldsize = 0, & - typeSize + i integer, dimension(8) :: & dateAndTime integer :: err @@ -87,44 +81,10 @@ subroutine DAMASK_interface_init external :: & quit - open(6, encoding='UTF-8') ! for special characters in output - -!-------------------------------------------------------------------------------------------------- -! PETSc Init -#ifdef _OPENMP - ! If openMP is enabled, check if the MPI libary supports it and initialize accordingly. - ! Otherwise, the first call to PETSc will do the initialization. - call MPI_Init_Thread(MPI_THREAD_FUNNELED,threadLevel,err) - if (err /= 0) call quit(1) - if (threadLevel>>' + open(6, encoding='UTF-8') ! for special characters in output + ! http://patorjk.com/software/taag/#p=display&f=Lean&t=DAMASK%203 #ifdef DEBUG write(6,*) achar(27)//'[31m' @@ -162,20 +122,6 @@ subroutine DAMASK_interface_init write(6,'(/,a,2(i2.2,a),i4.4)') ' Date: ',dateAndTime(3),'/',dateAndTime(2),'/', dateAndTime(1) write(6,'(a,2(i2.2,a),i2.2)') ' Time: ',dateAndTime(5),':', dateAndTime(6),':', dateAndTime(7) - call MPI_Type_size(MPI_INTEGER,typeSize,err) - if (err /= 0) call quit(1) - if (typeSize*8 /= bit_size(0)) then - write(6,'(a)') ' Mismatch between MPI and DAMASK integer' - call quit(1) - endif - - call MPI_Type_size(MPI_DOUBLE,typeSize,err) - if (err /= 0) call quit(1) - if (typeSize*8 /= storage_size(0.0_pReal)) then - write(6,'(a)') ' Mismatch between MPI and DAMASK real' - call quit(1) - endif - do i = 1, command_argument_count() call get_command_argument(i,arg,status=err) if (err /= 0) call quit(1) @@ -269,9 +215,6 @@ subroutine DAMASK_interface_init call setSIGUSR1(.false.) call setSIGUSR2(.false.) - call prec_init - call parallelization_init - end subroutine DAMASK_interface_init diff --git a/src/DAMASK_marc.f90 b/src/DAMASK_marc.f90 index 4db1f4f15..d61031023 100644 --- a/src/DAMASK_marc.f90 +++ b/src/DAMASK_marc.f90 @@ -26,7 +26,6 @@ #define PASTE(x,y) x ## y #include "prec.f90" -#include "parallelization.f90" module DAMASK_interface use prec @@ -44,7 +43,6 @@ module DAMASK_interface logical, protected, public :: symmetricSolver character(len=*), parameter, public :: INPUTFILEEXTENSION = '.dat' - public :: & DAMASK_interface_init, & getSolverJobName @@ -91,9 +89,6 @@ subroutine DAMASK_interface_init endif symmetricSolver = solverIsSymmetric() - call prec_init - call parallelization_init - end subroutine DAMASK_interface_init diff --git a/src/HDF5_utilities.f90 b/src/HDF5_utilities.f90 index 0ac32fd2e..d50ff6aa5 100644 --- a/src/HDF5_utilities.f90 +++ b/src/HDF5_utilities.f90 @@ -11,9 +11,9 @@ module HDF5_utilities #endif use prec + use parallelization use IO use rotations - use config implicit none public diff --git a/src/commercialFEM_fileList.f90 b/src/commercialFEM_fileList.f90 index 0e5066268..f4eb5d78c 100644 --- a/src/commercialFEM_fileList.f90 +++ b/src/commercialFEM_fileList.f90 @@ -3,6 +3,7 @@ !> @brief all DAMASK files without solver !> @details List of files needed by MSC.Marc !-------------------------------------------------------------------------------------------------- +#include "parallelization.f90" #include "IO.f90" #include "YAML_types.f90" #include "YAML_parse.f90" diff --git a/src/config.f90 b/src/config.f90 index 342ac6a1b..ac62101ae 100644 --- a/src/config.f90 +++ b/src/config.f90 @@ -2,7 +2,7 @@ !> @author Martin Diehl, Max-Planck-Institut für Eisenforschung GmbH !> @brief Reads in the material, numerics & debug configuration from their respective file !> @details Reads the material configuration file, where solverJobName.yaml takes -!! precedence over material.yaml. +!! precedence over material.yaml. !-------------------------------------------------------------------------------------------------- module config use prec @@ -24,10 +24,6 @@ module config numerics_root, & debug_root - integer, protected, public :: & - worldrank = 0, & !< MPI worldrank (/=0 for MPI simulations only) - worldsize = 1 !< MPI worldsize (/=1 for MPI simulations only) - public :: & config_init, & config_deallocate @@ -40,7 +36,7 @@ contains subroutine config_init write(6,'(/,a)') ' <<<+- config init -+>>>'; flush(6) - + call parse_material call parse_numerics call parse_debug @@ -76,12 +72,6 @@ end subroutine parse_material subroutine parse_numerics logical :: fexist - integer :: ierr - -#ifdef PETSc - call MPI_Comm_rank(PETSC_COMM_WORLD,worldrank,ierr);CHKERRQ(ierr) - call MPI_Comm_size(PETSC_COMM_WORLD,worldsize,ierr);CHKERRQ(ierr) -#endif numerics_root => emptyDict inquire(file='numerics.yaml', exist=fexist) @@ -90,7 +80,6 @@ subroutine parse_numerics numerics_root => parse_flow(to_flow(IO_read('numerics.yaml'))) endif - end subroutine parse_numerics @@ -99,7 +88,7 @@ end subroutine parse_numerics !-------------------------------------------------------------------------------------------------- subroutine parse_debug - logical :: fexist + logical :: fexist debug_root => emptyDict inquire(file='debug.yaml', exist=fexist) @@ -113,10 +102,11 @@ end subroutine parse_debug !-------------------------------------------------------------------------------------------------- !> @brief deallocates material.yaml structure +!ToDo: deallocation of numerics debug (optional) !-------------------------------------------------------------------------------------------------- subroutine config_deallocate - deallocate(material_root) !ToDo: deallocation of numerics debug (optional) + deallocate(material_root) end subroutine config_deallocate diff --git a/src/crystallite.f90 b/src/crystallite.f90 index 9d2d534d7..595c0f2c1 100644 --- a/src/crystallite.f90 +++ b/src/crystallite.f90 @@ -10,6 +10,7 @@ module crystallite use prec + use parallelization use IO use HDF5_utilities use DAMASK_interface diff --git a/src/grid/DAMASK_grid.f90 b/src/grid/DAMASK_grid.f90 index 0e30025d8..edef00fa0 100644 --- a/src/grid/DAMASK_grid.f90 +++ b/src/grid/DAMASK_grid.f90 @@ -10,6 +10,7 @@ program DAMASK_grid #include use PETScsys use prec + use parallelization use DAMASK_interface use IO use config diff --git a/src/grid/discretization_grid.f90 b/src/grid/discretization_grid.f90 index 1fdb6c922..d9db8dbb3 100644 --- a/src/grid/discretization_grid.f90 +++ b/src/grid/discretization_grid.f90 @@ -9,6 +9,7 @@ module discretization_grid use PETScsys use prec + use parallelization use system_routines use base64 use zlib diff --git a/src/grid/grid_damage_spectral.f90 b/src/grid/grid_damage_spectral.f90 index e7bbbff7f..aaa3c2172 100644 --- a/src/grid/grid_damage_spectral.f90 +++ b/src/grid/grid_damage_spectral.f90 @@ -11,12 +11,13 @@ module grid_damage_spectral use PETScsnes use prec + use parallelization use IO use spectral_utilities use discretization_grid use damage_nonlocal - use config use YAML_types + use config implicit none private diff --git a/src/grid/grid_mech_FEM.f90 b/src/grid/grid_mech_FEM.f90 index e8e1345ef..9d3af8ae0 100644 --- a/src/grid/grid_mech_FEM.f90 +++ b/src/grid/grid_mech_FEM.f90 @@ -11,6 +11,7 @@ module grid_mech_FEM use PETScsnes use prec + use parallelization use DAMASK_interface use HDF5_utilities use math diff --git a/src/grid/grid_mech_spectral_basic.f90 b/src/grid/grid_mech_spectral_basic.f90 index 66694e516..664c9d7ae 100644 --- a/src/grid/grid_mech_spectral_basic.f90 +++ b/src/grid/grid_mech_spectral_basic.f90 @@ -11,6 +11,7 @@ module grid_mech_spectral_basic use PETScsnes use prec + use parallelization use DAMASK_interface use HDF5_utilities use math diff --git a/src/grid/grid_mech_spectral_polarisation.f90 b/src/grid/grid_mech_spectral_polarisation.f90 index f6d1b9ebb..53013f0cf 100644 --- a/src/grid/grid_mech_spectral_polarisation.f90 +++ b/src/grid/grid_mech_spectral_polarisation.f90 @@ -11,6 +11,7 @@ module grid_mech_spectral_polarisation use PETScsnes use prec + use parallelization use DAMASK_interface use HDF5_utilities use math diff --git a/src/grid/grid_thermal_spectral.f90 b/src/grid/grid_thermal_spectral.f90 index 58c89fdc4..a0bcd99ee 100644 --- a/src/grid/grid_thermal_spectral.f90 +++ b/src/grid/grid_thermal_spectral.f90 @@ -11,6 +11,7 @@ module grid_thermal_spectral use PETScsnes use prec + use parallelization use IO use spectral_utilities use discretization_grid diff --git a/src/grid/spectral_utilities.f90 b/src/grid/spectral_utilities.f90 index f1daf8f08..9ccce2617 100644 --- a/src/grid/spectral_utilities.f90 +++ b/src/grid/spectral_utilities.f90 @@ -10,11 +10,11 @@ module spectral_utilities use prec use DAMASK_interface + use parallelization use math use rotations use IO use discretization_grid - use config use discretization use homogenization diff --git a/src/mesh/DAMASK_mesh.f90 b/src/mesh/DAMASK_mesh.f90 index 05339a280..132cc4f3e 100644 --- a/src/mesh/DAMASK_mesh.f90 +++ b/src/mesh/DAMASK_mesh.f90 @@ -11,6 +11,7 @@ program DAMASK_mesh use PetscDM use prec use DAMASK_interface + use parallelization use IO use math use CPFEM2 diff --git a/src/mesh/discretization_mesh.f90 b/src/mesh/discretization_mesh.f90 index ac6ccf528..8d24896ee 100644 --- a/src/mesh/discretization_mesh.f90 +++ b/src/mesh/discretization_mesh.f90 @@ -13,6 +13,7 @@ module discretization_mesh use PETScis use DAMASK_interface + use parallelization use IO use config use discretization diff --git a/src/parallelization.f90 b/src/parallelization.f90 index bb32baf77..fb0fb9f30 100644 --- a/src/parallelization.f90 +++ b/src/parallelization.f90 @@ -4,6 +4,7 @@ !-------------------------------------------------------------------------------------------------- module parallelization use prec + use, intrinsic :: iso_fortran_env #ifdef PETSc #include @@ -14,6 +15,10 @@ module parallelization implicit none private + integer, protected, public :: & + worldrank = 0, & !< MPI worldrank (/=0 for MPI simulations only) + worldsize = 1 !< MPI worldsize (/=1 for MPI simulations only) + public :: & parallelization_init @@ -24,13 +29,56 @@ contains !-------------------------------------------------------------------------------------------------- subroutine parallelization_init -!$ integer :: got_env, DAMASK_NUM_THREADS + integer :: err, typeSize +!$ integer :: got_env, DAMASK_NUM_THREADS, threadLevel !$ character(len=6) NumThreadsString +#ifdef PETSc + PetscErrorCode :: petsc_err +#else write(6,'(/,a)') ' <<<+- parallelization init -+>>>'; flush(6) +#endif -!$ call get_environment_variable(name='DAMASK_NUM_THREADS',value=NumThreadsString,STATUS=got_env) ! get environment variable DAMASK_NUM_THREADS... -!$ if(got_env /= 0) then ! could not get number of threads, set it to 1 +#ifdef PETSc +#ifdef _OPENMP + ! If openMP is enabled, check if the MPI libary supports it and initialize accordingly. + ! Otherwise, the first call to PETSc will do the initialization. + call MPI_Init_Thread(MPI_THREAD_FUNNELED,threadLevel,err) + if (err /= 0) error stop 'MPI init failed' + if (threadLevel>>'; flush(6) + + call MPI_Comm_size(PETSC_COMM_WORLD,worldsize,err) + if (err /= 0) error stop 'Could not determine worldsize' + + call MPI_Type_size(MPI_INTEGER,typeSize,err) + if (err /= 0) error stop 'Could not determine MPI integer size' + if (typeSize*8 /= bit_size(0)) error stop 'Mismatch between MPI and DAMASK integer' + + call MPI_Type_size(MPI_DOUBLE,typeSize,err) + if (err /= 0) error stop 'Could not determine MPI real size' + if (typeSize*8 /= storage_size(0.0_pReal)) error stop 'Mismatch between MPI and DAMASK real' +#endif + + mainProcess: if (worldrank == 0) then + if (output_unit /= 6) error stop 'STDOUT != 6' + if (error_unit /= 0) error stop 'STDERR != 0' + else mainProcess + close(6) ! disable output for non-master processes (open 6 to rank specific file for debug) + open(6,file='/dev/null',status='replace') ! close(6) alone will leave some temp files in cwd + endif mainProcess + + +!$ call get_environment_variable(name='DAMASK_NUM_THREADS',value=NumThreadsString,STATUS=got_env) +!$ if(got_env /= 0) then !$ write(6,*) 'Could not determine value of $DAMASK_NUM_THREADS' !$ DAMASK_NUM_THREADS = 1_pI32 !$ else @@ -42,7 +90,7 @@ subroutine parallelization_init !$ endif !$ write(6,'(a,i8,/)') ' DAMASK_NUM_THREADS: ',DAMASK_NUM_THREADS !$ call omp_set_num_threads(DAMASK_NUM_THREADS) - + end subroutine parallelization_init end module parallelization diff --git a/src/results.f90 b/src/results.f90 index 2e5b01937..a4c90f41b 100644 --- a/src/results.f90 +++ b/src/results.f90 @@ -6,8 +6,8 @@ !-------------------------------------------------------------------------------------------------- module results use DAMASK_interface + use parallelization use rotations - use config use HDF5_utilities #ifdef PETSc use PETSC