From 186b688b04f0440ab59d967869243525f91a391e Mon Sep 17 00:00:00 2001 From: Philip Eisenlohr Date: Tue, 9 Nov 2021 15:10:19 -0500 Subject: [PATCH 01/24] only look for opening part of tag --- src/grid/discretization_grid.f90 | 36 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/grid/discretization_grid.f90 b/src/grid/discretization_grid.f90 index e16001feb..515f9977f 100644 --- a/src/grid/discretization_grid.f90 +++ b/src/grid/discretization_grid.f90 @@ -75,7 +75,7 @@ subroutine discretization_grid_init(restart) print'(/,a)', ' <<<+- discretization_grid init -+>>>'; flush(IO_STDOUT) - if(worldrank == 0) then + if (worldrank == 0) then fileContent = IO_read(interface_geomFile) call readVTI(grid,geomSize,origin,materialAt_global,fileContent) fname = interface_geomFile @@ -100,7 +100,7 @@ subroutine discretization_grid_init(restart) print'(a,3(es12.5))', ' size x y z: ', geomSize print'(a,3(es12.5))', ' origin x y z: ', origin - if(worldsize>grid(3)) call IO_error(894, ext_msg='number of processes exceeds grid(3)') + if (worldsize>grid(3)) call IO_error(894, ext_msg='number of processes exceeds grid(3)') call fftw_mpi_init devNull = fftw_mpi_local_size_3d(int(grid(3),C_INTPTR_T), & @@ -109,7 +109,7 @@ subroutine discretization_grid_init(restart) PETSC_COMM_WORLD, & z, & ! domain grid size along z z_offset) ! domain grid offset along z - if(z==0_C_INTPTR_T) call IO_error(894, ext_msg='Cannot distribute MPI processes') + if (z==0_C_INTPTR_T) call IO_error(894, ext_msg='Cannot distribute MPI processes') grid3 = int(z) grid3Offset = int(z_offset) @@ -136,7 +136,7 @@ subroutine discretization_grid_init(restart) !-------------------------------------------------------------------------------------------------- ! store geometry information for post processing - if(.not. restart) then + if (.not. restart) then call results_openJobFile call results_closeGroup(results_addGroup('geometry')) call results_addAttribute('cells', grid, '/geometry') @@ -202,7 +202,7 @@ subroutine readVTI(grid,geomSize,origin,material, & if (endPos < startPos) endPos = len(fileContent,kind=pI64) ! end of file without new line if (.not. inFile) then - if(index(fileContent(startPos:endPos),'',kind=pI64) /= 0_pI64) then + if (index(fileContent(startPos:endPos),'',kind=pI64) == 0_pI64) if (index(fileContent(startPos:endPos),' Date: Wed, 10 Nov 2021 16:56:47 +0100 Subject: [PATCH 02/24] 'standard' style --- src/grid/discretization_grid.f90 | 53 +++++++++++++++++--------------- src/prec.f90 | 16 +++++----- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/src/grid/discretization_grid.f90 b/src/grid/discretization_grid.f90 index 515f9977f..69746bbe3 100644 --- a/src/grid/discretization_grid.f90 +++ b/src/grid/discretization_grid.f90 @@ -85,7 +85,7 @@ subroutine discretization_grid_init(restart) call results_closeJobFile else allocate(materialAt_global(0)) ! needed for IntelMPI - endif + end if call MPI_Bcast(grid,3,MPI_INTEGER,0,MPI_COMM_WORLD, ierr) @@ -143,7 +143,7 @@ subroutine discretization_grid_init(restart) call results_addAttribute('size', geomSize,'/geometry') call results_addAttribute('origin',origin, '/geometry') call results_closeJobFile - endif + end if !-------------------------------------------------------------------------------------------------- ! geometry information required by the nonlocal CP model @@ -207,13 +207,13 @@ subroutine readVTI(grid,geomSize,origin,material, & if (.not. fileFormatOk(fileContent(startPos:endPos))) call IO_error(error_ID = 844, ext_msg='file format') headerType = merge('UInt64','UInt32',getXMLValue(fileContent(startPos:endPos),'header_type')=='UInt64') compressed = getXMLValue(fileContent(startPos:endPos),'compressor') == 'vtkZLibDataCompressor' - endif + end if else if (.not. inImage) then if (index(fileContent(startPos:endPos),' Date: Thu, 11 Nov 2021 23:04:00 +0100 Subject: [PATCH 03/24] use fixed PETSc already includes new version of ML proposed in https://gitlab.com/petsc/petsc/-/merge_requests/4549 --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 165cdc68f..5fdd9869c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,8 +52,8 @@ variables: MPI_Intel: "MPI/Intel/19.1.2/IntelMPI/2019" MPI_GNU: "MPI/GNU/10/OpenMPI/4.1.1" # ++++++++++++ PETSc ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - PETSc_Intel: "Libraries/PETSc/3.16.0/Intel-19.1.2-IntelMPI-2019" - PETSc_GNU: "Libraries/PETSc/3.16.0/GNU-10-OpenMPI-4.1.1" + PETSc_Intel: "Libraries/PETSc/3.16.1/Intel-19.1.2-IntelMPI-2019" + PETSc_GNU: "Libraries/PETSc/3.16.1/GNU-10-OpenMPI-4.1.1" # ++++++++++++ MSC Marc +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ MSC: "FEM/MSC/2021.2" IntelMarc: "Compiler/Intel/19.1.2 Libraries/IMKL/2020" From ba31ff726337c770a5063ac4195f3efcd689fcb4 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 11 Nov 2021 23:23:14 +0100 Subject: [PATCH 04/24] do not rely on ML per default --- PRIVATE | 2 +- src/mesh/FEM_utilities.f90 | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/PRIVATE b/PRIVATE index 5a769ec75..532cfadc1 160000 --- a/PRIVATE +++ b/PRIVATE @@ -1 +1 @@ -Subproject commit 5a769ec759d9dacc1866c35c6663cd0001e198c5 +Subproject commit 532cfadc1fda4f0f7f6a993e2bd029f14a50668f diff --git a/src/mesh/FEM_utilities.f90 b/src/mesh/FEM_utilities.f90 index a3856ccaa..d9e985ebd 100644 --- a/src/mesh/FEM_utilities.f90 +++ b/src/mesh/FEM_utilities.f90 @@ -123,9 +123,7 @@ subroutine FEM_utilities_init call PetscOptionsInsertString(PETSC_NULL_OPTIONS,'-mechanical_snes_type newtonls & &-mechanical_snes_linesearch_type cp -mechanical_snes_ksp_ew & &-mechanical_snes_ksp_ew_rtol0 0.01 -mechanical_snes_ksp_ew_rtolmax 0.01 & - &-mechanical_ksp_type fgmres -mechanical_ksp_max_it 25 & - &-mechanical_pc_type ml -mechanical_mg_levels_ksp_type chebyshev & - &-mechanical_mg_levels_pc_type sor -mechanical_pc_ml_nullspace user',ierr) + &-mechanical_ksp_type fgmres -mechanical_ksp_max_it 25', ierr) CHKERRQ(ierr) call PetscOptionsInsertString(PETSC_NULL_OPTIONS,num_mesh%get_asString('PETSc_options',defaultVal=''),ierr) CHKERRQ(ierr) @@ -151,6 +149,7 @@ subroutine utilities_constitutiveResponse(timeinc,P_av,forwardData) PetscErrorCode :: ierr + print'(/,a)', ' ... evaluating constitutive response ......................................' call homogenization_mechanical_response(timeinc,[1,mesh_maxNips],[1,mesh_NcpElems]) ! calculate P field From 8138d3a180b121c81727f2550e42b1121e4d83e1 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 11 Nov 2021 23:59:48 +0100 Subject: [PATCH 05/24] style adjustments --- src/mesh/FEM_utilities.f90 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mesh/FEM_utilities.f90 b/src/mesh/FEM_utilities.f90 index d9e985ebd..fde8831db 100644 --- a/src/mesh/FEM_utilities.f90 +++ b/src/mesh/FEM_utilities.f90 @@ -177,6 +177,7 @@ subroutine utilities_projectBCValues(localVec,section,field,comp,bcPointsIS,BCVa PetscScalar :: BCValue,BCDotValue,timeinc PetscErrorCode :: ierr + call PetscSectionGetFieldComponents(section,field,numComp,ierr); CHKERRQ(ierr) call ISGetSize(bcPointsIS,nBcPoints,ierr); CHKERRQ(ierr) if (nBcPoints > 0) call ISGetIndicesF90(bcPointsIS,bcPoints,ierr) @@ -188,8 +189,8 @@ subroutine utilities_projectBCValues(localVec,section,field,comp,bcPointsIS,BCVa CHKERRQ(ierr) do dof = offset+comp+1, offset+numDof, numComp localArray(dof) = localArray(dof) + BCValue + BCDotValue*timeinc - enddo - enddo + end do + end do call VecRestoreArrayF90(localVec,localArray,ierr); CHKERRQ(ierr) call VecAssemblyBegin(localVec, ierr); CHKERRQ(ierr) call VecAssemblyEnd (localVec, ierr); CHKERRQ(ierr) From e8d2281bb58519a579aa68ba7800a3fa0b3d28a7 Mon Sep 17 00:00:00 2001 From: Test User Date: Fri, 12 Nov 2021 08:43:50 +0100 Subject: [PATCH 06/24] [skip ci] updated version information after successful test of v3.0.0-alpha5-72-g8138d3a18 --- python/damask/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/damask/VERSION b/python/damask/VERSION index bd451b119..7b46fdedc 100644 --- a/python/damask/VERSION +++ b/python/damask/VERSION @@ -1 +1 @@ -v3.0.0-alpha5-64-g8e08af31e +v3.0.0-alpha5-72-g8138d3a18 From 579f3ba5d3a1c76b0b6c85bbc2e6789dbc2f73bd Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Fri, 12 Nov 2021 19:48:53 +0100 Subject: [PATCH 07/24] more robust storage of runtime measurements --- .gitlab-ci.yml | 43 ++++++++++++++++--------------------------- PRIVATE | 2 +- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5fdd9869c..c76504989 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,6 @@ stages: - fortran - performance - deploy - - backup - update_master - distclean - clean @@ -127,7 +126,7 @@ compile_grid_GNU: script: - module load $GNUCompiler $MPI_GNU $PETSc_GNU - cd pytest - - COLUMNS=256 pytest -k 'compile and grid' --basetemp=${TESTROOT}/compile_grid_GNU + - COLUMNS=256 pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_GNU except: - master - release @@ -137,7 +136,7 @@ compile_mesh_GNU: script: - module load $GNUCompiler $MPI_GNU $PETSc_GNU - cd pytest - - COLUMNS=256 pytest -k 'compile and mesh' --basetemp=${TESTROOT}/compile_mesh_GNU + - COLUMNS=256 pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_GNU except: - master - release @@ -147,7 +146,7 @@ compile_Marc: script: - module load $IntelMarc $HDF5Marc $MSC - cd pytest - - COLUMNS=256 pytest -k 'compile and Marc' --basetemp=${TESTROOT}/compile_Marc + - COLUMNS=256 pytest -k 'compile and Marc' --basetemp ${TESTROOT}/compile_Marc except: - master - release @@ -158,8 +157,7 @@ setup_grid: stage: setup script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - - BUILD_DIR=$(mktemp -d) - - cd ${BUILD_DIR} + - cd $(mktemp -d) - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT} - make -j2 all install except: @@ -170,8 +168,7 @@ setup_mesh: stage: setup script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - - BUILD_DIR=$(mktemp -d) - - cd ${BUILD_DIR} + - cd $(mktemp -d) - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT} - make -j2 all install except: @@ -184,7 +181,7 @@ core: script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd pytest - - COLUMNS=256 pytest -k 'not compile' --basetemp=${TESTROOT}/fortran -v + - COLUMNS=256 pytest -k 'not compile' --basetemp ${TESTROOT}/fortran -v except: - master - release @@ -198,16 +195,18 @@ core: # - release ################################################################################################### -SpectralRuntime: +grid_runtime: stage: performance script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd $DAMASKROOT - make clean grid OPTIMIZATION=AGGRESSIVE - - cd $LOCAL_HOME/performance # location of old results - - git checkout . # undo any changes (i.e. run time data from non-development branch) - - cd $DAMASKROOT/PRIVATE/testing - - ./runtime.py --results ${LOCAL_HOME}/performance --damask_root ${DAMASKROOT} --tag ${CI_COMMIT_SHA} + - REPO_DIR=$(mktemp -d) + - git clone -q git@git.damask.mpie.de:damask/performance.git ${REPO_DIR} + - cd ${DAMASKROOT}/PRIVATE/testing/ + - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA} + - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi + resource_group: runtime except: - master - release @@ -217,28 +216,18 @@ source_distribution: stage: deploy script: - cd $(mktemp -d) - - $DAMASKROOT/PRIVATE/releasing/tar.xz/create.sh $DAMASKROOT $CI_COMMIT_SHA + - ${DAMASKROOT}/PRIVATE/releasing/tar.xz/create.sh ${DAMASKROOT} ${CI_COMMIT_SHA} except: - master - release -################################################################################################## -backup_runtime_measurement: - stage: backup - script: - - cd $LOCAL_HOME/performance # location of new runtime results - - git commit -am"${CI_PIPELINE_ID}_${CI_COMMIT_SHA}" - - git push - only: - - development - ################################################################################################## merge_into_master: stage: update_master script: - - cd $DAMASKROOT + - cd ${DAMASKROOT} - export TESTEDREV=$(git describe) # might be detached from development branch - - echo $TESTEDREV > python/damask/VERSION + - echo ${TESTEDREV} > python/damask/VERSION - git add python/damask/VERSION - > git diff-index --quiet HEAD || diff --git a/PRIVATE b/PRIVATE index 532cfadc1..f2a1c7087 160000 --- a/PRIVATE +++ b/PRIVATE @@ -1 +1 @@ -Subproject commit 532cfadc1fda4f0f7f6a993e2bd029f14a50668f +Subproject commit f2a1c70879a892fa3b139e14a2eef199bcde5abf From 97e289deccba870909785037f59b41c04c3cd004 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Fri, 12 Nov 2021 22:11:11 +0100 Subject: [PATCH 08/24] not needed automated push to master skips CI already --- .gitlab-ci.yml | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c76504989..5abb8fbc9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,9 +75,6 @@ checkout: - git checkout $CI_COMMIT_SHA - git submodule update --init - source env/DAMASK.sh - except: - - master - - release ################################################################################################### pytest: @@ -86,18 +83,12 @@ pytest: - cd $DAMASKROOT/python - COLUMNS=256 pytest --basetemp=${TESTROOT}/python -v --cov --cov-report=term - coverage report --fail-under=90 - except: - - master - - release mypy: stage: python script: - cd $DAMASKROOT/python - mypy -m damask - except: - - master - - release ################################################################################################### @@ -107,9 +98,6 @@ compile_grid_Intel: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd pytest - COLUMNS=256 pytest -k 'compile and grid' --basetemp=${TESTROOT}/compile_grid_Intel - except: - - master - - release compile_mesh_Intel: stage: compile @@ -117,9 +105,6 @@ compile_mesh_Intel: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd pytest - COLUMNS=256 pytest -k 'compile and mesh' --basetemp=${TESTROOT}/compile_mesh_Intel - except: - - master - - release compile_grid_GNU: stage: compile @@ -127,9 +112,6 @@ compile_grid_GNU: - module load $GNUCompiler $MPI_GNU $PETSc_GNU - cd pytest - COLUMNS=256 pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_GNU - except: - - master - - release compile_mesh_GNU: stage: compile @@ -137,9 +119,6 @@ compile_mesh_GNU: - module load $GNUCompiler $MPI_GNU $PETSc_GNU - cd pytest - COLUMNS=256 pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_GNU - except: - - master - - release compile_Marc: stage: compile @@ -147,9 +126,6 @@ compile_Marc: - module load $IntelMarc $HDF5Marc $MSC - cd pytest - COLUMNS=256 pytest -k 'compile and Marc' --basetemp ${TESTROOT}/compile_Marc - except: - - master - - release ################################################################################################### @@ -160,9 +136,6 @@ setup_grid: - cd $(mktemp -d) - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT} - make -j2 all install - except: - - master - - release setup_mesh: stage: setup @@ -171,9 +144,6 @@ setup_mesh: - cd $(mktemp -d) - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT} - make -j2 all install - except: - - master - - release ################################################################################################### core: @@ -182,17 +152,11 @@ core: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd pytest - COLUMNS=256 pytest -k 'not compile' --basetemp ${TESTROOT}/fortran -v - except: - - master - - release # Needs closer look # Phenopowerlaw_singleSlip: # stage: fortran # script: Phenopowerlaw_singleSlip/test.py -# except: -# - master -# - release ################################################################################################### grid_runtime: @@ -207,9 +171,6 @@ grid_runtime: - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA} - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi resource_group: runtime - except: - - master - - release ################################################################################################### source_distribution: @@ -217,9 +178,6 @@ source_distribution: script: - cd $(mktemp -d) - ${DAMASKROOT}/PRIVATE/releasing/tar.xz/create.sh ${DAMASKROOT} ${CI_COMMIT_SHA} - except: - - master - - release ################################################################################################## merge_into_master: @@ -251,9 +209,6 @@ remove_data: script: - rm -rf $LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID - sed -i "/$CI_PIPELINE_ID/d" $LOCAL_HOME/GitLabCI.queue # in case pipeline was manually (web GUI) restarted and releaseLock was performed already - except: - - master - - release ################################################################################################### remove_lock: @@ -262,6 +217,3 @@ remove_lock: - echo "Removing lock of pipeline $CI_PIPELINE_ID" when: always script: sed -i "/$CI_PIPELINE_ID/d" $LOCAL_HOME/GitLabCI.queue - except: - - master - - release From f112d11985065811b1d9a14b11c32060ed4269d5 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 10:57:58 +0100 Subject: [PATCH 09/24] new locking mechanism only runtime test needs to run exclusively. lock per job, not per pipeline. should be more stable and allow more concurrency needs python script for job management on gitlab-runner --- .gitlab-ci.yml | 75 +++++++++++++++++++------------------------------- 1 file changed, 29 insertions(+), 46 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5abb8fbc9..4d96b6693 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,26 +8,20 @@ stages: - performance - deploy - update_master - - distclean - - clean + ################################################################################################### before_script: - - if [ $(awk "/$CI_PIPELINE_ID/{print NR}" $LOCAL_HOME/GitLabCI.queue)x == 'x' ]; - then echo $CI_PIPELINE_ID >> $LOCAL_HOME/GitLabCI.queue; - fi - - while [ $(awk "/$CI_PIPELINE_ID/{print NR}" $LOCAL_HOME/GitLabCI.queue) != 1 ]; - do sleep 5m; - echo -e "Currently queued pipelines:\n$(cat $LOCAL_HOME/GitLabCI.queue)\n"; - done + - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking - source $DAMASKROOT/env/DAMASK.sh - cd $DAMASKROOT/PRIVATE/testing - echo Job start:" $(date)" -################################################################################################### after_script: + - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --unregister - echo Job end:" $(date)" + ################################################################################################### variables: # =============================================================================================== @@ -58,36 +52,35 @@ variables: IntelMarc: "Compiler/Intel/19.1.2 Libraries/IMKL/2020" HDF5Marc: "HDF5/1.12.1/Intel-19.1.2" + ################################################################################################### checkout: stage: prepare before_script: - - echo $CI_PIPELINE_ID >> $LOCAL_HOME/GitLabCI.queue - - while [ $(awk "/$CI_PIPELINE_ID/{print NR}" $LOCAL_HOME/GitLabCI.queue) != 1 ]; - do sleep 5m; - echo -e "Currently queued pipelines:\n$(cat $LOCAL_HOME/GitLabCI.queue)\n"; - done + - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking + - echo Job start:" $(date)" script: - - mkdir -p $DAMASKROOT - - mkdir -p $TESTROOT - - cd $DAMASKROOT + - mkdir -p ${DAMASKROOT} + - mkdir -p ${TESTROOT} + - cd ${DAMASKROOT} - git clone -q git@git.damask.mpie.de:damask/DAMASK.git . - - git checkout $CI_COMMIT_SHA + - git checkout ${CI_COMMIT_SHA} - git submodule update --init - source env/DAMASK.sh + ################################################################################################### pytest: stage: python script: - - cd $DAMASKROOT/python - - COLUMNS=256 pytest --basetemp=${TESTROOT}/python -v --cov --cov-report=term + - cd ${DAMASKROOT}/python + - pytest --basetemp ${TESTROOT}/python -v --cov --cov-report=term - coverage report --fail-under=90 mypy: stage: python script: - - cd $DAMASKROOT/python + - cd ${DAMASKROOT}/python - mypy -m damask @@ -97,35 +90,35 @@ compile_grid_Intel: script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd pytest - - COLUMNS=256 pytest -k 'compile and grid' --basetemp=${TESTROOT}/compile_grid_Intel + - pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_Intel compile_mesh_Intel: stage: compile script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd pytest - - COLUMNS=256 pytest -k 'compile and mesh' --basetemp=${TESTROOT}/compile_mesh_Intel + - pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_Intel compile_grid_GNU: stage: compile script: - module load $GNUCompiler $MPI_GNU $PETSc_GNU - cd pytest - - COLUMNS=256 pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_GNU + - pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_GNU compile_mesh_GNU: stage: compile script: - module load $GNUCompiler $MPI_GNU $PETSc_GNU - cd pytest - - COLUMNS=256 pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_GNU + - pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_GNU compile_Marc: stage: compile script: - module load $IntelMarc $HDF5Marc $MSC - cd pytest - - COLUMNS=256 pytest -k 'compile and Marc' --basetemp ${TESTROOT}/compile_Marc + - pytest -k 'compile and Marc' --basetemp ${TESTROOT}/compile_Marc ################################################################################################### @@ -145,19 +138,21 @@ setup_mesh: - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT} - make -j2 all install + ################################################################################################### core: stage: fortran script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd pytest - - COLUMNS=256 pytest -k 'not compile' --basetemp ${TESTROOT}/fortran -v + - pytest -k 'not compile' --basetemp ${TESTROOT}/fortran -v # Needs closer look # Phenopowerlaw_singleSlip: # stage: fortran # script: Phenopowerlaw_singleSlip/test.py + ################################################################################################### grid_runtime: stage: performance @@ -171,6 +166,9 @@ grid_runtime: - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA} - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi resource_group: runtime + before_script: + - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --blocking + ################################################################################################### source_distribution: @@ -179,7 +177,8 @@ source_distribution: - cd $(mktemp -d) - ${DAMASKROOT}/PRIVATE/releasing/tar.xz/create.sh ${DAMASKROOT} ${CI_COMMIT_SHA} -################################################################################################## + +################################################################################################### merge_into_master: stage: update_master script: @@ -192,6 +191,7 @@ merge_into_master: git commit -m "[skip ci] updated version information after successful test of $TESTEDREV" - export UPDATEDREV=$(git describe) # tested state + 1 commit - git checkout master + - git pull - git merge $UPDATEDREV -s recursive -X ours # conflicts occur only for inconsistent state - git push origin master # master is now tested version and has updated VERSION file - git checkout development @@ -200,20 +200,3 @@ merge_into_master: - git push origin development # development is unchanged (as master is based on it) but has updated VERSION file only: - development - -################################################################################################### -remove_data: - stage: distclean - before_script: - - echo "Removing data and lock of pipeline $CI_PIPELINE_ID" - script: - - rm -rf $LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID - - sed -i "/$CI_PIPELINE_ID/d" $LOCAL_HOME/GitLabCI.queue # in case pipeline was manually (web GUI) restarted and releaseLock was performed already - -################################################################################################### -remove_lock: - stage: clean - before_script: - - echo "Removing lock of pipeline $CI_PIPELINE_ID" - when: always - script: sed -i "/$CI_PIPELINE_ID/d" $LOCAL_HOME/GitLabCI.queue From 2682a26497cfb6a353a88f9fccf6cf3404aa3fe7 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 11:32:55 +0100 Subject: [PATCH 10/24] preparing for pipeline-wide location of executables --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d96b6693..ee981b2f6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,7 @@ stages: before_script: - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking - source $DAMASKROOT/env/DAMASK.sh + - export PATH=${BINDIR}:$PATH - cd $DAMASKROOT/PRIVATE/testing - echo Job start:" $(date)" @@ -34,6 +35,7 @@ variables: # =============================================================================================== DAMASKROOT: "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/DAMASK" TESTROOT: "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/tests" + BINDIR: "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/bin" # =============================================================================================== # Names of module files to load @@ -62,11 +64,11 @@ checkout: script: - mkdir -p ${DAMASKROOT} - mkdir -p ${TESTROOT} + - mkdir -p ${BINDIR} - cd ${DAMASKROOT} - git clone -q git@git.damask.mpie.de:damask/DAMASK.git . - git checkout ${CI_COMMIT_SHA} - git submodule update --init - - source env/DAMASK.sh ################################################################################################### From 45af45615a51f371b947f158868555bef3dc16e8 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 11:50:05 +0100 Subject: [PATCH 11/24] build to new bin location --- .gitlab-ci.yml | 4 ++-- PRIVATE | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ee981b2f6..5e120a660 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -129,7 +129,7 @@ setup_grid: script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd $(mktemp -d) - - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT} + - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${DAMASKROOT} - make -j2 all install setup_mesh: @@ -137,7 +137,7 @@ setup_mesh: script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd $(mktemp -d) - - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT} + - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${DAMASKROOT} - make -j2 all install diff --git a/PRIVATE b/PRIVATE index f2a1c7087..f730bcb8d 160000 --- a/PRIVATE +++ b/PRIVATE @@ -1 +1 @@ -Subproject commit f2a1c70879a892fa3b139e14a2eef199bcde5abf +Subproject commit f730bcb8ddd7224011e70c57d0a5c03068532d2d From c75ac5fe6ca2581c031933c68f08f9494b85ad24 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 14:03:06 +0100 Subject: [PATCH 12/24] let gitlab-runner handle the submodule --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e120a660..5c2afbd1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,7 @@ variables: # =============================================================================================== # GitLab Settings # =============================================================================================== - GIT_SUBMODULE_STRATEGY: none + GIT_SUBMODULE_STRATEGY: normal # =============================================================================================== # Shortcut names From 352a168621f50ddce18a2cb5edb7506705412c9a Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 14:09:12 +0100 Subject: [PATCH 13/24] new style (https://docs.gitlab.com/ee/ci/yaml) --- .gitlab-ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5c2afbd1d..555a212c6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,16 +11,16 @@ stages: ################################################################################################### -before_script: - - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking - - source $DAMASKROOT/env/DAMASK.sh - - export PATH=${BINDIR}:$PATH - - cd $DAMASKROOT/PRIVATE/testing - - echo Job start:" $(date)" - -after_script: - - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --unregister - - echo Job end:" $(date)" +default: + before_script: + - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking + - source $DAMASKROOT/env/DAMASK.sh + - export PATH=${BINDIR}:$PATH + - cd $DAMASKROOT/PRIVATE/testing + - echo Job start:" $(date)" + after_script: + - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --unregister + - echo Job end:" $(date)" ################################################################################################### From 06edd39a93ce84dc46385be14b5e81da36ed35f4 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 16:33:31 +0100 Subject: [PATCH 14/24] still need clean stage after_script does not run for failed/cancelled scripts. --- .gitlab-ci.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 555a212c6..f391c0e0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,18 +8,19 @@ stages: - performance - deploy - update_master + - clean ################################################################################################### default: before_script: - - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking + - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --nonblocking - source $DAMASKROOT/env/DAMASK.sh - export PATH=${BINDIR}:$PATH - cd $DAMASKROOT/PRIVATE/testing - echo Job start:" $(date)" after_script: - - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --unregister + - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --unregister - echo Job end:" $(date)" @@ -59,7 +60,7 @@ variables: checkout: stage: prepare before_script: - - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking + - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --nonblocking - echo Job start:" $(date)" script: - mkdir -p ${DAMASKROOT} @@ -169,7 +170,7 @@ grid_runtime: - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi resource_group: runtime before_script: - - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --blocking + - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --blocking ################################################################################################### @@ -202,3 +203,15 @@ merge_into_master: - git push origin development # development is unchanged (as master is based on it) but has updated VERSION file only: - development + + +################################################################################################### +clean_pool: + stage: clean + script: + - ${LOCAL_HOME}/bin/manage_pool -p ${CI_PIPELINE_ID} --clean + when: always + before_script: + - echo Job start:" $(date)" + after_script: + - echo Job end:" $(date)" From 51c262056777afd29c7168cfebd2e7300f58fd01 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 18:03:40 +0100 Subject: [PATCH 15/24] use (partly) repo cloned by gitlab-runner --- .gitlab-ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f391c0e0b..146b13d0c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -76,14 +76,15 @@ checkout: pytest: stage: python script: - - cd ${DAMASKROOT}/python + - PYTHONPATH=${CI_PROJECT_DIR}/python + - cd ${CI_PROJECT_DIR}/python - pytest --basetemp ${TESTROOT}/python -v --cov --cov-report=term - coverage report --fail-under=90 mypy: stage: python script: - - cd ${DAMASKROOT}/python + - cd ${CI_PROJECT_DIR}/python - mypy -m damask @@ -130,7 +131,7 @@ setup_grid: script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd $(mktemp -d) - - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${DAMASKROOT} + - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR} - make -j2 all install setup_mesh: @@ -138,7 +139,7 @@ setup_mesh: script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd $(mktemp -d) - - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${DAMASKROOT} + - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR} - make -j2 all install @@ -178,7 +179,7 @@ source_distribution: stage: deploy script: - cd $(mktemp -d) - - ${DAMASKROOT}/PRIVATE/releasing/tar.xz/create.sh ${DAMASKROOT} ${CI_COMMIT_SHA} + - ${CI_PROJECT_DIR}/PRIVATE/releasing/tar.xz/create.sh ${CI_PROJECT_DIR} ${CI_COMMIT_SHA} ################################################################################################### From 57817e78ae0a745cd6984a9abe6f544716d9cc90 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 18:39:19 +0100 Subject: [PATCH 16/24] PYTHONPATH was not set --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 146b13d0c..0ae4c605c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -172,6 +172,7 @@ grid_runtime: resource_group: runtime before_script: - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --blocking + - source $DAMASKROOT/env/DAMASK.sh ################################################################################################### From f97c9c7e8c6d941e357fd85a5576089b60013101 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 19:13:10 +0100 Subject: [PATCH 17/24] don't rely on Makefile --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ae4c605c..c95a16efc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -162,8 +162,9 @@ grid_runtime: stage: performance script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - - cd $DAMASKROOT - - make clean grid OPTIMIZATION=AGGRESSIVE + - cd $(mktemp -d) + - cmake -DOPTIMIZATION=AGGRESSIVE -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR} + - make -j2 all install - REPO_DIR=$(mktemp -d) - git clone -q git@git.damask.mpie.de:damask/performance.git ${REPO_DIR} - cd ${DAMASKROOT}/PRIVATE/testing/ From 9430ca9e6ec20f68df7968692c6f6755af344799 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 19:21:23 +0100 Subject: [PATCH 18/24] simplified --- .gitlab-ci.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c95a16efc..0b7c7d0d2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ default: before_script: - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --nonblocking - source $DAMASKROOT/env/DAMASK.sh - - export PATH=${BINDIR}:$PATH + - export PATH=${TESTROOT}/bin:$PATH - cd $DAMASKROOT/PRIVATE/testing - echo Job start:" $(date)" after_script: @@ -34,9 +34,8 @@ variables: # =============================================================================================== # Shortcut names # =============================================================================================== + TESTROOT: "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID" DAMASKROOT: "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/DAMASK" - TESTROOT: "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/tests" - BINDIR: "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/bin" # =============================================================================================== # Names of module files to load @@ -64,8 +63,6 @@ checkout: - echo Job start:" $(date)" script: - mkdir -p ${DAMASKROOT} - - mkdir -p ${TESTROOT} - - mkdir -p ${BINDIR} - cd ${DAMASKROOT} - git clone -q git@git.damask.mpie.de:damask/DAMASK.git . - git checkout ${CI_COMMIT_SHA} @@ -131,7 +128,7 @@ setup_grid: script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd $(mktemp -d) - - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR} + - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${TESTROOT} ${CI_PROJECT_DIR} - make -j2 all install setup_mesh: @@ -139,7 +136,7 @@ setup_mesh: script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd $(mktemp -d) - - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR} + - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${TESTROOT} ${CI_PROJECT_DIR} - make -j2 all install @@ -163,7 +160,7 @@ grid_runtime: script: - module load $IntelCompiler $MPI_Intel $PETSc_Intel - cd $(mktemp -d) - - cmake -DOPTIMIZATION=AGGRESSIVE -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR} + - cmake -DOPTIMIZATION=AGGRESSIVE -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${TESTROOT} ${CI_PROJECT_DIR} - make -j2 all install - REPO_DIR=$(mktemp -d) - git clone -q git@git.damask.mpie.de:damask/performance.git ${REPO_DIR} @@ -174,7 +171,8 @@ grid_runtime: before_script: - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --blocking - source $DAMASKROOT/env/DAMASK.sh - + - export PATH=${TESTROOT}/bin:$PATH + - echo Job start:" $(date)" ################################################################################################### source_distribution: From 4396d605ec08317b043436c54647a76fb111e623 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 19:37:40 +0100 Subject: [PATCH 19/24] test folder, not module --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b7c7d0d2..adaceb612 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -82,7 +82,7 @@ mypy: stage: python script: - cd ${CI_PROJECT_DIR}/python - - mypy -m damask + - mypy damask ################################################################################################### From 74921ebe190212d1e74fb7f9ca4d83715ee73c7d Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 21:28:42 +0100 Subject: [PATCH 20/24] not needed, only delays runtime test resource management is done with 'manage pool' script --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index adaceb612..5e95cd23b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -167,7 +167,6 @@ grid_runtime: - cd ${DAMASKROOT}/PRIVATE/testing/ - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA} - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi - resource_group: runtime before_script: - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --blocking - source $DAMASKROOT/env/DAMASK.sh From 4e57193a0240a2cbadd57d356685c6a3e6e03f3e Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sat, 13 Nov 2021 22:58:56 +0100 Subject: [PATCH 21/24] simplified use GitLab CI API to remove finished jobs, no user intervention needed after cancelling --- .gitlab-ci.yml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e95cd23b..3d21f3184 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,19 +8,17 @@ stages: - performance - deploy - update_master - - clean ################################################################################################### default: before_script: - - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --nonblocking + - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID} - source $DAMASKROOT/env/DAMASK.sh - export PATH=${TESTROOT}/bin:$PATH - cd $DAMASKROOT/PRIVATE/testing - echo Job start:" $(date)" after_script: - - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --unregister - echo Job end:" $(date)" @@ -59,7 +57,7 @@ variables: checkout: stage: prepare before_script: - - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --nonblocking + - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID} - echo Job start:" $(date)" script: - mkdir -p ${DAMASKROOT} @@ -168,7 +166,7 @@ grid_runtime: - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA} - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi before_script: - - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --blocking + - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID} --blocking - source $DAMASKROOT/env/DAMASK.sh - export PATH=${TESTROOT}/bin:$PATH - echo Job start:" $(date)" @@ -203,15 +201,3 @@ merge_into_master: - git push origin development # development is unchanged (as master is based on it) but has updated VERSION file only: - development - - -################################################################################################### -clean_pool: - stage: clean - script: - - ${LOCAL_HOME}/bin/manage_pool -p ${CI_PIPELINE_ID} --clean - when: always - before_script: - - echo Job start:" $(date)" - after_script: - - echo Job end:" $(date)" From 4a0fc7b150e8fc970c024cbaf7e73a660e27bcb6 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sun, 14 Nov 2021 09:45:01 +0100 Subject: [PATCH 22/24] ensure to be in correct repository --- .gitlab-ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3d21f3184..5f4ef7818 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -160,10 +160,9 @@ grid_runtime: - cd $(mktemp -d) - cmake -DOPTIMIZATION=AGGRESSIVE -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${TESTROOT} ${CI_PROJECT_DIR} - make -j2 all install - - REPO_DIR=$(mktemp -d) - - git clone -q git@git.damask.mpie.de:damask/performance.git ${REPO_DIR} - - cd ${DAMASKROOT}/PRIVATE/testing/ - - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA} + - cd $(mktemp -d) + - git clone -q git@git.damask.mpie.de:damask/performance.git . + - ${DAMASKROOT}/PRIVATE/testing/runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA} - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi before_script: - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID} --blocking From bb665504598f5e10bdec77cc1f8753cf62c56a2b Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sun, 14 Nov 2021 18:59:20 +0100 Subject: [PATCH 23/24] don't use undefined variable --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5f4ef7818..9c8e16073 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -162,7 +162,7 @@ grid_runtime: - make -j2 all install - cd $(mktemp -d) - git clone -q git@git.damask.mpie.de:damask/performance.git . - - ${DAMASKROOT}/PRIVATE/testing/runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA} + - ${DAMASKROOT}/PRIVATE/testing/runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir . --tag ${CI_COMMIT_SHA} - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi before_script: - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID} --blocking From a87f414146f974df19c140ea52b800719ac3a36c Mon Sep 17 00:00:00 2001 From: Test User Date: Sun, 14 Nov 2021 20:46:57 +0100 Subject: [PATCH 24/24] [skip ci] updated version information after successful test of v3.0.0-alpha5-90-gbb6655045 --- python/damask/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/damask/VERSION b/python/damask/VERSION index 7b46fdedc..f3e930998 100644 --- a/python/damask/VERSION +++ b/python/damask/VERSION @@ -1 +1 @@ -v3.0.0-alpha5-72-g8138d3a18 +v3.0.0-alpha5-90-gbb6655045