Merge remote-tracking branch 'origin/development' into Marc-spelling

.gitlab-ci.yml

@@ -7,40 +7,33 @@ stages:
   - fortran
   - performance
   - deploy
-  - backup
   - update_master
-  - distclean
+
-  - clean
 
 ###################################################################################################
+default:
   before_script:
-  - if [ $(awk "/$CI_PIPELINE_ID/{print NR}" $LOCAL_HOME/GitLabCI.queue)x == 'x' ];
+    - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID}
-      then echo $CI_PIPELINE_ID >> $LOCAL_HOME/GitLabCI.queue;
-    fi
-  - while [ $(awk "/$CI_PIPELINE_ID/{print NR}" $LOCAL_HOME/GitLabCI.queue) != 1 ];
-      do sleep 5m;
-      echo -e "Currently queued pipelines:\n$(cat $LOCAL_HOME/GitLabCI.queue)\n";
-    done
     - source $DAMASKROOT/env/DAMASK.sh
+    - export PATH=${TESTROOT}/bin:$PATH
     - cd $DAMASKROOT/PRIVATE/testing
     - echo Job start:" $(date)"
-
-###################################################################################################
   after_script:
     - echo Job end:" $(date)"
 
+
 ###################################################################################################
 variables:
   # ===============================================================================================
   # GitLab Settings
   # ===============================================================================================
-  GIT_SUBMODULE_STRATEGY: none
+  GIT_SUBMODULE_STRATEGY: normal
 
   # ===============================================================================================
   # Shortcut names
   # ===============================================================================================
+  TESTROOT:   "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID"
   DAMASKROOT: "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/DAMASK"
-  TESTROOT:   "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/tests"
 
   # ===============================================================================================
   # Names of module files to load
@@ -59,46 +52,35 @@ variables:
   IntelMarc:      "Compiler/Intel/19.1.2 Libraries/IMKL/2020"
   HDF5Marc:       "HDF5/1.12.1/Intel-19.1.2"
 
+
 ###################################################################################################
 checkout:
   stage: prepare
   before_script:
-    - echo $CI_PIPELINE_ID >> $LOCAL_HOME/GitLabCI.queue
+    - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID}
-    - while [ $(awk "/$CI_PIPELINE_ID/{print NR}" $LOCAL_HOME/GitLabCI.queue) != 1 ];
+    - echo Job start:" $(date)"
-        do sleep 5m;
-        echo -e "Currently queued pipelines:\n$(cat $LOCAL_HOME/GitLabCI.queue)\n";
-      done
   script:
-    - mkdir -p $DAMASKROOT
+    - mkdir -p ${DAMASKROOT}
-    - mkdir -p $TESTROOT
+    - cd ${DAMASKROOT}
-    - cd $DAMASKROOT
     - git clone -q git@git.damask.mpie.de:damask/DAMASK.git .
-    - git checkout $CI_COMMIT_SHA
+    - git checkout ${CI_COMMIT_SHA}
     - git submodule update --init
-    - source env/DAMASK.sh
+
-  except:
-    - master
-    - release
 
 ###################################################################################################
 pytest:
   stage: python
   script:
-    - cd $DAMASKROOT/python
+    - PYTHONPATH=${CI_PROJECT_DIR}/python
-    - COLUMNS=256 pytest --basetemp=${TESTROOT}/python -v --cov --cov-report=term
+    - cd ${CI_PROJECT_DIR}/python
+    - pytest --basetemp ${TESTROOT}/python -v --cov --cov-report=term
     - coverage report --fail-under=90
-  except:
-    - master
-    - release
 
 mypy:
   stage: python
   script:
-    - cd $DAMASKROOT/python
+    - cd ${CI_PROJECT_DIR}/python
-    - mypy -m damask
+    - mypy damask
-  except:
-    - master
-    - release
 
 
 ###################################################################################################
@@ -107,50 +89,35 @@ compile_grid_Intel:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and grid' --basetemp=${TESTROOT}/compile_grid_Intel
+    - pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_Intel
-  except:
-    - master
-    - release
 
 compile_mesh_Intel:
   stage: compile
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and mesh' --basetemp=${TESTROOT}/compile_mesh_Intel
+    - pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_Intel
-  except:
-    - master
-    - release
 
 compile_grid_GNU:
   stage: compile
   script:
     - module load $GNUCompiler $MPI_GNU $PETSc_GNU
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and grid' --basetemp=${TESTROOT}/compile_grid_GNU
+    - pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_GNU
-  except:
-    - master
-    - release
 
 compile_mesh_GNU:
   stage: compile
   script:
     - module load $GNUCompiler $MPI_GNU $PETSc_GNU
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and mesh' --basetemp=${TESTROOT}/compile_mesh_GNU
+    - pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_GNU
-  except:
-    - master
-    - release
 
 compile_Marc:
   stage: compile
   script:
     - module load $IntelMarc $HDF5Marc $MSC
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and Marc' --basetemp=${TESTROOT}/compile_Marc
+    - pytest -k 'compile and Marc' --basetemp ${TESTROOT}/compile_Marc
-  except:
-    - master
-    - release
 
 
 ###################################################################################################
@@ -158,25 +125,18 @@ setup_grid:
   stage: setup
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
-    - BUILD_DIR=$(mktemp -d)
+    - cd $(mktemp -d)
-    - cd ${BUILD_DIR}
+    - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${TESTROOT} ${CI_PROJECT_DIR}
-    - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT}
     - make -j2 all install
-  except:
-    - master
-    - release
 
 setup_mesh:
   stage: setup
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
-    - BUILD_DIR=$(mktemp -d)
+    - cd $(mktemp -d)
-    - cd ${BUILD_DIR}
+    - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${TESTROOT} ${CI_PROJECT_DIR}
-    - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT}
     - make -j2 all install
-  except:
+
-    - master
-    - release
 
 ###################################################################################################
 core:
@@ -184,67 +144,55 @@ core:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd pytest
-    - COLUMNS=256 pytest -k 'not compile' --basetemp=${TESTROOT}/fortran -v
+    - pytest -k 'not compile' --basetemp ${TESTROOT}/fortran -v
-  except:
-    - master
-    - release
 
 # Needs closer look
 # Phenopowerlaw_singleSlip:
 #   stage: fortran
 #   script: Phenopowerlaw_singleSlip/test.py
-#   except:
+
-#     - master
-#     - release
 
 ###################################################################################################
-SpectralRuntime:
+grid_runtime:
   stage: performance
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
-    - cd $DAMASKROOT
+    - cd $(mktemp -d)
-    - make clean grid OPTIMIZATION=AGGRESSIVE
+    - cmake -DOPTIMIZATION=AGGRESSIVE -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${TESTROOT} ${CI_PROJECT_DIR}
-    - cd $LOCAL_HOME/performance                                                                    # location of old results
+    - make -j2 all install
-    - git checkout .                                                                                # undo any changes (i.e. run time data from non-development branch)
+    - REPO_DIR=$(mktemp -d)
-    - cd $DAMASKROOT/PRIVATE/testing
+    - git clone -q git@git.damask.mpie.de:damask/performance.git ${REPO_DIR}
-    - ./runtime.py --results ${LOCAL_HOME}/performance --damask_root ${DAMASKROOT} --tag ${CI_COMMIT_SHA}
+    - cd ${DAMASKROOT}/PRIVATE/testing/
-  except:
+    - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA}
-    - master
+    - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi
-    - release
+  before_script:
+    - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID} --blocking
+    - source $DAMASKROOT/env/DAMASK.sh
+    - export PATH=${TESTROOT}/bin:$PATH
+    - echo Job start:" $(date)"
 
 ###################################################################################################
 source_distribution:
   stage: deploy
   script:
     - cd $(mktemp -d)
-    - $DAMASKROOT/PRIVATE/releasing/tar.xz/create.sh $DAMASKROOT $CI_COMMIT_SHA
+    - ${CI_PROJECT_DIR}/PRIVATE/releasing/tar.xz/create.sh ${CI_PROJECT_DIR} ${CI_COMMIT_SHA}
-  except:
-    - master
-    - release
 
-##################################################################################################
-backup_runtime_measurement:
-  stage: backup
-  script:
-    - cd $LOCAL_HOME/performance                                                                    # location of new runtime results
-    - git commit -am"${CI_PIPELINE_ID}_${CI_COMMIT_SHA}"
-    - git push
-  only:
-    - development
 
-##################################################################################################
+###################################################################################################
 merge_into_master:
   stage: update_master
   script:
-    - cd $DAMASKROOT
+    - cd ${DAMASKROOT}
     - export TESTEDREV=$(git describe)                                                              # might be detached from development branch
-    - echo $TESTEDREV > python/damask/VERSION
+    - echo ${TESTEDREV} > python/damask/VERSION
     - git add python/damask/VERSION
     - >
         git diff-index --quiet HEAD ||
         git commit -m "[skip ci] updated version information after successful test of $TESTEDREV"
     - export UPDATEDREV=$(git describe)                                                             # tested state + 1 commit
     - git checkout master
+    - git pull
     - git merge $UPDATEDREV -s recursive -X ours                                                    # conflicts occur only for inconsistent state
     - git push origin master                                                                        # master is now tested version and has updated VERSION file
     - git checkout development
@@ -253,26 +201,3 @@ merge_into_master:
     - git push origin development                                                                   # development is unchanged (as master is based on it) but has updated VERSION file
   only:
     - development
-
-###################################################################################################
-remove_data:
-  stage: distclean
-  before_script:
-    - echo "Removing data and lock of pipeline $CI_PIPELINE_ID"
-  script:
-    - rm -rf $LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID
-    - sed -i "/$CI_PIPELINE_ID/d" $LOCAL_HOME/GitLabCI.queue                                        # in case pipeline was manually (web GUI) restarted and releaseLock was performed already
-  except:
-    - master
-    - release
-
-###################################################################################################
-remove_lock:
-  stage: clean
-  before_script:
-    - echo "Removing lock of pipeline $CI_PIPELINE_ID"
-  when: always
-  script: sed -i "/$CI_PIPELINE_ID/d" $LOCAL_HOME/GitLabCI.queue
-  except:
-    - master
-    - release

PRIVATE

@@ -1 +1 @@
-Subproject commit 532cfadc1fda4f0f7f6a993e2bd029f14a50668f
+Subproject commit f730bcb8ddd7224011e70c57d0a5c03068532d2d