From 579f3ba5d3a1c76b0b6c85bbc2e6789dbc2f73bd Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Fri, 12 Nov 2021 19:48:53 +0100
Subject: [PATCH 01/15] more robust storage of runtime measurements

---
 .gitlab-ci.yml | 43 ++++++++++++++++---------------------------
 PRIVATE        |  2 +-
 2 files changed, 17 insertions(+), 28 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5fdd9869c..c76504989 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -7,7 +7,6 @@ stages:
   - fortran
   - performance
   - deploy
-  - backup
   - update_master
   - distclean
   - clean
@@ -127,7 +126,7 @@ compile_grid_GNU:
   script:
     - module load $GNUCompiler $MPI_GNU $PETSc_GNU
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and grid' --basetemp=${TESTROOT}/compile_grid_GNU
+    - COLUMNS=256 pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_GNU
   except:
     - master
     - release
@@ -137,7 +136,7 @@ compile_mesh_GNU:
   script:
     - module load $GNUCompiler $MPI_GNU $PETSc_GNU
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and mesh' --basetemp=${TESTROOT}/compile_mesh_GNU
+    - COLUMNS=256 pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_GNU
   except:
     - master
     - release
@@ -147,7 +146,7 @@ compile_Marc:
   script:
     - module load $IntelMarc $HDF5Marc $MSC
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and Marc' --basetemp=${TESTROOT}/compile_Marc
+    - COLUMNS=256 pytest -k 'compile and Marc' --basetemp ${TESTROOT}/compile_Marc
   except:
     - master
     - release
@@ -158,8 +157,7 @@ setup_grid:
   stage: setup
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
-    - BUILD_DIR=$(mktemp -d)
-    - cd ${BUILD_DIR}
+    - cd $(mktemp -d)
     - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT}
     - make -j2 all install
   except:
@@ -170,8 +168,7 @@ setup_mesh:
   stage: setup
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
-    - BUILD_DIR=$(mktemp -d)
-    - cd ${BUILD_DIR}
+    - cd $(mktemp -d)
     - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT}
     - make -j2 all install
   except:
@@ -184,7 +181,7 @@ core:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd pytest
-    - COLUMNS=256 pytest -k 'not compile' --basetemp=${TESTROOT}/fortran -v
+    - COLUMNS=256 pytest -k 'not compile' --basetemp ${TESTROOT}/fortran -v
   except:
     - master
     - release
@@ -198,16 +195,18 @@ core:
 #     - release
 
 ###################################################################################################
-SpectralRuntime:
+grid_runtime:
   stage: performance
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd $DAMASKROOT
     - make clean grid OPTIMIZATION=AGGRESSIVE
-    - cd $LOCAL_HOME/performance                                                                    # location of old results
-    - git checkout .                                                                                # undo any changes (i.e. run time data from non-development branch)
-    - cd $DAMASKROOT/PRIVATE/testing
-    - ./runtime.py --results ${LOCAL_HOME}/performance --damask_root ${DAMASKROOT} --tag ${CI_COMMIT_SHA}
+    - REPO_DIR=$(mktemp -d)
+    - git clone -q git@git.damask.mpie.de:damask/performance.git ${REPO_DIR}
+    - cd ${DAMASKROOT}/PRIVATE/testing/
+    - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA}
+    - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi
+  resource_group: runtime
   except:
     - master
     - release
@@ -217,28 +216,18 @@ source_distribution:
   stage: deploy
   script:
     - cd $(mktemp -d)
-    - $DAMASKROOT/PRIVATE/releasing/tar.xz/create.sh $DAMASKROOT $CI_COMMIT_SHA
+    - ${DAMASKROOT}/PRIVATE/releasing/tar.xz/create.sh ${DAMASKROOT} ${CI_COMMIT_SHA}
   except:
     - master
     - release
 
-##################################################################################################
-backup_runtime_measurement:
-  stage: backup
-  script:
-    - cd $LOCAL_HOME/performance                                                                    # location of new runtime results
-    - git commit -am"${CI_PIPELINE_ID}_${CI_COMMIT_SHA}"
-    - git push
-  only:
-    - development
-
 ##################################################################################################
 merge_into_master:
   stage: update_master
   script:
-    - cd $DAMASKROOT
+    - cd ${DAMASKROOT}
     - export TESTEDREV=$(git describe)                                                              # might be detached from development branch
-    - echo $TESTEDREV > python/damask/VERSION
+    - echo ${TESTEDREV} > python/damask/VERSION
     - git add python/damask/VERSION
     - >
         git diff-index --quiet HEAD ||
diff --git a/PRIVATE b/PRIVATE
index 532cfadc1..f2a1c7087 160000
--- a/PRIVATE
+++ b/PRIVATE
@@ -1 +1 @@
-Subproject commit 532cfadc1fda4f0f7f6a993e2bd029f14a50668f
+Subproject commit f2a1c70879a892fa3b139e14a2eef199bcde5abf

From 97e289deccba870909785037f59b41c04c3cd004 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Fri, 12 Nov 2021 22:11:11 +0100
Subject: [PATCH 02/15] not needed

automated push to master skips CI already
---
 .gitlab-ci.yml | 48 ------------------------------------------------
 1 file changed, 48 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c76504989..5abb8fbc9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -75,9 +75,6 @@ checkout:
     - git checkout $CI_COMMIT_SHA
     - git submodule update --init
     - source env/DAMASK.sh
-  except:
-    - master
-    - release
 
 ###################################################################################################
 pytest:
@@ -86,18 +83,12 @@ pytest:
     - cd $DAMASKROOT/python
     - COLUMNS=256 pytest --basetemp=${TESTROOT}/python -v --cov --cov-report=term
     - coverage report --fail-under=90
-  except:
-    - master
-    - release
 
 mypy:
   stage: python
   script:
     - cd $DAMASKROOT/python
     - mypy -m damask
-  except:
-    - master
-    - release
 
 
 ###################################################################################################
@@ -107,9 +98,6 @@ compile_grid_Intel:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd pytest
     - COLUMNS=256 pytest -k 'compile and grid' --basetemp=${TESTROOT}/compile_grid_Intel
-  except:
-    - master
-    - release
 
 compile_mesh_Intel:
   stage: compile
@@ -117,9 +105,6 @@ compile_mesh_Intel:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd pytest
     - COLUMNS=256 pytest -k 'compile and mesh' --basetemp=${TESTROOT}/compile_mesh_Intel
-  except:
-    - master
-    - release
 
 compile_grid_GNU:
   stage: compile
@@ -127,9 +112,6 @@ compile_grid_GNU:
     - module load $GNUCompiler $MPI_GNU $PETSc_GNU
     - cd pytest
     - COLUMNS=256 pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_GNU
-  except:
-    - master
-    - release
 
 compile_mesh_GNU:
   stage: compile
@@ -137,9 +119,6 @@ compile_mesh_GNU:
     - module load $GNUCompiler $MPI_GNU $PETSc_GNU
     - cd pytest
     - COLUMNS=256 pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_GNU
-  except:
-    - master
-    - release
 
 compile_Marc:
   stage: compile
@@ -147,9 +126,6 @@ compile_Marc:
     - module load $IntelMarc $HDF5Marc $MSC
     - cd pytest
     - COLUMNS=256 pytest -k 'compile and Marc' --basetemp ${TESTROOT}/compile_Marc
-  except:
-    - master
-    - release
 
 
 ###################################################################################################
@@ -160,9 +136,6 @@ setup_grid:
     - cd $(mktemp -d)
     - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT}
     - make -j2 all install
-  except:
-    - master
-    - release
 
 setup_mesh:
   stage: setup
@@ -171,9 +144,6 @@ setup_mesh:
     - cd $(mktemp -d)
     - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT}
     - make -j2 all install
-  except:
-    - master
-    - release
 
 ###################################################################################################
 core:
@@ -182,17 +152,11 @@ core:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd pytest
     - COLUMNS=256 pytest -k 'not compile' --basetemp ${TESTROOT}/fortran -v
-  except:
-    - master
-    - release
 
 # Needs closer look
 # Phenopowerlaw_singleSlip:
 #   stage: fortran
 #   script: Phenopowerlaw_singleSlip/test.py
-#   except:
-#     - master
-#     - release
 
 ###################################################################################################
 grid_runtime:
@@ -207,9 +171,6 @@ grid_runtime:
     - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA}
     - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi
   resource_group: runtime
-  except:
-    - master
-    - release
 
 ###################################################################################################
 source_distribution:
@@ -217,9 +178,6 @@ source_distribution:
   script:
     - cd $(mktemp -d)
     - ${DAMASKROOT}/PRIVATE/releasing/tar.xz/create.sh ${DAMASKROOT} ${CI_COMMIT_SHA}
-  except:
-    - master
-    - release
 
 ##################################################################################################
 merge_into_master:
@@ -251,9 +209,6 @@ remove_data:
   script:
     - rm -rf $LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID
     - sed -i "/$CI_PIPELINE_ID/d" $LOCAL_HOME/GitLabCI.queue                                        # in case pipeline was manually (web GUI) restarted and releaseLock was performed already
-  except:
-    - master
-    - release
 
 ###################################################################################################
 remove_lock:
@@ -262,6 +217,3 @@ remove_lock:
     - echo "Removing lock of pipeline $CI_PIPELINE_ID"
   when: always
   script: sed -i "/$CI_PIPELINE_ID/d" $LOCAL_HOME/GitLabCI.queue
-  except:
-    - master
-    - release

From f112d11985065811b1d9a14b11c32060ed4269d5 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 10:57:58 +0100
Subject: [PATCH 03/15] new locking mechanism

only runtime test needs to run exclusively. lock per job, not per
pipeline.
should be more stable and allow more concurrency

needs python script for job management on gitlab-runner
---
 .gitlab-ci.yml | 75 +++++++++++++++++++-------------------------------
 1 file changed, 29 insertions(+), 46 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5abb8fbc9..4d96b6693 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,26 +8,20 @@ stages:
   - performance
   - deploy
   - update_master
-  - distclean
-  - clean
+
 
 ###################################################################################################
 before_script:
-  - if [ $(awk "/$CI_PIPELINE_ID/{print NR}" $LOCAL_HOME/GitLabCI.queue)x == 'x' ];
-      then echo $CI_PIPELINE_ID >> $LOCAL_HOME/GitLabCI.queue;
-    fi
-  - while [ $(awk "/$CI_PIPELINE_ID/{print NR}" $LOCAL_HOME/GitLabCI.queue) != 1 ];
-      do sleep 5m;
-      echo -e "Currently queued pipelines:\n$(cat $LOCAL_HOME/GitLabCI.queue)\n";
-    done
+  - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking
   - source $DAMASKROOT/env/DAMASK.sh
   - cd $DAMASKROOT/PRIVATE/testing
   - echo Job start:" $(date)"
 
-###################################################################################################
 after_script:
+  - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --unregister
   - echo Job end:" $(date)"
 
+
 ###################################################################################################
 variables:
   # ===============================================================================================
@@ -58,36 +52,35 @@ variables:
   IntelMarc:      "Compiler/Intel/19.1.2 Libraries/IMKL/2020"
   HDF5Marc:       "HDF5/1.12.1/Intel-19.1.2"
 
+
 ###################################################################################################
 checkout:
   stage: prepare
   before_script:
-    - echo $CI_PIPELINE_ID >> $LOCAL_HOME/GitLabCI.queue
-    - while [ $(awk "/$CI_PIPELINE_ID/{print NR}" $LOCAL_HOME/GitLabCI.queue) != 1 ];
-        do sleep 5m;
-        echo -e "Currently queued pipelines:\n$(cat $LOCAL_HOME/GitLabCI.queue)\n";
-      done
+    - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking
+    - echo Job start:" $(date)"
   script:
-    - mkdir -p $DAMASKROOT
-    - mkdir -p $TESTROOT
-    - cd $DAMASKROOT
+    - mkdir -p ${DAMASKROOT}
+    - mkdir -p ${TESTROOT}
+    - cd ${DAMASKROOT}
     - git clone -q git@git.damask.mpie.de:damask/DAMASK.git .
-    - git checkout $CI_COMMIT_SHA
+    - git checkout ${CI_COMMIT_SHA}
     - git submodule update --init
     - source env/DAMASK.sh
 
+
 ###################################################################################################
 pytest:
   stage: python
   script:
-    - cd $DAMASKROOT/python
-    - COLUMNS=256 pytest --basetemp=${TESTROOT}/python -v --cov --cov-report=term
+    - cd ${DAMASKROOT}/python
+    - pytest --basetemp ${TESTROOT}/python -v --cov --cov-report=term
     - coverage report --fail-under=90
 
 mypy:
   stage: python
   script:
-    - cd $DAMASKROOT/python
+    - cd ${DAMASKROOT}/python
     - mypy -m damask
 
 
@@ -97,35 +90,35 @@ compile_grid_Intel:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and grid' --basetemp=${TESTROOT}/compile_grid_Intel
+    - pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_Intel
 
 compile_mesh_Intel:
   stage: compile
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and mesh' --basetemp=${TESTROOT}/compile_mesh_Intel
+    - pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_Intel
 
 compile_grid_GNU:
   stage: compile
   script:
     - module load $GNUCompiler $MPI_GNU $PETSc_GNU
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_GNU
+    - pytest -k 'compile and grid' --basetemp ${TESTROOT}/compile_grid_GNU
 
 compile_mesh_GNU:
   stage: compile
   script:
     - module load $GNUCompiler $MPI_GNU $PETSc_GNU
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_GNU
+    - pytest -k 'compile and mesh' --basetemp ${TESTROOT}/compile_mesh_GNU
 
 compile_Marc:
   stage: compile
   script:
     - module load $IntelMarc $HDF5Marc $MSC
     - cd pytest
-    - COLUMNS=256 pytest -k 'compile and Marc' --basetemp ${TESTROOT}/compile_Marc
+    - pytest -k 'compile and Marc' --basetemp ${TESTROOT}/compile_Marc
 
 
 ###################################################################################################
@@ -145,19 +138,21 @@ setup_mesh:
     - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT}
     - make -j2 all install
 
+
 ###################################################################################################
 core:
   stage: fortran
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd pytest
-    - COLUMNS=256 pytest -k 'not compile' --basetemp ${TESTROOT}/fortran -v
+    - pytest -k 'not compile' --basetemp ${TESTROOT}/fortran -v
 
 # Needs closer look
 # Phenopowerlaw_singleSlip:
 #   stage: fortran
 #   script: Phenopowerlaw_singleSlip/test.py
 
+
 ###################################################################################################
 grid_runtime:
   stage: performance
@@ -171,6 +166,9 @@ grid_runtime:
     - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA}
     - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi
   resource_group: runtime
+  before_script:
+    - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --blocking
+
 
 ###################################################################################################
 source_distribution:
@@ -179,7 +177,8 @@ source_distribution:
     - cd $(mktemp -d)
     - ${DAMASKROOT}/PRIVATE/releasing/tar.xz/create.sh ${DAMASKROOT} ${CI_COMMIT_SHA}
 
-##################################################################################################
+
+###################################################################################################
 merge_into_master:
   stage: update_master
   script:
@@ -192,6 +191,7 @@ merge_into_master:
         git commit -m "[skip ci] updated version information after successful test of $TESTEDREV"
     - export UPDATEDREV=$(git describe)                                                             # tested state + 1 commit
     - git checkout master
+    - git pull
     - git merge $UPDATEDREV -s recursive -X ours                                                    # conflicts occur only for inconsistent state
     - git push origin master                                                                        # master is now tested version and has updated VERSION file
     - git checkout development
@@ -200,20 +200,3 @@ merge_into_master:
     - git push origin development                                                                   # development is unchanged (as master is based on it) but has updated VERSION file
   only:
     - development
-
-###################################################################################################
-remove_data:
-  stage: distclean
-  before_script:
-    - echo "Removing data and lock of pipeline $CI_PIPELINE_ID"
-  script:
-    - rm -rf $LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID
-    - sed -i "/$CI_PIPELINE_ID/d" $LOCAL_HOME/GitLabCI.queue                                        # in case pipeline was manually (web GUI) restarted and releaseLock was performed already
-
-###################################################################################################
-remove_lock:
-  stage: clean
-  before_script:
-    - echo "Removing lock of pipeline $CI_PIPELINE_ID"
-  when: always
-  script: sed -i "/$CI_PIPELINE_ID/d" $LOCAL_HOME/GitLabCI.queue

From 2682a26497cfb6a353a88f9fccf6cf3404aa3fe7 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 11:32:55 +0100
Subject: [PATCH 04/15] preparing for pipeline-wide location of executables

---
 .gitlab-ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4d96b6693..ee981b2f6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,6 +14,7 @@ stages:
 before_script:
   - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking
   - source $DAMASKROOT/env/DAMASK.sh
+  - export PATH=${BINDIR}:$PATH
   - cd $DAMASKROOT/PRIVATE/testing
   - echo Job start:" $(date)"
 
@@ -34,6 +35,7 @@ variables:
   # ===============================================================================================
   DAMASKROOT: "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/DAMASK"
   TESTROOT:   "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/tests"
+  BINDIR:     "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/bin"
 
   # ===============================================================================================
   # Names of module files to load
@@ -62,11 +64,11 @@ checkout:
   script:
     - mkdir -p ${DAMASKROOT}
     - mkdir -p ${TESTROOT}
+    - mkdir -p ${BINDIR}
     - cd ${DAMASKROOT}
     - git clone -q git@git.damask.mpie.de:damask/DAMASK.git .
     - git checkout ${CI_COMMIT_SHA}
     - git submodule update --init
-    - source env/DAMASK.sh
 
 
 ###################################################################################################

From 45af45615a51f371b947f158868555bef3dc16e8 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 11:50:05 +0100
Subject: [PATCH 05/15] build to new bin location

---
 .gitlab-ci.yml | 4 ++--
 PRIVATE        | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ee981b2f6..5e120a660 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -129,7 +129,7 @@ setup_grid:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd $(mktemp -d)
-    - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT}
+    - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${DAMASKROOT}
     - make -j2 all install
 
 setup_mesh:
@@ -137,7 +137,7 @@ setup_mesh:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd $(mktemp -d)
-    - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${DAMASKROOT} ${DAMASKROOT}
+    - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${DAMASKROOT}
     - make -j2 all install
 
 
diff --git a/PRIVATE b/PRIVATE
index f2a1c7087..f730bcb8d 160000
--- a/PRIVATE
+++ b/PRIVATE
@@ -1 +1 @@
-Subproject commit f2a1c70879a892fa3b139e14a2eef199bcde5abf
+Subproject commit f730bcb8ddd7224011e70c57d0a5c03068532d2d

From c75ac5fe6ca2581c031933c68f08f9494b85ad24 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 14:03:06 +0100
Subject: [PATCH 06/15] let gitlab-runner handle the submodule

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5e120a660..5c2afbd1d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -28,7 +28,7 @@ variables:
   # ===============================================================================================
   # GitLab Settings
   # ===============================================================================================
-  GIT_SUBMODULE_STRATEGY: none
+  GIT_SUBMODULE_STRATEGY: normal
 
   # ===============================================================================================
   # Shortcut names

From 352a168621f50ddce18a2cb5edb7506705412c9a Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 14:09:12 +0100
Subject: [PATCH 07/15] new style (https://docs.gitlab.com/ee/ci/yaml)

---
 .gitlab-ci.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5c2afbd1d..555a212c6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,16 +11,16 @@ stages:
 
 
 ###################################################################################################
-before_script:
-  - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking
-  - source $DAMASKROOT/env/DAMASK.sh
-  - export PATH=${BINDIR}:$PATH
-  - cd $DAMASKROOT/PRIVATE/testing
-  - echo Job start:" $(date)"
-
-after_script:
-  - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --unregister
-  - echo Job end:" $(date)"
+default:
+  before_script:
+    - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking
+    - source $DAMASKROOT/env/DAMASK.sh
+    - export PATH=${BINDIR}:$PATH
+    - cd $DAMASKROOT/PRIVATE/testing
+    - echo Job start:" $(date)"
+  after_script:
+    - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --unregister
+    - echo Job end:" $(date)"
 
 
 ###################################################################################################

From 06edd39a93ce84dc46385be14b5e81da36ed35f4 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 16:33:31 +0100
Subject: [PATCH 08/15] still need clean stage

after_script does not run for failed/cancelled scripts.
---
 .gitlab-ci.yml | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 555a212c6..f391c0e0b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,18 +8,19 @@ stages:
   - performance
   - deploy
   - update_master
+  - clean
 
 
 ###################################################################################################
 default:
   before_script:
-    - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking
+    - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --nonblocking
     - source $DAMASKROOT/env/DAMASK.sh
     - export PATH=${BINDIR}:$PATH
     - cd $DAMASKROOT/PRIVATE/testing
     - echo Job start:" $(date)"
   after_script:
-    - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --unregister
+    - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --unregister
     - echo Job end:" $(date)"
 
 
@@ -59,7 +60,7 @@ variables:
 checkout:
   stage: prepare
   before_script:
-    - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --nonblocking
+    - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --nonblocking
     - echo Job start:" $(date)"
   script:
     - mkdir -p ${DAMASKROOT}
@@ -169,7 +170,7 @@ grid_runtime:
     - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi
   resource_group: runtime
   before_script:
-    - ${LOCAL_HOME}/bin/manage_pool ${CI_JOB_ID} --blocking
+    - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --blocking
 
 
 ###################################################################################################
@@ -202,3 +203,15 @@ merge_into_master:
     - git push origin development                                                                   # development is unchanged (as master is based on it) but has updated VERSION file
   only:
     - development
+
+
+###################################################################################################
+clean_pool:
+  stage: clean
+  script:
+    - ${LOCAL_HOME}/bin/manage_pool -p ${CI_PIPELINE_ID} --clean
+  when: always
+  before_script:
+    - echo Job start:" $(date)"
+  after_script:
+    - echo Job end:" $(date)"

From 51c262056777afd29c7168cfebd2e7300f58fd01 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 18:03:40 +0100
Subject: [PATCH 09/15] use (partly) repo cloned by gitlab-runner

---
 .gitlab-ci.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f391c0e0b..146b13d0c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -76,14 +76,15 @@ checkout:
 pytest:
   stage: python
   script:
-    - cd ${DAMASKROOT}/python
+    - PYTHONPATH=${CI_PROJECT_DIR}/python
+    - cd ${CI_PROJECT_DIR}/python
     - pytest --basetemp ${TESTROOT}/python -v --cov --cov-report=term
     - coverage report --fail-under=90
 
 mypy:
   stage: python
   script:
-    - cd ${DAMASKROOT}/python
+    - cd ${CI_PROJECT_DIR}/python
     - mypy -m damask
 
 
@@ -130,7 +131,7 @@ setup_grid:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd $(mktemp -d)
-    - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${DAMASKROOT}
+    - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR}
     - make -j2 all install
 
 setup_mesh:
@@ -138,7 +139,7 @@ setup_mesh:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd $(mktemp -d)
-    - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${DAMASKROOT}
+    - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR}
     - make -j2 all install
 
 
@@ -178,7 +179,7 @@ source_distribution:
   stage: deploy
   script:
     - cd $(mktemp -d)
-    - ${DAMASKROOT}/PRIVATE/releasing/tar.xz/create.sh ${DAMASKROOT} ${CI_COMMIT_SHA}
+    - ${CI_PROJECT_DIR}/PRIVATE/releasing/tar.xz/create.sh ${CI_PROJECT_DIR} ${CI_COMMIT_SHA}
 
 
 ###################################################################################################

From 57817e78ae0a745cd6984a9abe6f544716d9cc90 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 18:39:19 +0100
Subject: [PATCH 10/15] PYTHONPATH was not set

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 146b13d0c..0ae4c605c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -172,6 +172,7 @@ grid_runtime:
   resource_group: runtime
   before_script:
     - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --blocking
+    - source $DAMASKROOT/env/DAMASK.sh
 
 
 ###################################################################################################

From f97c9c7e8c6d941e357fd85a5576089b60013101 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 19:13:10 +0100
Subject: [PATCH 11/15] don't rely on Makefile

---
 .gitlab-ci.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0ae4c605c..c95a16efc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -162,8 +162,9 @@ grid_runtime:
   stage: performance
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
-    - cd $DAMASKROOT
-    - make clean grid OPTIMIZATION=AGGRESSIVE
+    - cd $(mktemp -d)
+    - cmake -DOPTIMIZATION=AGGRESSIVE -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR}
+    - make -j2 all install
     - REPO_DIR=$(mktemp -d)
     - git clone -q git@git.damask.mpie.de:damask/performance.git ${REPO_DIR}
     - cd ${DAMASKROOT}/PRIVATE/testing/

From 9430ca9e6ec20f68df7968692c6f6755af344799 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 19:21:23 +0100
Subject: [PATCH 12/15] simplified

---
 .gitlab-ci.yml | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c95a16efc..0b7c7d0d2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,7 +16,7 @@ default:
   before_script:
     - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --nonblocking
     - source $DAMASKROOT/env/DAMASK.sh
-    - export PATH=${BINDIR}:$PATH
+    - export PATH=${TESTROOT}/bin:$PATH
     - cd $DAMASKROOT/PRIVATE/testing
     - echo Job start:" $(date)"
   after_script:
@@ -34,9 +34,8 @@ variables:
   # ===============================================================================================
   # Shortcut names
   # ===============================================================================================
+  TESTROOT:   "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID"
   DAMASKROOT: "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/DAMASK"
-  TESTROOT:   "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/tests"
-  BINDIR:     "$LOCAL_HOME/GitLabCI_Pipeline_$CI_PIPELINE_ID/bin"
 
   # ===============================================================================================
   # Names of module files to load
@@ -64,8 +63,6 @@ checkout:
     - echo Job start:" $(date)"
   script:
     - mkdir -p ${DAMASKROOT}
-    - mkdir -p ${TESTROOT}
-    - mkdir -p ${BINDIR}
     - cd ${DAMASKROOT}
     - git clone -q git@git.damask.mpie.de:damask/DAMASK.git .
     - git checkout ${CI_COMMIT_SHA}
@@ -131,7 +128,7 @@ setup_grid:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd $(mktemp -d)
-    - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR}
+    - cmake -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${TESTROOT} ${CI_PROJECT_DIR}
     - make -j2 all install
 
 setup_mesh:
@@ -139,7 +136,7 @@ setup_mesh:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd $(mktemp -d)
-    - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR}
+    - cmake -DDAMASK_SOLVER=MESH -DCMAKE_INSTALL_PREFIX=${TESTROOT} ${CI_PROJECT_DIR}
     - make -j2 all install
 
 
@@ -163,7 +160,7 @@ grid_runtime:
   script:
     - module load $IntelCompiler $MPI_Intel $PETSc_Intel
     - cd $(mktemp -d)
-    - cmake -DOPTIMIZATION=AGGRESSIVE -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${LOCAL_HOME}/GitLabCI_Pipeline_${CI_PIPELINE_ID} ${CI_PROJECT_DIR}
+    - cmake -DOPTIMIZATION=AGGRESSIVE -DDAMASK_SOLVER=GRID -DCMAKE_INSTALL_PREFIX=${TESTROOT} ${CI_PROJECT_DIR}
     - make -j2 all install
     - REPO_DIR=$(mktemp -d)
     - git clone -q git@git.damask.mpie.de:damask/performance.git ${REPO_DIR}
@@ -174,7 +171,8 @@ grid_runtime:
   before_script:
     - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --blocking
     - source $DAMASKROOT/env/DAMASK.sh
-
+    - export PATH=${TESTROOT}/bin:$PATH
+    - echo Job start:" $(date)"
 
 ###################################################################################################
 source_distribution:

From 4396d605ec08317b043436c54647a76fb111e623 Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 19:37:40 +0100
Subject: [PATCH 13/15] test folder, not module

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0b7c7d0d2..adaceb612 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -82,7 +82,7 @@ mypy:
   stage: python
   script:
     - cd ${CI_PROJECT_DIR}/python
-    - mypy -m damask
+    - mypy damask
 
 
 ###################################################################################################

From 74921ebe190212d1e74fb7f9ca4d83715ee73c7d Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 21:28:42 +0100
Subject: [PATCH 14/15] not needed, only delays runtime test

resource management is done with 'manage pool' script
---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index adaceb612..5e95cd23b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -167,7 +167,6 @@ grid_runtime:
     - cd ${DAMASKROOT}/PRIVATE/testing/
     - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA}
     - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi
-  resource_group: runtime
   before_script:
     - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --blocking
     - source $DAMASKROOT/env/DAMASK.sh

From 4e57193a0240a2cbadd57d356685c6a3e6e03f3e Mon Sep 17 00:00:00 2001
From: Martin Diehl <mail@martin-diehl.net>
Date: Sat, 13 Nov 2021 22:58:56 +0100
Subject: [PATCH 15/15] simplified

use GitLab CI API to remove finished jobs, no user intervention needed
after cancelling
---
 .gitlab-ci.yml | 20 +++-----------------
 1 file changed, 3 insertions(+), 17 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5e95cd23b..3d21f3184 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,19 +8,17 @@ stages:
   - performance
   - deploy
   - update_master
-  - clean
 
 
 ###################################################################################################
 default:
   before_script:
-    - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --nonblocking
+    - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID}
     - source $DAMASKROOT/env/DAMASK.sh
     - export PATH=${TESTROOT}/bin:$PATH
     - cd $DAMASKROOT/PRIVATE/testing
     - echo Job start:" $(date)"
   after_script:
-    - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --unregister
     - echo Job end:" $(date)"
 
 
@@ -59,7 +57,7 @@ variables:
 checkout:
   stage: prepare
   before_script:
-    - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --nonblocking
+    - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID}
     - echo Job start:" $(date)"
   script:
     - mkdir -p ${DAMASKROOT}
@@ -168,7 +166,7 @@ grid_runtime:
     - ./runtime.py --input_dir $DAMASKROOT/examples/grid --output_dir ${REPO_DIR} --tag ${CI_COMMIT_SHA}
     - if [ ${CI_COMMIT_BRANCH} == development ]; then git commit -am ${CI_PIPELINE_ID}_${CI_COMMIT_SHA}; git push; fi
   before_script:
-    - ${LOCAL_HOME}/bin/manage_pool -j ${CI_JOB_ID} -p ${CI_PIPELINE_ID} --blocking
+    - ${LOCAL_HOME}/bin/queue ${CI_JOB_ID} --blocking
     - source $DAMASKROOT/env/DAMASK.sh
     - export PATH=${TESTROOT}/bin:$PATH
     - echo Job start:" $(date)"
@@ -203,15 +201,3 @@ merge_into_master:
     - git push origin development                                                                   # development is unchanged (as master is based on it) but has updated VERSION file
   only:
     - development
-
-
-###################################################################################################
-clean_pool:
-  stage: clean
-  script:
-    - ${LOCAL_HOME}/bin/manage_pool -p ${CI_PIPELINE_ID} --clean
-  when: always
-  before_script:
-    - echo Job start:" $(date)"
-  after_script:
-    - echo Job end:" $(date)"