From 1ce214a39aa13ff5c5063addde6417883bc6bdbb Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 1 Sep 2021 22:52:45 +0200 Subject: [PATCH 1/6] initial setup https://docs.github.com/en/actions/quickstart --- .github/workflows/.github-actions-demo.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/.github-actions-demo.yml diff --git a/.github/workflows/.github-actions-demo.yml b/.github/workflows/.github-actions-demo.yml new file mode 100644 index 000000000..02fdddc6e --- /dev/null +++ b/.github/workflows/.github-actions-demo.yml @@ -0,0 +1,18 @@ +name: GitHub Actions Demo +on: [push] +jobs: + Explore-GitHub-Actions: + runs-on: ubuntu-latest + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" + - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." + - name: Check out repository code + uses: actions/checkout@v2 + - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." + - run: echo "🖥️ The workflow is now ready to test your code on the runner." + - name: List files in the repository + run: | + ls ${{ github.workspace }} + - run: echo "🍏 This job's status is ${{ job.status }}." + From 0dd67b5d44df3876044ef7e0062caf6d0b954042 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 1 Sep 2021 23:11:39 +0200 Subject: [PATCH 2/6] do real tests https://github.community/t/create-matrix-with-multiple-os-and-env-for-each-one/16895 --- .github/workflows/.github-actions-demo.yml | 40 +++++++++++++--------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/workflows/.github-actions-demo.yml b/.github/workflows/.github-actions-demo.yml index 02fdddc6e..f167f05eb 100644 --- a/.github/workflows/.github-actions-demo.yml +++ b/.github/workflows/.github-actions-demo.yml @@ -1,18 +1,26 @@ -name: GitHub Actions Demo +name: Test Processing Tools on: [push] -jobs: - Explore-GitHub-Actions: - runs-on: ubuntu-latest - steps: - - run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event." - - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!" - - run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}." - - name: Check out repository code - uses: actions/checkout@v2 - - run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner." - - run: echo "🖥️ The workflow is now ready to test your code on the runner." - - name: List files in the repository - run: | - ls ${{ github.workspace }} - - run: echo "🍏 This job's status is ${{ job.status }}." +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + os: [ubuntu-latest, macos-latest] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest pandas scipy h5py vtk matplotlib pyyaml + - name: Test with pytest + run: | + python -m pip install ./python --no-deps -vv --use-feature=in-tree-build + pytest python From 06d62b42cbb59404a778ca5ee731121fad81a0a6 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 1 Sep 2021 23:37:46 +0200 Subject: [PATCH 3/6] also test Ubuntu packages --- .github/workflows/.github-actions-demo.yml | 23 ++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/.github-actions-demo.yml b/.github/workflows/.github-actions-demo.yml index f167f05eb..00f6565ff 100644 --- a/.github/workflows/.github-actions-demo.yml +++ b/.github/workflows/.github-actions-demo.yml @@ -2,7 +2,7 @@ name: Test Processing Tools on: [push] jobs: - build: + pip: runs-on: ${{ matrix.os }} strategy: @@ -20,7 +20,26 @@ jobs: run: | python -m pip install --upgrade pip pip install pytest pandas scipy h5py vtk matplotlib pyyaml - - name: Test with pytest + - name: Install and run unit tests run: | python -m pip install ./python --no-deps -vv --use-feature=in-tree-build pytest python + + apt: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Install pytest + run: | + python -m pip install --upgrade pip + pip install pytest + - name: Install dependencies + run: > + sudo apt-get update && + sudo apt-get install python3-pip python3-pytest python3-pandas python3-scipy + python3-h5py python3-vtk7 python3-matplotlib python3-yaml -y + - name: Run unit tests + run: | + PYTHONPATH=${PWD}/python python -m pytest python From 2b9d416734743e87f0b6414d625dd47352cbde4e Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 2 Sep 2021 08:04:09 +0200 Subject: [PATCH 4/6] fixes for old VTK and/or old numpy --- .github/workflows/.github-actions-demo.yml | 4 +++- python/damask/_vtk.py | 4 ++-- python/tests/test_Grid.py | 2 ++ python/tests/test_Result.py | 28 ++++++++++++---------- python/tests/test_VTK.py | 7 ++++-- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.github/workflows/.github-actions-demo.yml b/.github/workflows/.github-actions-demo.yml index 00f6565ff..427953f7b 100644 --- a/.github/workflows/.github-actions-demo.yml +++ b/.github/workflows/.github-actions-demo.yml @@ -42,4 +42,6 @@ jobs: python3-h5py python3-vtk7 python3-matplotlib python3-yaml -y - name: Run unit tests run: | - PYTHONPATH=${PWD}/python python -m pytest python + export PYTHONPATH=${PWD}/python + export COLUMNS=256 + python -m pytest python diff --git a/python/damask/_vtk.py b/python/damask/_vtk.py index 7ca794f70..cbf70c37a 100644 --- a/python/damask/_vtk.py +++ b/python/damask/_vtk.py @@ -121,7 +121,7 @@ class VTK: """ vtk_nodes = vtk.vtkPoints() - vtk_nodes.SetData(np_to_vtk(nodes)) + vtk_nodes.SetData(np_to_vtk(np.ascontiguousarray(nodes))) cells = vtk.vtkCellArray() cells.SetNumberOfCells(connectivity.shape[0]) T = np.concatenate((np.ones((connectivity.shape[0],1),dtype=np.int64)*connectivity.shape[1], @@ -157,7 +157,7 @@ class VTK: """ N = points.shape[0] vtk_points = vtk.vtkPoints() - vtk_points.SetData(np_to_vtk(points)) + vtk_points.SetData(np_to_vtk(np.ascontiguousarray(points))) vtk_cells = vtk.vtkCellArray() vtk_cells.SetNumberOfCells(N) diff --git a/python/tests/test_Grid.py b/python/tests/test_Grid.py index 36d489939..3538e3dd8 100644 --- a/python/tests/test_Grid.py +++ b/python/tests/test_Grid.py @@ -1,5 +1,6 @@ import pytest import numpy as np +import vtk from damask import VTK from damask import Grid @@ -410,6 +411,7 @@ class TestGrid: @pytest.mark.parametrize('periodic',[True,False]) @pytest.mark.parametrize('direction',['x','y','z',['x','y'],'zy','xz',['x','y','z']]) + @pytest.mark.xfail(int(vtk.vtkVersion.GetVTKVersion().split('.')[0])<8, reason='missing METADATA') def test_get_grain_boundaries(self,update,ref_path,periodic,direction): grid = Grid.load(ref_path/'get_grain_boundaries_8g12x15x20.vti') current = grid.get_grain_boundaries(periodic,direction) diff --git a/python/tests/test_Result.py b/python/tests/test_Result.py index 6c06e8636..197a839de 100644 --- a/python/tests/test_Result.py +++ b/python/tests/test_Result.py @@ -18,6 +18,12 @@ from damask import tensor from damask import mechanics from damask import grid_filters +def vtk_hasXdmfReader(): + if hasattr(vtk,'vtkXdmfReader'): + r = vtk.vtkXdmfReader() + if hasattr(r,'getOutput'): return True + return False + @pytest.fixture def default(tmp_path,ref_path): """Small Result file in temp location for modification.""" @@ -374,6 +380,7 @@ class TestResult: @pytest.mark.parametrize('output',['F','*',['P'],['P','F']],ids=range(4)) @pytest.mark.parametrize('fname',['12grains6x7x8_tensionY.hdf5'],ids=range(1)) @pytest.mark.parametrize('inc',[4,0],ids=range(2)) + @pytest.mark.xfail(int(vtk.vtkVersion.GetVTKVersion().split('.')[0])<9, reason='missing "Direction" attribute') def test_vtk(self,request,tmp_path,ref_path,update,patch_execution_stamp,patch_datetime_now,output,fname,inc): result = Result(ref_path/fname).view('increments',inc) os.chdir(tmp_path) @@ -425,7 +432,7 @@ class TestResult: assert sorted(open(tmp_path/fname).read()) == sorted(open(ref_path/fname).read()) # XML is not ordered - @pytest.mark.skipif(not hasattr(vtk,'vtkXdmfReader'),reason='https://discourse.vtk.org/t/2450') + @pytest.mark.skipif(not vtk_hasXdmfReader(),reason='https://discourse.vtk.org/t/2450') def test_XDMF_shape(self,tmp_path,single_phase): os.chdir(tmp_path) @@ -437,19 +444,14 @@ class TestResult: dim_xdmf = reader_xdmf.GetOutput().GetDimensions() bounds_xdmf = reader_xdmf.GetOutput().GetBounds() - single_phase.view('increments',0).export_VTK() + single_phase.view('increments',0).export_VTK(parallel=False) fname = os.path.splitext(os.path.basename(single_phase.fname))[0]+'_inc00.vti' - for i in range(10): # waiting for parallel IO - reader_vti = vtk.vtkXMLImageDataReader() - reader_vti.SetFileName(fname) - reader_vti.Update() - dim_vti = reader_vti.GetOutput().GetDimensions() - bounds_vti = reader_vti.GetOutput().GetBounds() - if dim_vti == dim_xdmf and bounds_vti == bounds_xdmf: - return - time.sleep(.5) - - assert False + reader_vti = vtk.vtkXMLImageDataReader() + reader_vti.SetFileName(fname) + reader_vti.Update() + dim_vti = reader_vti.GetOutput().GetDimensions() + bounds_vti = reader_vti.GetOutput().GetBounds() + assert dim_vti == dim_xdmf and bounds_vti == bounds_xdmf def test_XDMF_invalid(self,default): with pytest.raises(TypeError): diff --git a/python/tests/test_VTK.py b/python/tests/test_VTK.py index d4606d5c1..26d1c4a53 100644 --- a/python/tests/test_VTK.py +++ b/python/tests/test_VTK.py @@ -5,6 +5,7 @@ import time import pytest import numpy as np import numpy.ma as ma +import vtk from damask import VTK from damask import grid_filters @@ -162,6 +163,7 @@ class TestVTK: new = VTK.load(tmp_path/'with_comments.vtr') assert new.get_comments() == ['this is a comment'] + @pytest.mark.xfail(int(vtk.vtkVersion.GetVTKVersion().split('.')[0])<8, reason='missing METADATA') def test_compare_reference_polyData(self,update,ref_path,tmp_path): points=np.dstack((np.linspace(0.,1.,10),np.linspace(0.,2.,10),np.linspace(-1.,1.,10))).squeeze() polyData = VTK.from_poly_data(points) @@ -173,14 +175,15 @@ class TestVTK: assert polyData.__repr__() == reference.__repr__() and \ np.allclose(polyData.get('coordinates'),points) + @pytest.mark.xfail(int(vtk.vtkVersion.GetVTKVersion().split('.')[0])<8, reason='missing METADATA') def test_compare_reference_rectilinearGrid(self,update,ref_path,tmp_path): cells = np.array([5,6,7],int) size = np.array([.6,1.,.5]) rectilinearGrid = VTK.from_rectilinear_grid(cells,size) c = grid_filters.coordinates0_point(cells,size).reshape(-1,3,order='F') n = grid_filters.coordinates0_node(cells,size).reshape(-1,3,order='F') - rectilinearGrid.add(c,'cell') - rectilinearGrid.add(n,'node') + rectilinearGrid.add(np.ascontiguousarray(c),'cell') + rectilinearGrid.add(np.ascontiguousarray(n),'node') if update: rectilinearGrid.save(ref_path/'rectilinearGrid') else: From 727cc9da126ec4193dd5233368473e5e95afa395 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 2 Sep 2021 08:51:45 +0200 Subject: [PATCH 5/6] ready for production --- .../workflows/{.github-actions-demo.yml => .python.yml} | 9 ++++----- python/tests/test_Result.py | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) rename .github/workflows/{.github-actions-demo.yml => .python.yml} (89%) diff --git a/.github/workflows/.github-actions-demo.yml b/.github/workflows/.python.yml similarity index 89% rename from .github/workflows/.github-actions-demo.yml rename to .github/workflows/.python.yml index 427953f7b..77375b9dc 100644 --- a/.github/workflows/.github-actions-demo.yml +++ b/.github/workflows/.python.yml @@ -1,4 +1,4 @@ -name: Test Processing Tools +name: Test Python Library on: [push] jobs: @@ -23,7 +23,7 @@ jobs: - name: Install and run unit tests run: | python -m pip install ./python --no-deps -vv --use-feature=in-tree-build - pytest python + COLUMNS=256 pytest python apt: @@ -38,10 +38,9 @@ jobs: - name: Install dependencies run: > sudo apt-get update && - sudo apt-get install python3-pip python3-pytest python3-pandas python3-scipy + sudo apt-get install python3-pip python3-pytest python3-pandas python3-scipy python3-h5py python3-vtk7 python3-matplotlib python3-yaml -y - name: Run unit tests run: | export PYTHONPATH=${PWD}/python - export COLUMNS=256 - python -m pytest python + COLUMNS=256 python -m pytest python diff --git a/python/tests/test_Result.py b/python/tests/test_Result.py index 197a839de..b7911a1df 100644 --- a/python/tests/test_Result.py +++ b/python/tests/test_Result.py @@ -114,7 +114,7 @@ class TestResult: assert np.allclose(in_memory,in_file) @pytest.mark.parametrize('mode', - ['direct',pytest.param('function',marks=pytest.mark.xfail(sys.platform=="darwin",reason='n/a'))]) + ['direct',pytest.param('function',marks=pytest.mark.xfail(sys.platform=='darwin',reason='n/a'))]) def test_add_calculation(self,default,tmp_path,mode): if mode == 'direct': From 0653d0c481f25e77f0195f7610f5c39ecb95d838 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Thu, 2 Sep 2021 14:35:16 +0200 Subject: [PATCH 6/6] shortened thanks to lazy evaluation --- python/tests/test_Result.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/python/tests/test_Result.py b/python/tests/test_Result.py index b7911a1df..4f779857f 100644 --- a/python/tests/test_Result.py +++ b/python/tests/test_Result.py @@ -18,11 +18,6 @@ from damask import tensor from damask import mechanics from damask import grid_filters -def vtk_hasXdmfReader(): - if hasattr(vtk,'vtkXdmfReader'): - r = vtk.vtkXdmfReader() - if hasattr(r,'getOutput'): return True - return False @pytest.fixture def default(tmp_path,ref_path): @@ -432,7 +427,8 @@ class TestResult: assert sorted(open(tmp_path/fname).read()) == sorted(open(ref_path/fname).read()) # XML is not ordered - @pytest.mark.skipif(not vtk_hasXdmfReader(),reason='https://discourse.vtk.org/t/2450') + @pytest.mark.skipif(not (hasattr(vtk,'vtkXdmfReader') and hasattr(vtk.vtkXdmfReader(),'GetOutput')), + reason='https://discourse.vtk.org/t/2450') def test_XDMF_shape(self,tmp_path,single_phase): os.chdir(tmp_path)