From 429b84004d193e3216ab68f1fa77c712260beac8 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Sun, 6 Dec 2020 19:57:00 +0100 Subject: [PATCH] more filters - shuffle: significanlty better compression - Fletcher32: checksum to detect errors computational overhead is very small --- python/damask/_result.py | 7 ++++--- src/HDF5_utilities.f90 | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/python/damask/_result.py b/python/damask/_result.py index 6d83c8872..600469e79 100644 --- a/python/damask/_result.py +++ b/python/damask/_result.py @@ -1160,10 +1160,11 @@ class Result: shape = result[1]['data'].shape chunks = (chunk_size//np.prod(shape[1:]),)+shape[1:] dataset = f[result[0]].create_dataset(result[1]['label'],data=result[1]['data'], - maxshape=shape,chunks=chunks,compression = 'gzip') + maxshape=shape, chunks=chunks, + compression='gzip', compression_opts=6, + shuffle=True,fletcher32=True) else: - dataset = f[result[0]].create_dataset(result[1]['label'],data=result[1]['data'], - maxshape=result[1]['data'].shape) + dataset = f[result[0]].create_dataset(result[1]['label'],data=result[1]['data']) now = datetime.datetime.now().astimezone() dataset.attrs['Created'] = now.strftime('%Y-%m-%d %H:%M:%S%z') if h5py3 else \ diff --git a/src/HDF5_utilities.f90 b/src/HDF5_utilities.f90 index a2c023ef5..f976ad106 100644 --- a/src/HDF5_utilities.f90 +++ b/src/HDF5_utilities.f90 @@ -1887,8 +1887,12 @@ subroutine initialize_write(dset_id, filespace_id, memspace_id, plist_id, & if(product(totalShape) >= chunkSize*2_HSIZE_T) then call h5pset_chunk_f(dcpl, size(totalShape), getChunks(totalShape,chunkSize), hdferr) if(hdferr < 0) error stop 'HDF5 error' + call h5pset_shuffle_f(dcpl, hdferr) + if(hdferr < 0) error stop 'HDF5 error' call h5pset_deflate_f(dcpl, 6, hdferr) if(hdferr < 0) error stop 'HDF5 error' + call h5pset_Fletcher32_f(dcpl,hdferr) + if(hdferr < 0) error stop 'HDF5 error' endif !--------------------------------------------------------------------------------------------------