223031012
/
DAMASK_EICMD
mirror of https://github.com/achalhp/DAMASK_EICMD.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

more filters 

- shuffle: significanlty better compression
- Fletcher32: checksum to detect errors

computational overhead is very small
This commit is contained in:
Martin Diehl 2020-12-06 19:57:00 +01:00
parent bc4361c2ae
commit 429b84004d
2 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
python/damask/_result.py
View File

@ -1160,10 +1160,11 @@ class Result:
shape = result[1]['data'].shape shape = result[1]['data'].shape
chunks = (chunk_size//np.prod(shape[1:]),)+shape[1:] chunks = (chunk_size//np.prod(shape[1:]),)+shape[1:]
dataset = f[result[0]].create_dataset(result[1]['label'],data=result[1]['data'], dataset = f[result[0]].create_dataset(result[1]['label'],data=result[1]['data'],
maxshape=shape,chunks=chunks,compression = 'gzip') maxshape=shape, chunks=chunks,
compression='gzip', compression_opts=6,
shuffle=True,fletcher32=True)
else: else:
dataset = f[result[0]].create_dataset(result[1]['label'],data=result[1]['data'], dataset = f[result[0]].create_dataset(result[1]['label'],data=result[1]['data'])
maxshape=result[1]['data'].shape)
now = datetime.datetime.now().astimezone() now = datetime.datetime.now().astimezone()
dataset.attrs['Created'] = now.strftime('%Y-%m-%d %H:%M:%S%z') if h5py3 else \ dataset.attrs['Created'] = now.strftime('%Y-%m-%d %H:%M:%S%z') if h5py3 else \

4
src/HDF5_utilities.f90
View File

@ -1887,8 +1887,12 @@ subroutine initialize_write(dset_id, filespace_id, memspace_id, plist_id, &
if(product(totalShape) >= chunkSize*2_HSIZE_T) then if(product(totalShape) >= chunkSize*2_HSIZE_T) then
call h5pset_chunk_f(dcpl, size(totalShape), getChunks(totalShape,chunkSize), hdferr) call h5pset_chunk_f(dcpl, size(totalShape), getChunks(totalShape,chunkSize), hdferr)
if(hdferr < 0) error stop 'HDF5 error' if(hdferr < 0) error stop 'HDF5 error'
call h5pset_shuffle_f(dcpl, hdferr)
if(hdferr < 0) error stop 'HDF5 error'
call h5pset_deflate_f(dcpl, 6, hdferr) call h5pset_deflate_f(dcpl, 6, hdferr)
if(hdferr < 0) error stop 'HDF5 error' if(hdferr < 0) error stop 'HDF5 error'
call h5pset_Fletcher32_f(dcpl,hdferr)
if(hdferr < 0) error stop 'HDF5 error'
endif endif
!-------------------------------------------------------------------------------------------------- !--------------------------------------------------------------------------------------------------