no multiprocessing for adding datasets

multiprocessing was not reliable on different platform and caused all
kinds of weird behavior
This commit is contained in:
Martin Diehl 2023-09-21 21:24:45 +02:00
parent 0604e510ec
commit b0bb904c89
1 changed files with 4 additions and 16 deletions

View File

@ -1,5 +1,3 @@
import multiprocessing as mp
from multiprocessing.synchronize import Lock
import re
import fnmatch
import os
@ -1450,12 +1448,10 @@ class Result:
group: str,
callback: Callable,
datasets: Dict[str, str],
args: Dict[str, str],
lock: Lock) -> List[Union[None, Any]]:
args: Dict[str, str]) -> List[Union[None, Any]]:
"""Execute job for _add_generic_pointwise."""
try:
datasets_in = {}
lock.acquire()
with h5py.File(self.fname,'r') as f:
for arg,label in datasets.items():
loc = f[group+'/'+label]
@ -1463,7 +1459,6 @@ class Result:
'label':label,
'meta': {k:(v.decode() if not h5py3 and type(v) is bytes else v) \
for k,v in loc.attrs.items()}}
lock.release()
r = callback(**datasets_in,**args)
return [group,r]
except Exception as err:
@ -1490,9 +1485,6 @@ class Result:
Arguments parsed to func.
"""
pool = mp.Pool(int(os.environ.get('OMP_NUM_THREADS',4)))
lock = mp.Manager().Lock()
groups = []
with h5py.File(self.fname,'r') as f:
for inc in self.visible['increments']:
@ -1506,12 +1498,12 @@ class Result:
print('No matching dataset found, no data was added.')
return
default_arg = partial(self._job_pointwise,callback=func,datasets=datasets,args=args,lock=lock)
default_arg = partial(self._job_pointwise,callback=func,datasets=datasets,args=args)
for group,result in util.show_progress(pool.imap_unordered(default_arg,groups),len(groups)):# type: ignore
for grp in util.show_progress(groups):
group, result = default_arg(grp) # type: ignore
if not result:
continue
lock.acquire()
with h5py.File(self.fname, 'a') as f:
try:
if not self._protected and '/'.join([group,result['label']]) in f:
@ -1543,10 +1535,6 @@ class Result:
except (OSError,RuntimeError) as err:
print(f'Could not add dataset: {err}.')
lock.release()
pool.close()
pool.join()
def _mappings(self):