From bb6f54963a397d84bf3558cabdfb687d98efaed4 Mon Sep 17 00:00:00 2001 From: Martin Diehl Date: Wed, 29 May 2019 07:49:43 +0200 Subject: [PATCH] save some memory (avoid copies if not needed) --- processing/pre/geom_clean.py | 8 +++----- processing/pre/geom_grainGrowth.py | 5 ++--- processing/pre/geom_pack.py | 29 ++++++++++++--------------- processing/pre/geom_renumber.py | 7 +++---- processing/pre/geom_rescale.py | 14 +++++++------ processing/pre/geom_rotate.py | 1 - processing/pre/geom_toTable.py | 5 ++--- processing/pre/geom_translate.py | 5 ++--- processing/pre/geom_vicinityOffset.py | 18 ++++++++--------- 9 files changed, 41 insertions(+), 51 deletions(-) diff --git a/processing/pre/geom_clean.py b/processing/pre/geom_clean.py index 8fa7f5953..8e477c1c1 100755 --- a/processing/pre/geom_clean.py +++ b/processing/pre/geom_clean.py @@ -45,12 +45,10 @@ for name in filenames: damask.util.report(scriptName,name) geom = damask.Geom.from_file(StringIO(''.join(sys.stdin.read())) if name is None else name) - microstructure = geom.get_microstructure() - microstructure = ndimage.filters.generic_filter(microstructure,mostFrequent, - size=(options.stencil,)*3).astype(microstructure.dtype) - - damask.util.croak(geom.update(microstructure)) + damask.util.croak(geom.update(ndimage.filters.generic_filter( + geom.microstructure,mostFrequent, + size=(options.stencil,)*3).astype(geom.microstructure.dtype))) geom.add_comments(scriptID + ' ' + ' '.join(sys.argv[1:])) if name is None: diff --git a/processing/pre/geom_grainGrowth.py b/processing/pre/geom_grainGrowth.py index 952c15c13..4c8b6c8bc 100755 --- a/processing/pre/geom_grainGrowth.py +++ b/processing/pre/geom_grainGrowth.py @@ -60,11 +60,10 @@ for name in filenames: damask.util.report(scriptName,name) geom = damask.Geom.from_file(StringIO(''.join(sys.stdin.read())) if name is None else name) - microstructure = geom.get_microstructure() + grid_original = geom.get_grid() damask.util.croak(geom) - grid_original = geom.get_grid() - microstructure = np.tile(geom.get_microstructure(),np.where(grid_original == 1, 2,1)) # make one copy along dimensions with grid == 1 + microstructure = np.tile(geom.microstructure,np.where(grid_original == 1, 2,1)) # make one copy along dimensions with grid == 1 grid = np.array(microstructure.shape) # --- initialize support data --------------------------------------------------------------------- diff --git a/processing/pre/geom_pack.py b/processing/pre/geom_pack.py index 1a002830e..68045defa 100755 --- a/processing/pre/geom_pack.py +++ b/processing/pre/geom_pack.py @@ -30,7 +30,6 @@ for name in filenames: damask.util.report(scriptName,name) geom = damask.Geom.from_file(StringIO(''.join(sys.stdin.read())) if name is None else name) - microstructure = geom.get_microstructure().flatten('F') damask.util.croak(geom) geom.add_comments(scriptID + ' ' + ' '.join(sys.argv[1:])) @@ -38,7 +37,12 @@ for name in filenames: former = start = -1 reps = 0 - for current in microstructure: + if name is None: + f = sys.stdout + else: + f= open(name,'w') + + for current in geom.microstructure.flatten('F'): if abs(current - former) == 1 and (start - current) == reps*(former - current): compressType = 'to' reps += 1 @@ -47,13 +51,13 @@ for name in filenames: reps += 1 else: if compressType is None: - out = geom.get_header() + f.write('\n'.join(geom.get_header())+'\n') elif compressType == '.': - out.append('{}'.format(former)) + f.write('{}\n'.format(former)) elif compressType == 'to': - out.append('{} to {}'.format(start,former)) + f.write('{} to {}\n'.format(start,former)) elif compressType == 'of': - out.append('{} of {}'.format(reps,former)) + f.write('{} of {}\n'.format(reps,former)) compressType = '.' start = current @@ -62,15 +66,8 @@ for name in filenames: former = current if compressType == '.': - out.append('{}'.format(former)) + f.write('{}\n'.format(former)) elif compressType == 'to': - out.append('{} to {}'.format(start,former)) + f.write('{} to {}\n'.format(start,former)) elif compressType == 'of': - out.append('{} of {}'.format(reps,former)) - - - if name is None: - sys.stdout.write('\n'.join(out)+'\n') - else: - with open(name,'w') as f: - f.write('\n'.join(out)+'\n') + f.write('{} of {}\n'.format(reps,former)) diff --git a/processing/pre/geom_renumber.py b/processing/pre/geom_renumber.py index ef21cd1ab..403aeaf5b 100755 --- a/processing/pre/geom_renumber.py +++ b/processing/pre/geom_renumber.py @@ -32,11 +32,10 @@ for name in filenames: damask.util.report(scriptName,name) geom = damask.Geom.from_file(StringIO(''.join(sys.stdin.read())) if name is None else name) - microstructure = geom.get_microstructure() - renumbered = np.copy(microstructure) - for i, oldID in enumerate(np.unique(microstructure)): - renumbered = np.where(microstructure == oldID, i+1, renumbered) + renumbered = np.empty(geom.get_grid(),dtype=geom.microstructure.dtype) + for i, oldID in enumerate(np.unique(geom.microstructure)): + renumbered = np.where(geom.microstructure == oldID, i+1, renumbered) damask.util.croak(geom.update(renumbered)) geom.add_comments(scriptID + ' ' + ' '.join(sys.argv[1:])) diff --git a/processing/pre/geom_rescale.py b/processing/pre/geom_rescale.py index 1a94f327e..aa639a146 100755 --- a/processing/pre/geom_rescale.py +++ b/processing/pre/geom_rescale.py @@ -43,7 +43,6 @@ for name in filenames: damask.util.report(scriptName,name) geom = damask.Geom.from_file(StringIO(''.join(sys.stdin.read())) if name is None else name) - microstructure = geom.get_microstructure() grid = geom.get_grid() size = geom.get_size() @@ -55,11 +54,14 @@ for name in filenames: np.array([o*float(n.lower().replace('x','')) if n.lower().endswith('x') \ else float(n) for o,n in zip(size,options.size)],dtype=float) - if np.any(new_grid != grid): - microstructure = ndimage.interpolation.zoom(microstructure, new_grid/grid,output=microstructure.dtype, - order=0,mode='nearest', prefilter=False) - - damask.util.croak(geom.update(microstructure,new_size)) + damask.util.croak(geom.update(microstructure = + ndimage.interpolation.zoom( + geom.microstructure, + new_grid/grid,output=geom.microstructure.dtype, + order=0,mode='nearest', prefilter=False\ + ) \ + if np.any(new_grid != grid) else None, + size = new_size)) geom.add_comments(scriptID + ' ' + ' '.join(sys.argv[1:])) if name is None: diff --git a/processing/pre/geom_rotate.py b/processing/pre/geom_rotate.py index 4189fefea..476e26c29 100755 --- a/processing/pre/geom_rotate.py +++ b/processing/pre/geom_rotate.py @@ -88,7 +88,6 @@ for name in filenames: microstructure = ndimage.rotate(microstructure,eulers[0],(0,1),order=0, prefilter=False,output=dtype,cval=fill) # rotation around z - damask.util.croak(geom.update(microstructure,rescale=True)) geom.add_comments(scriptID + ' ' + ' '.join(sys.argv[1:])) diff --git a/processing/pre/geom_toTable.py b/processing/pre/geom_toTable.py index 352238524..7db7f8430 100755 --- a/processing/pre/geom_toTable.py +++ b/processing/pre/geom_toTable.py @@ -32,8 +32,7 @@ for name in filenames: damask.util.report(scriptName,name) geom = damask.Geom.from_file(StringIO(''.join(sys.stdin.read())) if name is None else name) - damask.util.croak(geom) - microstructure = geom.get_microstructure().flatten('F') + damask.util.croak(geom) grid = geom.get_grid() size = geom.get_size() origin = geom.get_origin() @@ -55,6 +54,6 @@ for name in filenames: table.labels_append(['{}_{}'.format(1+i,'pos') for i in range(3)]+['microstructure']) table.head_write() table.output_flush() - table.data = np.squeeze(np.dstack((xx,yy,zz,microstructure)),axis=0) + table.data = np.squeeze(np.dstack((xx,yy,zz,geom.microstructure.flatten('F'))),axis=0) table.data_writeArray() table.close() diff --git a/processing/pre/geom_translate.py b/processing/pre/geom_translate.py index 488da5103..a63774b22 100755 --- a/processing/pre/geom_translate.py +++ b/processing/pre/geom_translate.py @@ -51,10 +51,9 @@ for name in filenames: damask.util.report(scriptName,name) geom = damask.Geom.from_file(StringIO(''.join(sys.stdin.read())) if name is None else name) - microstructure = geom.get_microstructure() - substituted = np.copy(microstructure) + substituted = geom.get_microstructure() - for old,new in zip(sub[0::2],sub[1::2]): substituted[microstructure==old] = new # substitute microstructure indices + for old,new in zip(sub[0::2],sub[1::2]): substituted[substituted==old] = new # substitute microstructure indices substituted += options.microstructure # constant shift damask.util.croak(geom.update(substituted,origin=geom.get_origin()+options.origin)) diff --git a/processing/pre/geom_vicinityOffset.py b/processing/pre/geom_vicinityOffset.py index e36aac301..bda6b6c5f 100755 --- a/processing/pre/geom_vicinityOffset.py +++ b/processing/pre/geom_vicinityOffset.py @@ -70,18 +70,16 @@ for name in filenames: damask.util.report(scriptName,name) geom = damask.Geom.from_file(StringIO(''.join(sys.stdin.read())) if name is None else name) - microstructure = geom.get_microstructure() - offset = np.nanmax(microstructure) if options.offset is None else options.offset + offset = np.nanmax(geom.microstructure) if options.offset is None else options.offset - microstructure = np.where(ndimage.filters.generic_filter(microstructure, - taintedNeighborhood, - size=1+2*options.vicinity,mode=options.mode, - extra_arguments=(), - extra_keywords={"trigger":options.trigger,"size":1+2*options.vicinity}), - microstructure + offset,microstructure) - - damask.util.croak(geom.update(microstructure)) + damask.util.croak(geom.update(np.where(ndimage.filters.generic_filter( + geom.microstructure, + taintedNeighborhood, + size=1+2*options.vicinity,mode=options.mode, + extra_arguments=(), + extra_keywords={"trigger":options.trigger,"size":1+2*options.vicinity}), + geom.microstructure + offset,geom.microstructure))) geom.add_comments(scriptID + ' ' + ' '.join(sys.argv[1:])) if name is None: