From 6ac2b4cf88a8a6fd165a063b8ae13991629289ad Mon Sep 17 00:00:00 2001 From: Philip Eisenlohr Date: Fri, 11 Mar 2011 16:32:01 +0000 Subject: [PATCH] deleted useless storage of data resulting in partial speed-up. large datasets (>1 mio elements) cannot be processed at the moment---investigation required, why each element eats about 20kB of memory? --- processing/post/postResults | 149 +++++++++++++++++------------------- 1 file changed, 71 insertions(+), 78 deletions(-) diff --git a/processing/post/postResults b/processing/post/postResults index f88c6baa5..f2eeb44b7 100755 --- a/processing/post/postResults +++ b/processing/post/postResults @@ -61,8 +61,6 @@ class MPIEspectral_result: # mimic py_post result object N_elements = 0 N_element_scalars = 0 N_element_tensors = 0 - theNodes = [] - theElements = [] def __init__(self,filename): @@ -76,23 +74,7 @@ class MPIEspectral_result: # mimic py_post result object self.resolution = self._keyedPackedArray('resolution',3,'i') self.N_nodes = (self.resolution[0]+1)*(self.resolution[1]+1)*(self.resolution[2]+1) self.N_elements = self.resolution[0]*self.resolution[1]*self.resolution[2] - self.dimension = self._keyedPackedArray('dimension',3,'d') - a = self.resolution[0]+1 - b = self.resolution[1]+1 - c = self.resolution[2]+1 - for n in range(self.N_nodes): - self.theNodes.append(vector([self.dimension[0] * (n%a) / self.resolution[0], - self.dimension[1] * ((n/a)%b) / self.resolution[1], - self.dimension[2] * ((n/a/b)%c) / self.resolution[2], - ])) - - for e in range(self.N_elements): - basenode = e+e/self.resolution[0] + e/self.resolution[0]/self.resolution[1]*a - basenode2 = basenode+a*b - self.theElements.append(element([basenode ,basenode+1 ,basenode+a+1 ,basenode+a, - basenode2,basenode2+1,basenode2+a+1,basenode2+a, - ],117)) self.file.seek(0) self.dataOffset = self.file.read(2048).find('eoh')+7 @@ -150,20 +132,33 @@ class MPIEspectral_result: # mimic py_post result object def extrapolation(self,value): self.extrapolate = value - def node_sequence(self,node): - return node + def node_sequence(self,n): + return n - def node_id(self,node): - return node+1 + def node_id(self,n): + return n+1 - def node(self,node): - return self.theNodes[node] + def node(self,n): + a = self.resolution[0]+1 + b = self.resolution[1]+1 + c = self.resolution[2]+1 + return vector([self.dimension[0] * (n%a) / self.resolution[0], + self.dimension[1] * ((n/a)%b) / self.resolution[1], + self.dimension[2] * ((n/a/b)%c) / self.resolution[2], + ]) - def element_id(self,elem): - return elem+1 + def element_id(self,e): + return e+1 - def element(self,elem): - return self.theElements[elem] + def element(self,e): + a = self.resolution[0]+1 + b = self.resolution[1]+1 + c = self.resolution[2]+1 + basenode = e+e/self.resolution[0] + e/self.resolution[0]/self.resolution[1]*a + basenode2 = basenode+a*b + return (element([basenode ,basenode+1 ,basenode+a+1 ,basenode+a, + basenode2,basenode2+1,basenode2+a+1,basenode2+a, + ],117)) def increments(self): return self.N_increments @@ -180,10 +175,10 @@ class MPIEspectral_result: # mimic py_post result object def element_scalars(self): return self.N_element_scalars - def element_scalar(self,elem,idx): - self.file.seek(self.dataOffset+(self.increment*(4+self.N_elements*self.N_element_scalars*8+4) + 4+(elem*self.N_element_scalars + idx)*8)) + def element_scalar(self,e,idx): + self.file.seek(self.dataOffset+(self.increment*(4+self.N_elements*self.N_element_scalars*8+4) + 4+(e*self.N_element_scalars + idx)*8)) value = struct.unpack('d',self.file.read(8))[0] - return [elemental_scalar(node,value) for node in self.theElements[elem].items] + return [elemental_scalar(node,value) for node in self.element(e).items] def element_scalar_label(elem,idx): return 'User Defined Variable %i'%(idx+1) @@ -221,7 +216,7 @@ class backgroundMessage(threading.Thread): self.message = '' self.new_message = '' self.counter = 0 - self.symbols = ['- ', '\ ', '| ', '/ '] + self.symbols = ['- ', '\ ', '| ', '/ ',] self.waittime = 0.5 def __quit__(self): @@ -529,9 +524,10 @@ def ParsePostfile(p,filename, outputFormat): def SummarizePostfile(stat,where=sys.stdout): # ----------------------------- + where.write('\n\n') where.write('title:\t%s'%stat['Title'] + '\n\n') where.write('extraplation:\t%s'%stat['Extrapolation'] + '\n\n') - where.write('increments:\t%i+1'%(stat['NumberOfIncrements']-1) + '\n\n') + where.write('increments:\t%i'%(stat['NumberOfIncrements']) + '\n\n') where.write('nodes:\t%i'%stat['NumberOfNodes'] + '\n\n') where.write('elements:\t%i'%stat['NumberOfElements'] + '\n\n') where.write('nodal scalars:\t%i'%stat['NumberOfNodalScalars'] + '\n\n ' + '\n '.join(stat['LabelOfNodalScalar']) + '\n\n') @@ -625,7 +621,7 @@ parser.set_defaults(inc = False) parser.set_defaults(time = False) parser.set_defaults(separateFiles = False) -(options, file) = parser.parse_args() +(options, files) = parser.parse_args() options.filetype = options.filetype.lower() @@ -658,19 +654,15 @@ else: -bg = backgroundMessage() -bg.start() - - # --- sanity checks -if file == []: +if files == []: parser.print_help() parser.error('no file specified...') -if not os.path.exists(file[0]): +if not os.path.exists(files[0]): parser.print_help() - parser.error('invalid file "%s" specified...'%file[0]) + parser.error('invalid file "%s" specified...'%files[0]) if options.filetype not in ['marc','spectral']: parser.print_help() @@ -686,11 +678,14 @@ if options.nodalScalar and ( options.elementalScalar or options.elementalTenso parser.error('not allowed to mix nodal with elemental results...') +# --- start background messaging + +bg = backgroundMessage() +bg.start() + # --- parse .output and .t16 files -bg.set_message('parsing .output and result files...') - -filename = os.path.splitext(file[0])[0] +filename = os.path.splitext(files[0])[0] dirname = os.path.abspath(os.path.dirname(filename))+os.sep+options.directory if not os.path.isdir(dirname): os.mkdir(dirname,0755) @@ -701,6 +696,9 @@ me = { 'Crystallite': options.cryst, 'Constitutive': options.phase, } + +bg.set_message('parsing .output files...') + for what in me: outputFormat[what] = ParseOutputFormat(filename, what, me[what]) if not '_id' in outputFormat[what]['specials']: @@ -708,9 +706,12 @@ for what in me: print '\n'.join(map(lambda x:' '+x, outputFormat[what]['specials']['brothers'])) sys.exit(1) +bg.set_message('opening result file...') p = OpenPostfile(filename,options.filetype) +bg.set_message('parsing result file...') stat = ParsePostfile(p, filename, outputFormat) - +if options.filetype == 'marc': + stat['NumberOfIncrements'] -= 1 # t16 contains one "virtual" increment (at 0) # --- sanity check for output variables # for mentat variables (nodalScalar,elementalScalar,elementalTensor) we simply have to check whether the label is found in the stat[indexOfLabel] dictionary @@ -729,7 +730,7 @@ for opt in ['nodalScalar','elementalScalar','elementalTensor','homogenizationRes if options.info: if options.filetype == 'marc': - print '\nMentat release %s\n'%release + print '\n\nMentat release %s'%release SummarizePostfile(stat,sys.stderr) @@ -745,12 +746,19 @@ if options.info: # --- get output data from .t16 file -if options.range: - increments = range( max(0,options.range[0]), - min(stat['NumberOfIncrements'],options.range[1]+1), - options.range[2]) +increments = range(stat['NumberOfIncrements']) +if options.filetype == 'marc': + offset_inc = 1 else: - increments = range(stat['NumberOfIncrements']-1) + offset_inc = 0 +if options.range: + options.range = list(options.range) + if options.filetype == 'spectral': + options.range[0] -= 1 + options.range[1] -= 1 + increments = range( max(0,options.range[0]), + min(stat['NumberOfIncrements'],options.range[1]+1), + options.range[2]) fileOpen = False assembleHeader = True @@ -759,40 +767,22 @@ header = [] element_scalar = {} element_tensor = {} -# --- store geometry information - -p.moveto(0) - -nodeID = [ 0 for n in range(stat['NumberOfNodes'])] -nodeCoordinates = [[] for n in range(stat['NumberOfNodes'])] - -elemID = [ 0 for e in range(stat['NumberOfElements'])] -elemNodeID = [[] for e in range(stat['NumberOfElements'])] -ipCoordinates = [[] for e in range(stat['NumberOfElements'])] - -for n in range(stat['NumberOfNodes']): - nodeID[n] = p.node_id(n) - nodeCoordinates[n] = [p.node(n).x, p.node(n).y, p.node(n).z] - -for e in range(stat['NumberOfElements']): - elemID[e] = p.element_id(e) - elemNodeID[e] = p.element(e).items - ipCoordinates[e] = ipCoords(p.element(e).type, map(lambda node: [node.x, node.y, node.z], map(p.node, map(p.node_sequence,p.element(e).items)))) # --- loop over increments time_start = time.time() for incCount,increment in enumerate(increments): - p.moveto(increment+1) - time_delta = (len(increments)-incCount)*(time.time()-time_start)/max(1.0,incCount) - bg.set_message('(%02i:%02i:%02i) read data from increment %i...'%(time_delta//3600,time_delta%3600//60,time_delta%60,increment)) + p.moveto(increment+offset_inc) data = {} if options.nodalScalar: for n in range(stat['NumberOfNodes']): - myNodeID = nodeID[n] - myNodeCoordinates = nodeCoordinates[n] + if n%100 == 0: + time_delta = (len(increments)-incCount)*(time.time()-time_start)/max(1.0,incCount) + bg.set_message('(%02i:%02i:%02i) read node %i from increment %i...'%(time_delta//3600,time_delta%3600//60,time_delta%60,n,increment)) + myNodeID = p.node_id(n) + myNodeCoordinates = [p.node(n).x, p.node(n).y, p.node(n).z] myElemID = 0 myGrainID = 0 @@ -827,9 +817,12 @@ for incCount,increment in enumerate(increments): else: for e in range(stat['NumberOfElements']): - myElemID = elemID[e] - myIpCoordinates = ipCoordinates[e] - for n,myNodeID in enumerate(elemNodeID[e]): + if e%100 == 0: + time_delta = (len(increments)-incCount)*(time.time()-time_start)/max(1.0,incCount) + bg.set_message('(%02i:%02i:%02i) read elem %i from increment %i...'%(time_delta//3600,time_delta%3600//60,time_delta%60,e,increment)) + myElemID = p.element_id(e) + myIpCoordinates = ipCoords(p.element(e).type, map(lambda node: [node.x, node.y, node.z], map(p.node, map(p.node_sequence,p.element(e).items)))) + for n,myNodeID in enumerate(p.element(e).items): for g in range(('GrainCount' in stat['IndexOfLabel'] and int(p.element_scalar(e, stat['IndexOfLabel']['GrainCount'])[0].value)) or 1): myGrainID = g + 1