223031012
/
DAMASK_EICMD
mirror of https://github.com/achalhp/DAMASK_EICMD.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fixed ‘too many files open’ issue when processing a large list of input tables.

This commit is contained in:
Philip Eisenlohr 2015-02-11 17:22:47 +00:00
parent eed00007f9
commit 9eae665365
4 changed files with 38 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
processing/post/permuteData.py
View File

@ -18,7 +18,7 @@ Permute all values in given column(s).
""", version = scriptID)
parser.add_option('-l','--label', dest='label', action='extend', metavar='<strig LIST>',
parser.add_option('-l','--label', dest='label', action='extend', metavar='<string LIST>',
help='heading(s) of column to permute')
parser.add_option('-r', '--rnd', dest='randomSeed', type='int', metavar='int',
help='seed of random number generator [%default]')
@ -39,17 +39,13 @@ datainfo = {
if options.label != None: datainfo['scalar']['label'] += options.label
np.random.seed(options.randomSeed)
# ------------------------------------------ setup file handles ------------------------------------
files = []
# --- loop over input files -------------------------------------------------------------------------
for name in filenames:
if os.path.exists(name):
files.append({'name':name, 'input':open(name), 'output':open(name+'_tmp','w'), 'croak':sys.stderr})
#--- loop over input files -------------------------------------------------------------------------
for file in files:
if not os.path.exists(name): continue
file = {'name':name, 'input':open(name), 'output':open(name+'_tmp','w'), 'croak':sys.stderr}
file['croak'].write('\033[1m'+scriptName+'\033[0m: '+file['name']+'\n')
table = damask.ASCIItable(file['input'],file['output'],True) # make unbuffered ASCII_table
table = damask.ASCIItable(file['input'],file['output'],buffered=False) # make unbuffered ASCII_table
table.head_read() # read ASCII header info
table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))

23
processing/post/rotateData.py
View File

@ -49,21 +49,20 @@ toRadians = math.pi/180.0 if options.degrees else 1.0
r = damask.Quaternion().fromAngleAxis(toRadians*options.rotation[0],options.rotation[1:])
R = r.asMatrix()
# ------------------------------------------ setup file handles ------------------------------------
files = []
# --- loop over input files -------------------------------------------------------------------------
if filenames == []:
files.append({'name':'STDIN', 'input':sys.stdin, 'output':sys.stdout, 'croak':sys.stderr})
else:
for name in filenames:
if os.path.exists(name):
files.append({'name':name, 'input':open(name), 'output':open(name+'_tmp','w'), 'croak':sys.stderr})
filenames = ['STDIN']
# ------------------------------------------ loop over input files ---------------------------------
for file in files:
if file['name'] != 'STDIN': file['croak'].write('\033[1m'+scriptName+'\033[0m: '+file['name']+'\n')
else: file['croak'].write('\033[1m'+scriptName+'\033[0m\n')
for name in filenames:
if name == 'STDIN':
file = {'name':'STDIN', 'input':sys.stdin, 'output':sys.stdout, 'croak':sys.stderr}
file['croak'].write('\033[1m'+scriptName+'\033[0m\n')
else:
if not os.path.exists(name): continue
file = {'name':name, 'input':open(name), 'output':open(name+'_tmp','w'), 'croak':sys.stderr}
file['croak'].write('\033[1m'+scriptName+'\033[0m: '+file['name']+'\n')
table = damask.ASCIItable(file['input'],file['output'],False) # make unbuffered ASCII_table
table = damask.ASCIItable(file['input'],file['output'],buffered=False) # make unbuffered ASCII_table
table.head_read() # read ASCII header info
table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))

23
processing/post/scaleData.py
View File

@ -55,21 +55,20 @@ if options.tensor != []: datainfo['tensor']['label'] += options.tensor; lengt
if len(options.factor) != length:
parser.error('length of scaling vector does not match column count...')
# ------------------------------------------ setup file handles ------------------------------------
files = []
# --- loop over input files -------------------------------------------------------------------------
if filenames == []:
files.append({'name':'STDIN', 'input':sys.stdin, 'output':sys.stdout, 'croak':sys.stderr})
else:
for name in filenames:
if os.path.exists(name):
files.append({'name':name, 'input':open(name), 'output':open(name+'_tmp','w'), 'croak':sys.stderr})
filenames = ['STDIN']
# ------------------------------------------ loop over input files ---------------------------------
for file in files:
if file['name'] != 'STDIN': file['croak'].write('\033[1m'+scriptName+'\033[0m: '+file['name']+'\n')
else: file['croak'].write('\033[1m'+scriptName+'\033[0m\n')
for name in filenames:
if name == 'STDIN':
file = {'name':'STDIN', 'input':sys.stdin, 'output':sys.stdout, 'croak':sys.stderr}
file['croak'].write('\033[1m'+scriptName+'\033[0m\n')
else:
if not os.path.exists(name): continue
file = {'name':name, 'input':open(name), 'output':open(name+'_tmp','w'), 'croak':sys.stderr}
file['croak'].write('\033[1m'+scriptName+'\033[0m: '+file['name']+'\n')
table = damask.ASCIItable(file['input'],file['output'],False) # make unbuffered ASCII_table
table = damask.ASCIItable(file['input'],file['output'],buffered=False) # make unbuffered ASCII_table
table.head_read() # read ASCII header info
table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))

23
processing/post/shiftData.py
View File

@ -55,21 +55,20 @@ if options.tensor != []: datainfo['tensor']['label'] += options.tensor; lengt
if len(options.delta) != length:
parser.error('length of offset vector does not match column types...')
# ------------------------------------------ setup file handles ------------------------------------
files = []
# --- loop over input files -------------------------------------------------------------------------
if filenames == []:
files.append({'name':'STDIN', 'input':sys.stdin, 'output':sys.stdout, 'croak':sys.stderr})
else:
for name in filenames:
if os.path.exists(name):
files.append({'name':name, 'input':open(name), 'output':open(name+'_tmp','w'), 'croak':sys.stderr})
filenames = ['STDIN']
# ------------------------------------------ loop over input files ---------------------------------
for file in files:
if file['name'] != 'STDIN': file['croak'].write('\033[1m'+scriptName+'\033[0m: '+file['name']+'\n')
else: file['croak'].write('\033[1m'+scriptName+'\033[0m\n')
for name in filenames:
if name == 'STDIN':
file = {'name':'STDIN', 'input':sys.stdin, 'output':sys.stdout, 'croak':sys.stderr}
file['croak'].write('\033[1m'+scriptName+'\033[0m\n')
else:
if not os.path.exists(name): continue
file = {'name':name, 'input':open(name), 'output':open(name+'_tmp','w'), 'croak':sys.stderr}
file['croak'].write('\033[1m'+scriptName+'\033[0m: '+file['name']+'\n')
table = damask.ASCIItable(file['input'],file['output'],False) # make unbuffered ASCII_table
table = damask.ASCIItable(file['input'],file['output'],buffered=False) # make unbuffered ASCII_table
table.head_read() # read ASCII header info
table.info_append(scriptID + '\t' + ' '.join(sys.argv[1:]))