4123 lines
107 KiB
Bash
4123 lines
107 KiB
Bash
#!/bin/ksh
|
|
##############################################################################
|
|
# #
|
|
# run_marc - run a marc job #
|
|
# ------------------------- #
|
|
# #
|
|
# usage: run_marc -j jid { options } #
|
|
# #
|
|
# where standard options are: required: defaults: #
|
|
# -------------------------- #
|
|
# #
|
|
# -j* jid job id number. ** YES ** . #
|
|
# -pr* prog program name. . marc #
|
|
# -v* y|n do or do not verify inputs. . yes #
|
|
# -q* s|l|v|b|f batch queue name or background, . short #
|
|
# foreground. #
|
|
# -b* as alternative to option -q* #
|
|
# #
|
|
# ( batch queues only : #
|
|
# -pq* intra queue priority. . . #
|
|
# -at DATE/TIME delay start of job. . . #
|
|
# format : January,1,1990,12:31 #
|
|
# or : today,5pm #
|
|
# -cpu* secs job CPU limit . . ) #
|
|
# #
|
|
# -r* rid restart file job id. . . #
|
|
# -si* sid substructure file id. . . #
|
|
# -pi* post post file job id. . . #
|
|
# -de* did defaults file . no #
|
|
# -vf vid viewfactor . no #
|
|
# #
|
|
# -u* user user subroutine. . . #
|
|
# -obj obj user objects or libraries. . . #
|
|
# -sa* y|n do or do not save load module. . no #
|
|
# -autorst auto restart flag for auto forge . no #
|
|
# -me manual remeshing control . no #
|
|
# -ml memory limit in Mbyte #
|
|
# -mo This option is deprecated. As of Marc 2015, only #
|
|
# the integer*8 version is available. #
|
|
# -mpi selects MPI version #
|
|
# each platform has a default MPI version and some #
|
|
# have an alternative version. see the include file #
|
|
# for the respective platform #
|
|
# MPI_DEFAULT defines the default MPI version #
|
|
# MPI_OTHER defines versions one can switch to #
|
|
# -dcoup for contact decoupling #
|
|
# currently not supported #
|
|
# -dir directory where the job i/o should take place. #
|
|
# defaults to current directory. #
|
|
# -sdir directory where scratch files are created #
|
|
# defaults to current directory. #
|
|
# #
|
|
# -alloc only perform memory allocation test, no analysis #
|
|
# -list y only list options in the input file, no analysis #
|
|
# -fe num set feature number "num" for the run. only one allowed #
|
|
# -dytran flag to switch from Dytran to Marc #
|
|
# dytran = 0, program will run w/o Marc-Dytran Switch #
|
|
# = 1, program will restart Marc after Dytran run #
|
|
# >= 2, Not supported yet. #
|
|
# currently not supported #
|
|
# -ou force analysis to use out-of-core control #
|
|
# =0, not used #
|
|
# =1, element storage out-of-core #
|
|
# -dll run marc using shared library libmarc.so and exe_marc #
|
|
# =1, used #
|
|
# =2, do not free streaming input memory #
|
|
# =3, run with marc input deck #
|
|
# -trk run marc for post-tracking #
|
|
# -gpuid run marc using GPGPU capability #
|
|
# specify gpuid on to be used in the analysis. Multiple #
|
|
# IDs may be assigned for DDM runs. #
|
|
# Separate a list of IDs with a colon. Each DMP #
|
|
# process will be assigned a GPU ID in round robin fastion#
|
|
# = 0 #
|
|
# = 0:1 etc... #
|
|
# #
|
|
# where parallel options are: #
|
|
# -------------------------- #
|
|
# #
|
|
# itree, host, and comp options are available for the domain #
|
|
# decomposition only. #
|
|
# MARC_NUMBER_OF_THREADS, nthread, and dir options always available. #
|
|
# #
|
|
# #
|
|
# -nprocd number of domains. #
|
|
# defaults to single domain solution. #
|
|
# -nprocds number of domains if single input file. #
|
|
# defaults to single domain solution. #
|
|
# -nps same as -nprocds. #
|
|
# -nsolver number of solver tasks for solver types 12 and 13 #
|
|
# these are distributed tasks operating via MPI #
|
|
# -nthread_elem number of threads for element assembly and recovery #
|
|
# = 0: use defaults. #
|
|
# defaults to 1 for single domain solution. #
|
|
# defaults to number of domains for multi-domain #
|
|
# solution. #
|
|
# > 1: number of threads to be used by element assembly #
|
|
# recovery. #
|
|
# Also can be set through MARC_NUMBER_OF_THREADS #
|
|
# environment variable. #
|
|
# if both specified, -nthread_elem option will be used. #
|
|
# defaults if neither MARC_NUMBER_OF_THREADS environment #
|
|
# variable set nor -nthread_elem specified. #
|
|
# -nthread_solver number of threads for solver types 6, 8, and 11 #
|
|
# = 0: use defaults. #
|
|
# defaults to 1 for single domain solution. #
|
|
# defaults to number of domains for multi-domain #
|
|
# solution. #
|
|
# > 1: number of threads to be used by 6, 8, and 11 #
|
|
# Also can be set through MARC_NUMBER_OF_THREADS #
|
|
# environment variable. #
|
|
# if both specified, -nthread_solver option will be used. #
|
|
# defaults if neither MARC_NUMBER_OF_THREADS environment #
|
|
# variable set nor -nthread_solver specified. #
|
|
# -nthread Same as -nthread_solver. #
|
|
# -itree message passing tree type for domain decomposition. #
|
|
# for debugging purposes; should not normally be used. #
|
|
# -host hostfile name for distributed execution on network. #
|
|
# defaults to no hostfile, unless jobid.defhost exists. #
|
|
# if jobid.defhost exists, only -np(s) necessary #
|
|
# -comp* y|n to be used with user routines on a network of #
|
|
# incompatible machines. #
|
|
# if set to no, a separate executable will be created #
|
|
# for each machine on the network. #
|
|
# if set to yes, the executable located on the machine #
|
|
# from which marc is started will be used on all machines.#
|
|
# defaults to no if O/S versions different on machines. #
|
|
# #
|
|
# -ci y|n copy input files to remote hosts (default: yes) #
|
|
# if "yes", input files are automatically copied to #
|
|
# remote hosts for a network run if necessary. #
|
|
# -cr y|n copy post files from remote hosts (default: yes) #
|
|
# if "yes", post files are automatically copied back from #
|
|
# remote hosts for a network run if necessary. #
|
|
##############################################################################
|
|
# set DIR to the directory in which this script is
|
|
REALCOM="`/bin/ls -l $0 |awk '{ print $NF; }'`"
|
|
DIR=`dirname $REALCOM`
|
|
# make sure DIR has an absolute path
|
|
case $DIR in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIR=`pwd`/$DIR
|
|
;;
|
|
esac
|
|
DIRSCRIPT=$DIR
|
|
AWK=awk
|
|
ARCH=`uname -a | cut -f 1 -d " "`
|
|
# Sun has a bad awk, use nawk instead
|
|
if test $ARCH = "SunOS"
|
|
then
|
|
AWK=nawk
|
|
fi
|
|
BASENAME=basename
|
|
# Sun has an incorrect /bin/basename, check if /usr/ucb/basename exists
|
|
if test $ARCH = "SunOS"
|
|
then
|
|
if test -x /usr/ucb/basename
|
|
then
|
|
BASENAME=/usr/ucb/basename
|
|
fi
|
|
fi
|
|
|
|
# echo command line in the case of ECHO_COMMAND is true
|
|
if test "$ECHO_COMMAND" = true ; then
|
|
echo command "$0" "$@"
|
|
fi
|
|
|
|
#
|
|
# "mode" selects version, i4 or i8
|
|
# default is i4
|
|
# this can be changed by a file run_marc_defaults
|
|
# located in the tools directory of the Marc installation
|
|
# or in the user's home directory
|
|
# format:
|
|
# MARC_MODE i8
|
|
# it can also be set by the environmental variable MARC_INTEGER_SIZE
|
|
# and by the command line option "-mo"
|
|
#
|
|
mode=
|
|
modeerror=
|
|
modeoption=
|
|
if test -f $DIRSCRIPT/run_marc_defaults; then
|
|
line=`$AWK '{if ($1 == "MARC_MODE") {print $1}}' $DIRSCRIPT/run_marc_defaults`
|
|
if test "$line" = "MARC_MODE"; then
|
|
echo
|
|
echo warning: the option MARC_MODE is deprecated, as of Marc 2015, only the integer*8 version is available
|
|
echo
|
|
line=
|
|
fi
|
|
line=`$AWK '{if ($1 == "MARC_MODE") {print $2}}' $DIRSCRIPT/run_marc_defaults`
|
|
line=`echo $line | $AWK '{print $NF}'`
|
|
if test "$line" = "i4"; then
|
|
modeerror="defaults file $DIRSCRIPT/run_marc_defaults used mode $line ; this must be i8"
|
|
modeoption=error
|
|
echo $modeerror
|
|
fi
|
|
if test "$line" = "i8"; then
|
|
mode=i8
|
|
fi
|
|
fi
|
|
if test -f $HOME/run_marc_defaults; then
|
|
line=`$AWK '{if ($1 == "MARC_MODE") {print $1}}' $HOME/run_marc_defaults`
|
|
if test "$line" = "MARC_MODE"; then
|
|
echo
|
|
echo warning: the option MARC_MODE is deprecated, as of Marc 2015, only the integer*8 version is available
|
|
echo
|
|
line=
|
|
fi
|
|
line=`$AWK '{if ($1 == "MARC_MODE") {print $2}}' $HOME/run_marc_defaults`
|
|
line=`echo $line | $AWK '{print $NF}'`
|
|
if test "$line" = "i4"; then
|
|
modeerror="defaults file $HOME/run_marc_defaults used mode $line ; this must be i8"
|
|
modeoption=error
|
|
echo $modeerror
|
|
fi
|
|
if test "$line" = "i8"; then
|
|
mode=i8
|
|
fi
|
|
fi
|
|
if test -n "$MARC_INTEGER_SIZE" ; then
|
|
mode=$MARC_INTEGER_SIZE
|
|
fi
|
|
if test -z "$mode" ; then
|
|
mode=i8
|
|
fi
|
|
case $mode in
|
|
i4)
|
|
modeerror="bad value for MARC_INTEGER_SIZE variable; only i8 is supported."
|
|
modeoption=error
|
|
echo $modeerror
|
|
;;
|
|
i8)
|
|
MARC_INTEGER_SIZE=i8
|
|
export MARC_INTEGER_SIZE
|
|
;;
|
|
*)
|
|
echo "bad value for MARC_INTEGER_SIZE variable; only i8 is supported."
|
|
exit
|
|
;;
|
|
esac
|
|
|
|
setmode=false
|
|
for arg in $* ; do
|
|
if $setmode ; then
|
|
mode=$arg
|
|
case $mode in
|
|
i4)
|
|
modeerror="bad value for mode option; only i8 is supported."
|
|
modeoption=error
|
|
echo
|
|
echo $modeerror
|
|
echo
|
|
;;
|
|
i8)
|
|
MARC_INTEGER_SIZE=i8
|
|
export MARC_INTEGER_SIZE
|
|
;;
|
|
*)
|
|
echo " "
|
|
echo "error, version mode must be i8"
|
|
echo " "
|
|
echo " use -mo i8 "
|
|
echo " "
|
|
exit
|
|
;;
|
|
esac
|
|
setmode=false
|
|
fi
|
|
if [ ${arg}X = -moX -o ${arg}X = -MOX ] ; then
|
|
echo
|
|
echo warning: the option -mo is deprecated, as of Marc 2015, only the integer*8 version is available
|
|
echo
|
|
setmode=true
|
|
fi
|
|
if [ ${arg}X = -i8X -o ${arg}X = -I8X ] ; then
|
|
MARC_INTEGER_SIZE=i8
|
|
export MARC_INTEGER_SIZE
|
|
fi
|
|
if [ ${arg}X = -i4X -o ${arg}X = -I4X ] ; then
|
|
modeerror="bad value for mode option; only i8 is supported."
|
|
modeoption=error
|
|
echo
|
|
echo $modeerror
|
|
echo
|
|
fi
|
|
done
|
|
|
|
# set to i4 version for 32 bit Linux
|
|
if test "`uname -s`" = "Linux"; then
|
|
if test "`uname -m`" = "i686"; then
|
|
mode=i4
|
|
MARC_INTEGER_SIZE=i4
|
|
export MARC_INTEGER_SIZE
|
|
fi
|
|
fi
|
|
|
|
|
|
. "$DIR/getarch"
|
|
|
|
|
|
# getting user subroutine file name
|
|
found=0
|
|
for i in "$@"; do
|
|
if test $found = 1; then
|
|
DAMASK_USER=$i
|
|
found=0
|
|
fi
|
|
case $i in
|
|
-u* | -U*)
|
|
found=1
|
|
;;
|
|
esac
|
|
done
|
|
# sourcing include_linux64 (needs DAMASK_USER to be set)
|
|
. $MARC_INCLUDE
|
|
|
|
#
|
|
|
|
#
|
|
# Dynamically determine the echo syntax
|
|
#
|
|
|
|
case "`echo '\c'`" in
|
|
'\c')
|
|
ECHO='echo -n'
|
|
ECHOTXT=' '
|
|
;;
|
|
*)
|
|
ECHO='echo'
|
|
ECHOTXT=' \c'
|
|
;;
|
|
esac
|
|
|
|
#
|
|
# Variables for the MARC environment
|
|
#
|
|
|
|
PRODUCT="Marc"
|
|
EXITMSG=$MARC_TOOLS/MESSAGES
|
|
export EXITMSG
|
|
FLEXDIR=$DIR/../flexlm/licenses
|
|
export FLEXDIR
|
|
TIMCHK=3600
|
|
export TIMCHK
|
|
BINDIR=$MARC_BIN
|
|
export BINDIR
|
|
AFMATDAT=$MARC_RUNTIME/AF_flowmat/
|
|
export AFMATDAT
|
|
export MESHERDIR
|
|
MSC_LICENSE_FINPROC=0
|
|
export MSC_LICENSE_FINPROC
|
|
#
|
|
# define directory path to global unified material database
|
|
#
|
|
MATFILE=
|
|
export MATFILE
|
|
|
|
#
|
|
# define memory limit
|
|
# first set to MEMLIMIT from include
|
|
# -ml option overrules if specified
|
|
memlimit=$MEMLIMIT
|
|
#
|
|
# Define share library path based on platforms
|
|
# This is required for using the Patran Mesher
|
|
#
|
|
if test $MACHINENAME = "HP"
|
|
then
|
|
SHLIB_PATH=$MARC_LIB:$MARC_LIB_SHARED:$SHLIB_PATH
|
|
export SHLIB_PATH
|
|
fi
|
|
# the one for IBM is defined futher down
|
|
|
|
LD_LIBRARY_PATH=$MARC_LIB_SHARED:$LD_LIBRARY_PATH
|
|
LD_LIBRARY_PATH=$MARC_LIB:$LD_LIBRARY_PATH
|
|
LD_LIBRARY_PATH=$MESHERDIR:$LD_LIBRARY_PATH
|
|
LD_LIBRARY_PATH=$SFMATDIR:$LD_LIBRARY_PATH
|
|
LD_LIBRARY64_PATH=$MARC_LIB:$LD_LIBRARY64_PATH
|
|
LD_LIBRARYN32_PATH=$MARC_LIB:$LD_LIBRARYN32_PATH
|
|
export LD_LIBRARY_PATH
|
|
export LD_LIBRARY64_PATH
|
|
export LD_LIBRARYN32_PATH
|
|
|
|
atexit() {
|
|
kill -15 $$
|
|
#
|
|
if test $MPITYPE = "myrinet"
|
|
then
|
|
if test -f "$host_filt"
|
|
then
|
|
/bin/rm $host_filt
|
|
fi
|
|
fi
|
|
}
|
|
|
|
trap "atexit" 2
|
|
|
|
#
|
|
# defaults
|
|
#
|
|
|
|
prog=marc
|
|
exefile=marc
|
|
jid=
|
|
rid=
|
|
pid=
|
|
sid=
|
|
did=
|
|
vid=
|
|
user=
|
|
usernoext=
|
|
objs=
|
|
qid=background
|
|
cpu=
|
|
priority=
|
|
att=
|
|
trk=
|
|
verify=yes
|
|
prgsav=no
|
|
rmdll=no
|
|
cpdll=no
|
|
progdll=
|
|
pathdll=
|
|
error=
|
|
nprocd=0
|
|
nprocdddm=1
|
|
nprocdddmprint=
|
|
icreated=0
|
|
nprocdarg=
|
|
nsolver=0
|
|
nsolverarg=-ns
|
|
if test $nprocds
|
|
then
|
|
if test $nprocds -gt 1
|
|
then
|
|
nprocdddm=$nprocds
|
|
nprocdddmprint=$nprocds
|
|
icreated=1
|
|
nprocdarg=-nprocds
|
|
fi
|
|
fi
|
|
ntprint=0
|
|
nt=-1
|
|
nte=-1
|
|
nts=-1
|
|
ntarg=-nt
|
|
ntearg=-nte
|
|
ntsarg=-nts
|
|
nteprint=
|
|
ntsprint=
|
|
gpuids=
|
|
nauto=0
|
|
ndcoup=0
|
|
ndytran=0
|
|
noutcore=0
|
|
dllrun=0
|
|
mesh=0
|
|
itree=0
|
|
iam=
|
|
ddm_arc=0
|
|
link=
|
|
trkrun=0
|
|
DIRJOB=`pwd`
|
|
DIRSCR=$DIRJOB
|
|
DIRSCRSET=
|
|
autoforge=0
|
|
dotdat=.dat
|
|
dotdefhost=.defhost
|
|
host=
|
|
numhost=
|
|
mfile=
|
|
userhost=
|
|
makebdf=
|
|
cpinput=yes
|
|
cpresults=yes
|
|
marcdll=libmarc.$EXT_DLL
|
|
# define hostname and strip off extensions (alpha.aaa.com)
|
|
thishost=`hostname`
|
|
thishost=${thishost%%.*}
|
|
compatible=unknown
|
|
numfield=1
|
|
justlist=
|
|
feature=
|
|
mpioption=false
|
|
iprintsimufact=
|
|
MDSRCLIB=$MARC_LIB/mdsrc.a
|
|
#
|
|
# check run_marc_defaults file for default MPI setting
|
|
# located in the tools directory of the Marc installation
|
|
# or in the user's home directory
|
|
# format:
|
|
# MARC_MPI <mpiversion>
|
|
#
|
|
value=
|
|
file=
|
|
if test -f $DIRSCRIPT/run_marc_defaults; then
|
|
value=`$AWK '{if ($1 == "MARC_MPI") {print $2}}' $DIRSCRIPT/run_marc_defaults`
|
|
value=`echo $value | $AWK '{print $NF}'`
|
|
if test -n "$value"; then
|
|
file=$DIRSCRIPT/run_marc_defaults
|
|
fi
|
|
fi
|
|
if test -f $HOME/run_marc_defaults; then
|
|
value=`$AWK '{if ($1 == "MARC_MPI") {print $2}}' $HOME/run_marc_defaults`
|
|
value=`echo $value | $AWK '{print $NF}'`
|
|
if test -n "$value"; then
|
|
file=$HOME/run_marc_defaults
|
|
fi
|
|
fi
|
|
if test -n "$value"; then
|
|
MARC_MPITYPE=$value
|
|
notok=true
|
|
for i in "$MPI_OTHER"; do
|
|
if test "$MARC_MPITYPE" = "$i"; then
|
|
notok=false
|
|
fi
|
|
done
|
|
if test "$MARC_MPITYPE" = "$MPI_DEFAULT"; then
|
|
notok=false
|
|
fi
|
|
if $notok; then
|
|
echo " "
|
|
echo " error, incorrect option for MARC_MPI"
|
|
echo " defined in $file: $MARC_MPITYPE"
|
|
echo " valid options: $MPI_DEFAULT $MPI_OTHER"
|
|
echo " "
|
|
exit
|
|
fi
|
|
if test "$value" != "$MPI_DEFAULT"; then
|
|
exefile=marc_$value
|
|
. $MARC_INCLUDE
|
|
MDSRCLIB=$MARC_LIB/mdsrc.a_$value
|
|
if test "$MUMPSSOLVER" = MUMPS; then
|
|
MUMPSSOLVERLIBS="$MARC_LIB/libmumps.a_$value"
|
|
fi
|
|
fi
|
|
fi
|
|
#
|
|
#
|
|
# allow scratch directory to be specified with environmental variable
|
|
# MARCSCRATCH
|
|
if test $MARCSCRATCH
|
|
then
|
|
if test -d $MARCSCRATCH
|
|
then
|
|
DIRSCR=$MARCSCRATCH
|
|
else
|
|
echo "error, scratch directory '$MARCSCRATCH'"
|
|
echo " specified via environmental variable MARCSCRATCH does not exist"
|
|
exit
|
|
fi
|
|
fi
|
|
#
|
|
##############################################################################
|
|
# parse input - arguments always come in pairs #
|
|
##############################################################################
|
|
|
|
arg=$1
|
|
if [ ${arg}X = -i8X -o ${arg}X = -I8X ] ; then
|
|
shift
|
|
arg=$1
|
|
fi
|
|
while [ -n "$arg" ]
|
|
do
|
|
shift
|
|
value=$1
|
|
case $arg in
|
|
-al* | -AL*)
|
|
LD_LIBRARY_PATH=$CUDALIB1:$LD_LIBRARY_PATH
|
|
export LD_LIBRARY_PATH
|
|
$MARC_BIN/marc -alloc 1
|
|
exit
|
|
;;
|
|
-li* | -LI*)
|
|
justlist=yes
|
|
;;
|
|
-fe* | -FE*)
|
|
feature=$value
|
|
|
|
;;
|
|
-pr* | -PR*)
|
|
if test `dirname $value` = '.'
|
|
then
|
|
prog=`$BASENAME $value .marc`
|
|
progdll=`$BASENAME $value`
|
|
else
|
|
prog=`dirname $value`/`$BASENAME $value .marc`
|
|
progdll=`dirname $value`/`$BASENAME $value`
|
|
fi
|
|
prdir=`dirname $value`
|
|
case $prdir in
|
|
\/*)
|
|
;;
|
|
*)
|
|
prog=`pwd`/$prdir/$prog
|
|
;;
|
|
esac
|
|
;;
|
|
-j* | -J*)
|
|
jid=`$BASENAME $value $dotdat`
|
|
DIRJID=`dirname $value`
|
|
case $DIRJID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRJID=`pwd`/$DIRJID
|
|
;;
|
|
esac
|
|
;;
|
|
-r* | -R*)
|
|
rid=`$BASENAME $value .t08`
|
|
DIRRID=`dirname $value`
|
|
case $DIRRID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRRID=`pwd`/$DIRRID
|
|
;;
|
|
esac
|
|
;;
|
|
-si* | -SI*)
|
|
sid=$value
|
|
DIRSID=`dirname $value`
|
|
case $DIRSID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRSID=`pwd`/$DIRSID
|
|
;;
|
|
esac
|
|
;;
|
|
-pi* | -PI*)
|
|
if test -f $value.t19
|
|
then
|
|
pid=`$BASENAME $value .t19`
|
|
else
|
|
pid=`$BASENAME $value .t16`
|
|
fi
|
|
DIRPID=`dirname $value`
|
|
case $DIRPID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRPID=`pwd`/$DIRPID
|
|
;;
|
|
esac
|
|
;;
|
|
-bdf | -BDF)
|
|
makebdf=1
|
|
;;
|
|
-de* | -DE*)
|
|
did=`$BASENAME $value $dotdat`
|
|
DIRDID=`dirname $value`
|
|
case $DIRDID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRDID=`pwd`/$DIRDID
|
|
;;
|
|
esac
|
|
;;
|
|
-vf | -VF)
|
|
vid=`$BASENAME $value .vfs`
|
|
DIRVID=`dirname $value`
|
|
case $DIRVID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRVID=`pwd`/$DIRVID
|
|
;;
|
|
esac
|
|
;;
|
|
-u* | -U*)
|
|
user=$value
|
|
case $user in
|
|
\/*)
|
|
;;
|
|
*)
|
|
user=`pwd`/$user
|
|
;;
|
|
esac
|
|
usernoext=$user
|
|
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .f`
|
|
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .F`
|
|
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .for`
|
|
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .f90`
|
|
;;
|
|
-obj | -OBJ)
|
|
objs="$value"
|
|
;;
|
|
-q* | -Q*)
|
|
qid=$value
|
|
;;
|
|
-b* | -B*)
|
|
case $value in
|
|
y* | Y*)
|
|
qid=background
|
|
;;
|
|
n* | N*)
|
|
qid=foreground
|
|
;;
|
|
*)
|
|
;;
|
|
esac
|
|
;;
|
|
-at | -AT)
|
|
att=$value
|
|
;;
|
|
-cpu* | -CPU*)
|
|
cpu=$value
|
|
;;
|
|
-pq | -PQ*)
|
|
priority=$value
|
|
;;
|
|
-v* | -V*)
|
|
verify=$value
|
|
;;
|
|
-sa* | -SA*)
|
|
prgsav=$value
|
|
;;
|
|
-np* | -NP*)
|
|
nprocdddm=$value
|
|
nprocdddmprint=$value
|
|
case $arg in
|
|
-nps* | -NPS* | -nprocds* | -NPROCDS*)
|
|
icreated=1
|
|
nprocdarg=-nprocds
|
|
;;
|
|
esac
|
|
case $arg in
|
|
-np | -NP | -nprocd | -NPROCD)
|
|
icreated=0
|
|
nprocdarg=-nprocd
|
|
;;
|
|
esac
|
|
;;
|
|
-ns* | -NS*)
|
|
nsolver=$value
|
|
;;
|
|
-nt* | -NT*)
|
|
case $arg in
|
|
-nte | -NTE | -nthread_e* | -NTHREAD_E*)
|
|
nte=$value
|
|
;;
|
|
esac
|
|
case $arg in
|
|
-nts | -NTS | -nthread_s* | -NTHREAD_S*)
|
|
nts=$value
|
|
;;
|
|
esac
|
|
case $arg in
|
|
-nt | -NT | -nth* | -NTH* | -nthread* | -NTHREAD*)
|
|
nt=$value
|
|
;;
|
|
esac
|
|
;;
|
|
-gp* | -GP*)
|
|
gpuids=$value
|
|
;;
|
|
-it* | -IT*)
|
|
itree=$value
|
|
;;
|
|
-iam | -IAM)
|
|
iam=$value
|
|
case $value in
|
|
sfg | sfm | sim)
|
|
iprintsimufact=true
|
|
;;
|
|
esac
|
|
;;
|
|
-au* | -AU*)
|
|
nauto=$value
|
|
;;
|
|
-dc* | -DC*)
|
|
ndcoup=$value
|
|
;;
|
|
-dy* | -DY*)
|
|
ndytran=$value
|
|
;;
|
|
-ou* | -OU*)
|
|
noutcore=$value
|
|
;;
|
|
-dll | -DLL)
|
|
dllrun=$value
|
|
;;
|
|
-trk | -TRK)
|
|
trkrun=$value
|
|
;;
|
|
-ddm | -DDM)
|
|
ddm_arc=$value
|
|
;;
|
|
-me | -ME )
|
|
mesh=$value
|
|
;;
|
|
-ml | -ML )
|
|
memlimit=$value
|
|
;;
|
|
-mo | -MO )
|
|
;;
|
|
-mpi | -MPI )
|
|
mpioption=true
|
|
MARC_MPITYPE=$value
|
|
if test "$value" != "$MPI_DEFAULT"; then
|
|
exefile=marc_$value
|
|
. $MARC_INCLUDE
|
|
MDSRCLIB=$MARC_LIB/mdsrc.a_$value
|
|
if test "$MUMPSSOLVER" = MUMPS; then
|
|
MUMPSSOLVERLIBS="$MARC_LIB/libmumps.a_$value"
|
|
fi
|
|
else
|
|
exefile=marc
|
|
. $MARC_INCLUDE
|
|
MDSRCLIB=$MARC_LIB/mdsrc.a
|
|
if test "$MUMPSSOLVER" = MUMPS; then
|
|
MUMPSSOLVERLIBS="$MARC_LIB/libmumps.a"
|
|
fi
|
|
fi
|
|
;;
|
|
-dir* | -DIR*)
|
|
DIRJOB=$value
|
|
case $DIRJOB in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRJOB=`pwd`/$DIRJOB
|
|
;;
|
|
esac
|
|
if test -z "$DIRSCRSET"
|
|
then
|
|
DIRSCR=$DIRJOB
|
|
fi
|
|
;;
|
|
-sd* | -SD*)
|
|
DIRSCR=$value
|
|
DIRSCRSET=yes
|
|
case $DIRSCR in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRSCR=`pwd`/$DIRSCR
|
|
;;
|
|
esac
|
|
;;
|
|
-ho* | -HO*)
|
|
host=$value
|
|
;;
|
|
-co* | -CO*)
|
|
compatible=$value
|
|
;;
|
|
-ci* | -CI*)
|
|
cpinput=$value
|
|
;;
|
|
-cr* | -CR*)
|
|
cpresults=$value
|
|
;;
|
|
*)
|
|
error="$error
|
|
$arg: invalid option"
|
|
break
|
|
;;
|
|
esac
|
|
case $value in
|
|
-*)
|
|
error="$error
|
|
$arg: invalid name $value"
|
|
break
|
|
;;
|
|
esac
|
|
shift
|
|
arg=$1
|
|
if [ ${arg}X = -i8X -o ${arg}X = -I8X -o ${arg}X = -i4X -o ${arg}X = -I4X ] ; then
|
|
shift
|
|
arg=$1
|
|
fi
|
|
done
|
|
argc=`expr $# % 2`
|
|
if test $argc -eq 1
|
|
then
|
|
#
|
|
# odd number of arguments
|
|
#
|
|
error="$error
|
|
argument list incomplete"
|
|
fi
|
|
|
|
if test $nprocdddm -gt 0
|
|
then
|
|
nprocd=$nprocdddm
|
|
fi
|
|
|
|
if test $nsolver -gt 0
|
|
then
|
|
if test $nsolver -gt $nprocd
|
|
then
|
|
nprocd=$nsolver
|
|
fi
|
|
fi
|
|
# Set defaults
|
|
if test $nt -eq -1
|
|
then
|
|
nt=${MARC_NUMBER_OF_THREADS:-0}
|
|
fi
|
|
if test $nt -lt 0
|
|
then
|
|
nt=0
|
|
fi
|
|
if test $nte -eq -1
|
|
then
|
|
nte=${MARC_NUMBER_OF_THREADS:-0}
|
|
fi
|
|
if test $nte -lt 0
|
|
then
|
|
nte=0
|
|
fi
|
|
if test $nts -eq -1
|
|
then
|
|
nts=${MARC_NUMBER_OF_THREADS:-0}
|
|
fi
|
|
if test $nts -lt 0
|
|
then
|
|
nts=0
|
|
fi
|
|
#
|
|
# set number of element loop threads
|
|
#
|
|
ntprint=$nt
|
|
nteprint=$nte
|
|
# copy from -nprocd[s]
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
nteprint=$nprocdddm
|
|
fi
|
|
# override with -nthread_elem option
|
|
if test $nte -ne 0
|
|
then
|
|
nteprint=$nte
|
|
fi
|
|
# check for minimum 1 threads per processes for DDM
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
if test $nteprint -lt $nprocdddm
|
|
then
|
|
nteprint=$nprocdddm
|
|
fi
|
|
fi
|
|
nte=$nteprint
|
|
#
|
|
# set number of Solver threads
|
|
#
|
|
ntsprint=$nts
|
|
# copy from -nthread or -nprocd[s]
|
|
if test $ntprint -ne 0
|
|
then
|
|
ntsprint=$ntprint
|
|
else
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
ntsprint=$nprocdddm
|
|
fi
|
|
fi
|
|
# override with -nthread_solver option
|
|
if test $nts -ne 0
|
|
then
|
|
ntsprint=$nts
|
|
fi
|
|
# check for minimum 1 threads per solver process.
|
|
if test $nsolver -lt $nprocdddm
|
|
then
|
|
if test $ntsprint -lt $nsolver
|
|
then
|
|
ntsprint=$nsolver
|
|
fi
|
|
else
|
|
if test $ntsprint -lt $nprocdddm
|
|
then
|
|
ntsprint=$nprocdddm
|
|
fi
|
|
fi
|
|
if test $ntsprint -eq 1
|
|
then
|
|
set ntsprint=0
|
|
fi
|
|
nts=$ntsprint
|
|
|
|
# set stack size for multi-threading.
|
|
export KMP_MONITOR_STACKSIZE=7M
|
|
export OMP_STACKSIZE=7M
|
|
|
|
#
|
|
# deprecate -nthread option at arugment of marc
|
|
nt=0
|
|
# Reset nprocdddmm, nsolver and threads if not given.
|
|
if test $nprocdddm -eq 0
|
|
then
|
|
nprocdarg=
|
|
fi
|
|
if test $nprocdddm -eq 0
|
|
then
|
|
nprocdddmprint=
|
|
fi
|
|
if test $nprocdddm -eq 0
|
|
then
|
|
nprocdddm=
|
|
fi
|
|
|
|
nsolverprint=$nsolver
|
|
if test $nsolver -eq 0
|
|
then
|
|
nsolverprint=
|
|
fi
|
|
# end of threads setting.
|
|
gpuoption=
|
|
if test "$gpuids" = "" ; then
|
|
gpuoption=
|
|
else
|
|
gpuoption="-gp $gpuids"
|
|
fi
|
|
|
|
if test "$gpuids" = "" ; then
|
|
export LD_LIBRARY_PATH=$CUDALIB1:$LD_LIBRARY_PATH
|
|
else
|
|
MARCCUDALIBS=$MARCCUDALIBS2
|
|
export LD_LIBRARY_PATH=$CUDALIB2:$LD_LIBRARY_PATH
|
|
fi
|
|
# Linux 64 + HPMPI, Below code is taken from include_linux64
|
|
if test $MPITYPE = hpmpi -a "$ARCHITECTURE" = "linux_amd64"
|
|
then
|
|
export MPIHPSPECIAL="$MPIHPSPECIAL -e LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
|
|
fi
|
|
|
|
if test $nprocd -gt 1; then
|
|
if test -f $jid$dotdefhost; then
|
|
if test "$host" = ""; then
|
|
host=$jid$dotdefhost
|
|
fi
|
|
fi
|
|
if test -f hostfile_qa_$nprocd; then
|
|
if test "$host" = ""; then
|
|
host=hostfile_qa_$nprocd
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if test "$dllrun" -gt 0; then
|
|
exefile=exe_marc
|
|
prog=exe_marc
|
|
program=$exefile
|
|
bd=$MARC_BIN/
|
|
if test "$dllrun" -eq 1 || test "$dllrun" -eq 2; then
|
|
dotdat=.inp
|
|
fi
|
|
|
|
if test "$progdll"; then
|
|
/bin/cp ${progdll}_$marcdll $DIRJOB/$marcdll
|
|
rmdll=yes
|
|
pathdll=yes
|
|
progdll=${progdll}_$marcdll
|
|
else
|
|
progdll=$marcdll
|
|
fi
|
|
|
|
if test "$user"; then
|
|
. $MARC_TOOLS/make_marc_user_dll $DIRJOB $user
|
|
user=
|
|
if test $prgsav = no; then
|
|
rmdll=yes
|
|
fi
|
|
if test $prgsav = yes; then
|
|
cpdll=yes
|
|
rmdll=yes
|
|
fi
|
|
pathdll=yes
|
|
fi
|
|
fi
|
|
|
|
##############################################################################
|
|
# check parameter validity #
|
|
##############################################################################
|
|
|
|
while test forever; do
|
|
|
|
#
|
|
# check for input file existence
|
|
#
|
|
if test $nprocdddm -gt 1 -a $icreated -eq 0; then
|
|
if test ! -f $DIRJID/1$jid$dotdat; then
|
|
if test "$jid" != "" ; then
|
|
error="$error
|
|
input file $DIRJID/1$jid$dotdat not accessible"
|
|
fi
|
|
fi
|
|
else
|
|
if test ! -f $DIRJID/$jid$dotdat; then
|
|
if test "$jid" != "" ; then
|
|
error="$error
|
|
input file $DIRJID/$jid$dotdat not accessible"
|
|
fi
|
|
fi
|
|
fi
|
|
if test $nprocd -gt 1; then
|
|
if test "$host" ; then
|
|
if test ! -f $host; then
|
|
error="$error
|
|
host name file $host not accessible"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
#
|
|
# check if the job is already running in the background
|
|
#
|
|
if test -f $DIRJOB/$jid.pid; then
|
|
error="$error
|
|
job is already running (the file $jid.pid exists)"
|
|
fi
|
|
|
|
#
|
|
# if the program name is other than marc, then
|
|
# assume that this is a program in the users local directory
|
|
#
|
|
|
|
bd=$MARC_BIN/
|
|
|
|
case $prog in
|
|
marc | MARC | $exefile)
|
|
program=$exefile
|
|
if test "$rid"
|
|
then
|
|
if test ! -f $DIRRID/$rid.t08
|
|
then
|
|
error="$error
|
|
restart file $DIRRID/$rid.t08 not accessible"
|
|
fi
|
|
fi
|
|
if test "$pid"
|
|
then
|
|
if test ! -f $DIRPID/$pid.t16
|
|
then
|
|
if test ! -f $DIRPID/$pid.t19
|
|
then
|
|
error="$error
|
|
post file $DIRPID/$pid.t16 or $DIRPID/$pid.t19 not accessible"
|
|
fi
|
|
fi
|
|
fi
|
|
if test "$user"
|
|
then
|
|
if test ! -f $user
|
|
then
|
|
error="$error
|
|
user subroutine file $user not accessible"
|
|
fi
|
|
fi
|
|
if test "$objs"
|
|
then
|
|
missingobjs=
|
|
for o in $objs
|
|
do
|
|
if test ! -f "$o"
|
|
then
|
|
if test -z "$missingobjs"
|
|
then
|
|
missingobjs="$o"
|
|
else
|
|
missingobjs="$missingobjs $o"
|
|
fi
|
|
fi
|
|
done
|
|
if test -n "$missingobjs"
|
|
then
|
|
error="$error
|
|
user object/library file(s) $missingobjs not accessible"
|
|
fi
|
|
fi
|
|
if test "$did"
|
|
then
|
|
if test $nprocdddm -gt 1 -a $icreated -eq 0
|
|
then
|
|
if test ! -f $DIRDID/1$did$dotdat
|
|
then
|
|
error="$error
|
|
defaults file $DIRDID/1$did$dotdat not accessible"
|
|
fi
|
|
else
|
|
if test ! -f $DIRDID/$did$dotdat
|
|
then
|
|
error="$error
|
|
defaults file $DIRDID/$did$dotdat not accessible"
|
|
fi
|
|
fi
|
|
fi
|
|
if test "$vid"
|
|
then
|
|
if test $nprocdddm -gt 1 -a $icreated -eq 0
|
|
then
|
|
if test ! -f $DIRVID/1$vid.vfs
|
|
then
|
|
error="$error
|
|
view factor file $DIRVID/1$vid.vfs not accessible"
|
|
fi
|
|
else
|
|
if test ! -f $DIRVID/$vid.vfs
|
|
then
|
|
error="$error
|
|
view factor file $DIRVID/$vid.vfs not accessible"
|
|
fi
|
|
fi
|
|
fi
|
|
if $mpioption
|
|
then
|
|
notok=true
|
|
for i in "$MPI_OTHER"; do
|
|
if test "$MARC_MPITYPE" = "$i"; then
|
|
notok=false
|
|
fi
|
|
done
|
|
if test "$MARC_MPITYPE" = "$MPI_DEFAULT"; then
|
|
notok=false
|
|
fi
|
|
if $notok; then
|
|
error="$error
|
|
incorrect option for -mpi option: $MARC_MPITYPE (valid: $MPI_OTHER)"
|
|
fi
|
|
fi
|
|
;;
|
|
*)
|
|
program=$prog.marc
|
|
case $prog in
|
|
\/* | \.\/*)
|
|
bd=
|
|
;;
|
|
*)
|
|
bd=`pwd`/
|
|
;;
|
|
esac
|
|
if test "$rid"
|
|
then
|
|
if test ! -f $DIRRID/$rid.t08
|
|
then
|
|
error="$error
|
|
restart file $DIRRID/$rid.t08 not accessible"
|
|
fi
|
|
fi
|
|
if test "$pid"
|
|
then
|
|
if test ! -f $DIRPID/$pid.t16
|
|
then
|
|
if test ! -f $DIRPID/$pid.t19
|
|
then
|
|
error="$error
|
|
post file $DIRPID/$pid.t16 and $DIRPID/$pid.t19 not accessible"
|
|
fi
|
|
fi
|
|
fi
|
|
if test "$user"
|
|
then
|
|
error="$error
|
|
program option may not be used with user subroutine"
|
|
fi
|
|
if test "$objs"
|
|
then
|
|
error="$error
|
|
program option may not be used with user objects or libraries"
|
|
fi
|
|
if test "$did"
|
|
then
|
|
if test $nprocdddm -gt 1 -a $icreated -eq 0
|
|
then
|
|
if test ! -f $DIRDID/1$did$dotdat
|
|
then
|
|
error="$error
|
|
defaults file $DIRDID/1$did$dotdat not accessible"
|
|
fi
|
|
else
|
|
if test ! -f $DIRDID/$did$dotdat
|
|
then
|
|
error="$error
|
|
defaults file $DIRDID/$did$dotdat not accessible"
|
|
fi
|
|
fi
|
|
fi
|
|
if test "$nauto"
|
|
then
|
|
if test $nauto -gt 2
|
|
then
|
|
error="$error
|
|
incorrect option for auto restart "
|
|
fi
|
|
fi
|
|
if test "$ndcoup"
|
|
then
|
|
if test $ndcoup -gt 3
|
|
then
|
|
error="$error
|
|
incorrect option for contact decoupling "
|
|
fi
|
|
fi
|
|
if test "$ndytran"
|
|
then
|
|
if test $ndytran -gt 1
|
|
then
|
|
error="$error
|
|
incorrect option for Marc-Dytran Switch "
|
|
fi
|
|
fi
|
|
if $mpioption
|
|
then
|
|
if test ! -x $MARC_BIN/$exefile
|
|
then
|
|
error="$error
|
|
incorrect option for -mpi option: $MARC_MPITYPE "
|
|
fi
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
##############################################################################
|
|
# check argument integrity #
|
|
##############################################################################
|
|
|
|
if test "$jid"
|
|
then
|
|
:
|
|
else
|
|
if test "$user"
|
|
then
|
|
# allow user sub without giving job id
|
|
qid=foreground
|
|
verify=no
|
|
else
|
|
error="$error
|
|
job id required"
|
|
fi
|
|
fi
|
|
|
|
if test $nprocd -gt 1
|
|
then
|
|
if test $nauto -gt 0
|
|
then
|
|
error="$error
|
|
cannot run DDM job with auto restart (-au) option "
|
|
fi
|
|
fi
|
|
case $qid in
|
|
S* | s*)
|
|
qid=short
|
|
;;
|
|
L* | l*)
|
|
qid=long
|
|
;;
|
|
V* | v*)
|
|
qid=verylong
|
|
;;
|
|
B* | b*)
|
|
qid=background
|
|
;;
|
|
F* | f*)
|
|
qid=foreground
|
|
;;
|
|
A* | a*)
|
|
qid=at
|
|
;;
|
|
*)
|
|
error="$error
|
|
bad value for queue_id option"
|
|
;;
|
|
esac
|
|
|
|
case $prgsav in
|
|
N* | n*)
|
|
prgsav=no
|
|
;;
|
|
Y* | y*)
|
|
prgsav=yes
|
|
;;
|
|
*)
|
|
error="$error
|
|
bad value for save option"
|
|
;;
|
|
esac
|
|
|
|
case $verify in
|
|
N* | n*)
|
|
verify=no
|
|
;;
|
|
Y* | y*)
|
|
verify=yes
|
|
;;
|
|
*)
|
|
error="$error
|
|
bad value for verify option"
|
|
;;
|
|
esac
|
|
|
|
case $nprocdddm in
|
|
-* )
|
|
error="$error
|
|
bad value for nprocd option"
|
|
;;
|
|
esac
|
|
|
|
case $nt in
|
|
-* )
|
|
error="$error
|
|
bad value for nt option"
|
|
;;
|
|
esac
|
|
|
|
case $itree in
|
|
-* )
|
|
error="$error
|
|
bad value for itree option"
|
|
;;
|
|
esac
|
|
case $iam in
|
|
-* )
|
|
error="$error
|
|
bad value for iam option"
|
|
;;
|
|
esac
|
|
case $compatible in
|
|
N* | n*)
|
|
compatible=no
|
|
;;
|
|
Y* | y*)
|
|
compatible=yes
|
|
;;
|
|
unknown)
|
|
;;
|
|
*)
|
|
error="$error
|
|
bad value for comp option"
|
|
;;
|
|
esac
|
|
case $cpinput in
|
|
N* | n*)
|
|
cpinput=no
|
|
;;
|
|
Y* | y*)
|
|
cpinput=yes
|
|
;;
|
|
*)
|
|
error="$error
|
|
bad value for copy input option"
|
|
;;
|
|
esac
|
|
case $cpresults in
|
|
N* | n*)
|
|
cpresults=no
|
|
;;
|
|
Y* | y*)
|
|
cpresults=yes
|
|
;;
|
|
*)
|
|
error="$error
|
|
bad value for copy results option"
|
|
;;
|
|
esac
|
|
|
|
#
|
|
# check for external file to run
|
|
#
|
|
if test -f $MARC_TOOLS/run_marc_check
|
|
then
|
|
. $MARC_TOOLS/run_marc_check
|
|
fi
|
|
|
|
##############################################################################
|
|
# interact with the user to get the required information to run marc or #
|
|
# other marc system program #
|
|
##############################################################################
|
|
|
|
deletelog=yes
|
|
if test $qid = background -a $verify = no
|
|
then
|
|
echo \
|
|
"
|
|
Program name : $prog
|
|
Marc shared lib : $progdll
|
|
Version type : $mode
|
|
Job ID : $DIRJID/$jid
|
|
User subroutine name : $user
|
|
User objects/libs : $objs
|
|
Restart file job ID : $rid
|
|
Substructure file ID : $sid
|
|
Post file job ID : $pid
|
|
Defaults file ID : $did
|
|
View Factor file ID : $vid
|
|
Save generated module: $prgsav
|
|
MPI library : $MPITYPE
|
|
DDM processes : $nprocdddmprint
|
|
Element loop threads : $nteprint
|
|
Solver processes : $nsolverprint
|
|
Solver threads : $ntsprint
|
|
GPGPU option : $gpuids
|
|
Host file name : $host" > $jid.log
|
|
if test "$iprintsimufact" = true ; then
|
|
echo "DDM with ARC Mapper : $ddm_arc" >> $jid.log
|
|
fi
|
|
echo \
|
|
"Message passing type : $itree
|
|
Run job in queue : $qid
|
|
Run directory : $DIRJOB
|
|
Scratch directory : $DIRSCR
|
|
Memory limit in Mbyte: $memlimit
|
|
Auto Restart : $nauto " >> $jid.log
|
|
deletelog=no
|
|
fi
|
|
echo \
|
|
"
|
|
Program name : $prog
|
|
Marc shared lib : $progdll
|
|
Version type : $mode
|
|
Job ID : $DIRJID/$jid
|
|
User subroutine name : $user
|
|
User objects/libs : $objs
|
|
Restart file job ID : $rid
|
|
Substructure file ID : $sid
|
|
Post file job ID : $pid
|
|
Defaults file ID : $did
|
|
View Factor file ID : $vid
|
|
Save generated module: $prgsav
|
|
MPI library : $MPITYPE
|
|
DDM processes : $nprocdddmprint
|
|
Element loop threads : $nteprint
|
|
Solver processes : $nsolverprint
|
|
Solver threads : $ntsprint"
|
|
if test "$iprintsimufact" = true ; then
|
|
echo "DDM with ARC Mapper : $ddm_arc"
|
|
fi
|
|
echo \
|
|
"GPGPU option : $gpuids
|
|
Host file name : $host
|
|
Message passing type : $itree
|
|
Run job in queue : $qid
|
|
Run directory : $DIRJOB
|
|
Scratch directory : $DIRSCR
|
|
Memory limit in Mbyte: $memlimit
|
|
Auto Restart : $nauto"
|
|
|
|
|
|
case $qid in
|
|
s* | S* | l* | L* | v* | V* )
|
|
echo \
|
|
"Queue priority : $priority
|
|
Queue CPU limit : $cpu
|
|
Queue start time : $att"
|
|
;;
|
|
# * )
|
|
# echo \
|
|
#" "
|
|
# ;;
|
|
esac
|
|
|
|
if test "$modeoption"
|
|
then
|
|
error=$modeerror
|
|
fi
|
|
|
|
if test "$error"
|
|
then
|
|
if test $verify = yes
|
|
then
|
|
$ECHO "$error
|
|
|
|
Please correct or quit(correct,quit,<correct>): $ECHOTXT"
|
|
error=
|
|
read answer
|
|
case $answer in
|
|
q* | Q*)
|
|
answer=quit
|
|
;;
|
|
*)
|
|
answer=correct
|
|
;;
|
|
esac
|
|
else
|
|
$ECHO "$error
|
|
$ECHOTXT"
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
$ECHO "$error
|
|
$ECHOTXT" >> $jid.log
|
|
echo " " >> $jid.log
|
|
fi
|
|
answer=quit
|
|
fi
|
|
else
|
|
if test $verify = yes
|
|
then
|
|
$ECHO "
|
|
Are these parameters correct (yes,no,quit,<no>)? $ECHOTXT"
|
|
read answer
|
|
case $answer in
|
|
q* | Q*)
|
|
answer=quit
|
|
;;
|
|
y* | Y*)
|
|
answer=yes
|
|
;;
|
|
*)
|
|
answer=no
|
|
;;
|
|
esac
|
|
else
|
|
answer=yes
|
|
fi
|
|
fi
|
|
|
|
case $answer in
|
|
no | correct)
|
|
|
|
##############################################################################
|
|
# prompt for each value #
|
|
##############################################################################
|
|
|
|
$ECHO "
|
|
Program name ($prog)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
prog=$value
|
|
fi
|
|
$ECHO "Job ID ($jid)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
jid=`$BASENAME $value $dotdat`
|
|
DIRJID=`dirname $value`
|
|
case $DIRJID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRJID=`pwd`/$DIRJID
|
|
;;
|
|
esac
|
|
fi
|
|
$ECHO "User subroutine name ($user)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
case $value in
|
|
-*)
|
|
user=
|
|
;;
|
|
*)
|
|
user=$value
|
|
case $user in
|
|
\/*)
|
|
;;
|
|
*)
|
|
user=`pwd`/$user
|
|
;;
|
|
esac
|
|
usernoext=$user
|
|
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .f`
|
|
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .F`
|
|
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .for`
|
|
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .f90`
|
|
;;
|
|
esac
|
|
fi
|
|
$ECHO "User objects or libraries ($objs)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
case $value in
|
|
-*)
|
|
objs=
|
|
;;
|
|
*)
|
|
objs="$value"
|
|
;;
|
|
esac
|
|
fi
|
|
$ECHO "Restart File Job ID ($rid)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
case $value in
|
|
-*)
|
|
rid=
|
|
;;
|
|
*)
|
|
rid=`$BASENAME $value .t08`
|
|
DIRRID=`dirname $value`
|
|
case $DIRRID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRRID=`pwd`/$DIRRID
|
|
;;
|
|
esac
|
|
;;
|
|
esac
|
|
fi
|
|
$ECHO "Substructure File ID ($sid)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
case $value in
|
|
-*)
|
|
sid=
|
|
;;
|
|
*)
|
|
sid=$value
|
|
DIRSID=`dirname $value`
|
|
case $DIRSID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRSID=`pwd`/$DIRSID
|
|
;;
|
|
esac
|
|
;;
|
|
esac
|
|
fi
|
|
$ECHO "Post File Job ID ($pid)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
case $value in
|
|
-*)
|
|
pid=
|
|
;;
|
|
*)
|
|
pid=$value
|
|
DIRPID=`dirname $value`
|
|
case $DIRPID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRPID=`pwd`/$DIRPID
|
|
;;
|
|
esac
|
|
;;
|
|
esac
|
|
fi
|
|
$ECHO "Defaults File ID ($did)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
case $value in
|
|
-*)
|
|
did=
|
|
;;
|
|
*)
|
|
did=`$BASENAME $value $dotdat`
|
|
DIRDID=`dirname $value`
|
|
case $DIRDID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRDID=`pwd`/$DIRDID
|
|
;;
|
|
esac
|
|
;;
|
|
esac
|
|
fi
|
|
$ECHO "View Factor File ID ($vid)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
case $value in
|
|
-*)
|
|
vid=
|
|
;;
|
|
*)
|
|
vid=`$BASENAME $value .vfs`
|
|
DIRVID=`dirname $value`
|
|
case $DIRVID in
|
|
\/*)
|
|
;;
|
|
*)
|
|
DIRVID=`pwd`/$DIRVID
|
|
;;
|
|
esac
|
|
;;
|
|
esac
|
|
fi
|
|
$ECHO "Save generated module ($prgsav)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
prgsav=$value
|
|
fi
|
|
$ECHO "Run on tasks ($nprocdddm) tasks? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
nprocdddm=$value
|
|
nprocdddmprint=$value
|
|
fi
|
|
$ECHO "Run on ($nte) Element loop threads ? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
nte=$value
|
|
fi
|
|
$ECHO "Run on ($nsolver) solvers ? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
nsolver=$value
|
|
fi
|
|
$ECHO "Run on ($nts) Solver threads ? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
nts=$value
|
|
fi
|
|
#
|
|
if test $nprocdddm -gt 0
|
|
then
|
|
nprocd=$nprocdddm
|
|
fi
|
|
if test $nsolver -gt 0
|
|
then
|
|
if test $nsolver -gt $nprocd
|
|
then
|
|
nprocd=$nsolver
|
|
fi
|
|
fi
|
|
# Element loop threads.
|
|
if test $nte -eq -1
|
|
then
|
|
nte=${MARC_NUMBER_OF_THREADS:-0}
|
|
fi
|
|
if test $nte -lt 0
|
|
then
|
|
nte=0
|
|
fi
|
|
nteprint=$nte
|
|
# Copy from ddm
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
nteprint=$nprocdddm
|
|
fi
|
|
# override with -nthread_elem option
|
|
if test $nte -ne 0
|
|
then
|
|
nteprint=$nte
|
|
fi
|
|
# check for minimum 1 threads per processes for DDM
|
|
if test $nprocdddm -ne 0
|
|
then
|
|
if test $nteprint -lt $nprocdddm
|
|
then
|
|
nteprint=$nprocdddm
|
|
fi
|
|
fi
|
|
nte=$nteprint
|
|
# Solver threads.
|
|
if test $nts -eq -1
|
|
then
|
|
nts=${MARC_NUMBER_OF_THREADS:-0}
|
|
fi
|
|
if test $nts -lt 0
|
|
then
|
|
nts=0
|
|
fi
|
|
ntsprint=$nts
|
|
# Copy from ddm
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
ntsprint=$nprocdddm
|
|
fi
|
|
# override with -nthread_solver option
|
|
if test $nts -ne 0
|
|
then
|
|
ntsprint=$nts
|
|
fi
|
|
# check for minimum 1 threads per solver process.
|
|
if test $nsolver -lt $nprocdddm
|
|
then
|
|
if test $ntsprint -lt $nsolver
|
|
then
|
|
ntsprint=$nsolver
|
|
fi
|
|
else
|
|
if test $ntsprint -lt $nprocdddm
|
|
then
|
|
ntsprint=$nprocdddm
|
|
fi
|
|
fi
|
|
if test $ntsprint -eq 1
|
|
then
|
|
set ntsprint=0
|
|
fi
|
|
nts=$ntsprint
|
|
# Update print variable for -nsolver option
|
|
nsolverprint=$nsolver
|
|
if test $nsolver -eq 0
|
|
then
|
|
nsolverprint=
|
|
fi
|
|
$ECHO "GPGPU id option ($gpuids)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
gpuids=$value
|
|
fi
|
|
if test "$gpuids" = "" ; then
|
|
gpuoption=
|
|
else
|
|
gpuoption="-gp $gpuids"
|
|
fi
|
|
if test "$gpuids" = "" ; then
|
|
export LD_LIBRARY_PATH=$CUDALIB1:$LD_LIBRARY_PATH
|
|
else
|
|
MARCCUDALIBS=$MARCCUDALIBS2
|
|
export LD_LIBRARY_PATH=$CUDALIB2:$LD_LIBRARY_PATH
|
|
fi
|
|
if test $MPITYPE = hpmpi -a "$ARCHITECTURE" = "linux_amd64"
|
|
then
|
|
export MPIHPSPECIAL="$MPIHPSPECIAL -e LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
|
|
fi
|
|
#
|
|
if test $nprocd -gt 1
|
|
then
|
|
$ECHO "Message passing type ($itree)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
itree=$value
|
|
fi
|
|
$ECHO "Host file name ($host)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
host=$value
|
|
fi
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
$ECHO "Single input file? $ECHOTXT"
|
|
read value
|
|
case $value in
|
|
y* | Y*)
|
|
icreated=1
|
|
nprocdarg=-nprocds
|
|
;;
|
|
esac
|
|
$ECHO "Compatible machines for DDM ($compatible)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
compatible=$value
|
|
fi
|
|
$ECHO "Copy input files to remote hosts ($cpinput)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
cpinput=$value
|
|
fi
|
|
$ECHO "Copy post files from remote hosts ($cpresults)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
cpresults=$value
|
|
fi
|
|
fi
|
|
fi
|
|
$ECHO "Run the job in the queue ($qid)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
qid=$value
|
|
fi
|
|
case $qid in
|
|
s* | S* | l* | L* | v* | V* )
|
|
$ECHO "Queue priority ($priority)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
priority=$value
|
|
fi
|
|
$ECHO "Job starts at ($att)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
att=$value
|
|
fi
|
|
$ECHO "Queue CPU limit ($cpu)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
cpu=$value
|
|
fi
|
|
;;
|
|
* )
|
|
;;
|
|
esac
|
|
$ECHO "Auto Restart option ($nauto)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
nauto=$value
|
|
fi
|
|
$ECHO "Run directory ($DIRJOB)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
DIRJOB=$value
|
|
DIRSCR=$DIRJOB
|
|
fi
|
|
$ECHO "Scratch directory ($DIRSCR)? $ECHOTXT"
|
|
read value
|
|
if test "$value"
|
|
then
|
|
DIRSCR=$value
|
|
fi
|
|
;;
|
|
quit)
|
|
exit 1
|
|
;;
|
|
*)
|
|
break
|
|
;;
|
|
|
|
esac
|
|
|
|
if test $nt -eq -1
|
|
then
|
|
nt=${MARC_NUMBER_OF_THREADS:-0}
|
|
fi
|
|
if test $nt -lt 0
|
|
then
|
|
nt=0
|
|
fi
|
|
|
|
done
|
|
#
|
|
if test $nt -eq 0
|
|
then
|
|
ntarg=
|
|
fi
|
|
if test $nt -eq 0
|
|
then
|
|
ntprint=
|
|
fi
|
|
if test $nt -eq 0
|
|
then
|
|
nt=
|
|
fi
|
|
|
|
if test $nte -eq 0
|
|
then
|
|
ntearg=
|
|
fi
|
|
if test $nte -eq 0
|
|
then
|
|
nteprint=
|
|
fi
|
|
if test $nte -eq 0
|
|
then
|
|
nte=
|
|
fi
|
|
|
|
if test $nts -eq 0
|
|
then
|
|
ntsarg=
|
|
fi
|
|
if test $nts -eq 0
|
|
then
|
|
ntsprint=
|
|
fi
|
|
if test $nts -eq 0
|
|
then
|
|
nts=
|
|
fi
|
|
#
|
|
if test "$dllrun" -gt 0; then
|
|
exefile=exe_marc
|
|
prog=exe_marc
|
|
program=$exefile
|
|
bd=$MARC_BIN/
|
|
if test "$user"; then
|
|
. $MARC_TOOLS/make_marc_user_dll $DIRJOB $user
|
|
user=
|
|
pathdll=yes
|
|
if test $prgsav = no; then
|
|
rmdll=yes
|
|
fi
|
|
if test $prgsav = yes; then
|
|
cpdll=yes
|
|
rmdll=yes
|
|
fi
|
|
fi
|
|
|
|
if test "$pathdll"; then
|
|
#
|
|
# reset share lib path
|
|
#
|
|
if test $MACHINENAME = "HP"
|
|
then
|
|
SHLIB_PATH=$DIRJOB:$SHLIB_PATH
|
|
export SHLIB_PATH
|
|
fi
|
|
if test $MACHINENAME = "IBM"
|
|
then
|
|
LIBPATH=$DIRJOB:$LIBPATH
|
|
export LIBPATH
|
|
fi
|
|
#
|
|
LD_LIBRARY_PATH=$DIRJOB:$LD_LIBRARY_PATH
|
|
LD_LIBRARY64_PATH=$DIRJOB:$LD_LIBRARY64_PATH
|
|
LD_LIBRARYN32_PATH=$DIRJOB:$LD_LIBRARYN32_PATH
|
|
export LD_LIBRARY_PATH
|
|
export LD_LIBRARY64_PATH
|
|
export LD_LIBRARYN32_PATH
|
|
fi
|
|
fi
|
|
# end of dllrun>0
|
|
|
|
|
|
if test $program = $exefile -o $program = $prog.marc
|
|
then
|
|
|
|
# delete the old .log file unless we run in the background
|
|
if test "$deletelog" = yes
|
|
then
|
|
if test "$jid"
|
|
then
|
|
/bin/rm $jid.log 2>/dev/null
|
|
fi
|
|
else
|
|
echo
|
|
echo running the job in the background, see $jid.log
|
|
echo
|
|
fi
|
|
|
|
#
|
|
# check if this is an autoforge or rezoning or radiation job
|
|
#
|
|
if test $nprocd -eq 1 -a "$jid"
|
|
|
|
then
|
|
line=`$AWK '/^[eE][nN][dD]/ {exit} ; {print}' $DIRJID/${jid}$dotdat | grep -i "^autoforge"`
|
|
if test "$line"
|
|
then
|
|
autoforge=1
|
|
fi
|
|
line=`$AWK '/^[eE][nN][dD]/ {exit} ; {print}' $DIRJID/${jid}$dotdat | grep -i "^rezoning"`
|
|
if test "$line"
|
|
then
|
|
autoforge=1
|
|
fi
|
|
line=`$AWK '/^[eE][nN][dD]/ {exit} ; {print}' $DIRJID/${jid}$dotdat | grep -i "^radiation"`
|
|
if test "$line"
|
|
then
|
|
autoforge=1
|
|
fi
|
|
fi
|
|
#
|
|
# check that jobname for restarted run is not the same
|
|
# as restart file basename
|
|
#
|
|
if test "$rid"
|
|
then
|
|
if test "$jid" = "$rid"
|
|
then
|
|
echo " "
|
|
echo "ERROR: job name of current run is the same as job name"
|
|
echo " of the restarted job"
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
echo "ERROR: job name of current run is the same as job name" >> $jid.log
|
|
echo " of the restarted job" >> $jid.log
|
|
echo " " >> $jid.log
|
|
echo " Exit number 8" >> $jid.log
|
|
echo " " >> $jid.log
|
|
fi
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
#
|
|
# user objects/libraries used
|
|
#
|
|
|
|
if test "$objs"
|
|
then
|
|
program="$DIRJOB/$jid.marc"
|
|
case $program in
|
|
\/* | \.\/*)
|
|
bd=
|
|
;;
|
|
*)
|
|
bd=`pwd`/
|
|
;;
|
|
esac
|
|
link=yes
|
|
fi
|
|
|
|
#
|
|
# user subroutine used
|
|
#
|
|
# add DAMASK options for linking
|
|
DAMASK="-lstdc++"
|
|
|
|
if test "$user"
|
|
then
|
|
program=$usernoext.marc
|
|
case $program in
|
|
\/* | \.\/*)
|
|
bd=
|
|
;;
|
|
*)
|
|
bd=`pwd`/
|
|
;;
|
|
esac
|
|
link=yes
|
|
fi
|
|
|
|
#
|
|
# Special case for IBM using POE but not an SP machine
|
|
# in this case we always need a host file, also for serial jobs.
|
|
#
|
|
if test $MACHINENAME = IBM -a $MPITYPE = hardware -a "$MACHINETYPE" = NONSP
|
|
then
|
|
MP_HOSTFILE=${jid}.host
|
|
if test -f $jid.host
|
|
then
|
|
/bin/rm $jid.host 2> /dev/null
|
|
fi
|
|
if test $nprocd -gt 1
|
|
then
|
|
numdom=$nprocd
|
|
while test $numdom -gt 0
|
|
do
|
|
hostname -s >> $MP_HOSTFILE
|
|
numdom=`echo $numdom | $AWK '{sum=$1-1}; {print sum}'`
|
|
done
|
|
else
|
|
hostname -s > $MP_HOSTFILE
|
|
fi
|
|
fi
|
|
#
|
|
# check ssh for all hosts in host file
|
|
#
|
|
if test $nprocd -gt 1
|
|
then
|
|
if test $MPITYPE = "intelmpi" -a "$INTELMPI_VERSION" = "HYDRA"
|
|
then
|
|
# get host list
|
|
if test "$host"
|
|
then
|
|
line=`grep -v '^#' $host | $AWK '{host=$1;num=$2;for (i=1;i<=num;i++) print host}' | uniq`
|
|
# count failing hosts
|
|
counter=0
|
|
for i in $line
|
|
do
|
|
$RSH -o BatchMode=yes -o ConnectTimeout=10 $i uname -n
|
|
status=$?
|
|
if [[ $status != 0 ]] ; then
|
|
counter=$((counter+1))
|
|
if [ "$counter" = "1" ]; then
|
|
echo " "
|
|
echo " error - connection test failed... "
|
|
echo " "
|
|
fi
|
|
echo " "
|
|
echo " connection test with ssh failed on host $i"
|
|
echo " check the following command: ssh $i uname -n "
|
|
echo " "
|
|
fi
|
|
done
|
|
# echo error message and quit
|
|
if test $counter -ne 0
|
|
then
|
|
echo " "
|
|
echo " A parallel job using IntelMPI cannot be started. "
|
|
echo " The ssh command must be working correctly between "
|
|
echo " the computers used in the analysis. Furthermore, "
|
|
echo " it must be set up such that it does not prompt the "
|
|
echo " user for a password. "
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
echo " A parallel job using IntelMPI cannot be started. ">> $jid.log
|
|
echo " The ssh command must be working correctly between ">> $jid.log
|
|
echo " the computers used in the analysis. Furthermore, ">> $jid.log
|
|
echo " it must be set up such that it does not prompt the ">> $jid.log
|
|
echo " user for a password. ">> $jid.log
|
|
echo " " >> $jid.log
|
|
echo " Exit number 8" >> $jid.log
|
|
echo " " >> $jid.log
|
|
fi
|
|
exit 1
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
#
|
|
# check correctness of host file; fix for user sub
|
|
#
|
|
if test $nprocd -gt 1
|
|
then
|
|
|
|
# construct the path name to the executable (execpath)
|
|
execpath=$MARC_BIN/$exefile
|
|
usersub=0
|
|
if test $program = $prog.marc
|
|
then
|
|
execpath=$prog.marc
|
|
usersub=1
|
|
fi
|
|
if test "$objs"
|
|
then
|
|
execpath="$DIRJOB/$jid.marc"
|
|
usersub=1
|
|
fi
|
|
if test "$user"
|
|
then
|
|
execpath=$usernoext.marc
|
|
usersub=1
|
|
fi
|
|
export execpath
|
|
execname=`$BASENAME $execpath`
|
|
|
|
if test "$host"
|
|
then
|
|
userhost=$host
|
|
case $userhost in
|
|
\/* | \.\/*)
|
|
;;
|
|
*)
|
|
userhost=`pwd`/$userhost
|
|
;;
|
|
esac
|
|
|
|
# check that the number of processes specified in the hostfile is
|
|
# equal to nprocd specified by -nprocd.
|
|
numproc=`grep -v '^#' $host | $AWK -v sum=0 '{sum=sum+$2}; END {print sum}'`
|
|
if test $nprocd -ne $numproc
|
|
then
|
|
echo " "
|
|
echo "error, the number of processes specified in the host file"
|
|
echo "must be equal to the number of processes given by -nprocd/-nsolver"
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
echo "error, the number of processes specified in the host file" >> $jid.log
|
|
echo "must be equal to the number of processes given by -nprocd/-nsolver" >> $jid.log
|
|
echo " " >> $jid.log
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
# check for Myrinet that the number of processes per host is
|
|
# less than number of available user ports, 5
|
|
# .gmpi directory must exist in user's home directory
|
|
# and must have write permission from remote hosts
|
|
if test $MPITYPE = "myrinet"
|
|
then
|
|
numproc=`grep -v '^#' $host | $AWK -v sum=1 '{if( $2 > 5) sum=6}; END {print sum}'`
|
|
if test $numproc -gt 5
|
|
then
|
|
echo " "
|
|
echo "error, for Myrinet the number of processes specified "
|
|
echo "in the hostfile must not exceed 5 for a hostname"
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
echo "error, for Myrinet the number of processes specified " >> $jid.log
|
|
echo "in the hostfile must not exceed 5 for a hostname" >> $jid.log
|
|
echo " " >> $jid.log
|
|
fi
|
|
exit 1
|
|
fi
|
|
if test $MPIVERSION = "MPICH-GM1.2.1..7"
|
|
then
|
|
if test ! -d ~/.gmpi
|
|
then
|
|
echo " "
|
|
echo "error, for Myrinet a .gmpi directory must exist "
|
|
echo "under the user's home directory"
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
echo "error, for Myrinet a .gmpi directory must exist " >> $jid.log
|
|
echo "under the user's home directory" >> $jid.log
|
|
echo " " >> $jid.log
|
|
fi
|
|
exit 1
|
|
fi
|
|
fi
|
|
if test $MPIVERSION = "MPICH-GM1.2.1..7"
|
|
then
|
|
homedir=`echo ~`
|
|
for i in `grep -v '^#' $host | $AWK '{if (NF > 0) print $1}'`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
$RSH $i /bin/touch $homedir/.gmpi/$jid.$$ 2> tmp.$$
|
|
if test -s tmp.$$
|
|
then
|
|
echo " "
|
|
echo "error, for Myrinet a shared .gmpi directory must exist "
|
|
echo "under the user's home directory "
|
|
echo "with remote write permission"
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
echo "error, for Myrinet a shared .gmpi directory must exist " >> $jid.log
|
|
echo "under the user's home directory " >> $jid.log
|
|
echo "with remote write permission" >> $jid.log
|
|
echo " " >> $jid.log
|
|
fi
|
|
exit 1
|
|
else
|
|
/bin/rm tmp.$$
|
|
if test -f $jid.$$
|
|
then
|
|
/bin/rm $jid.$$
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
|
|
# construct the host file $jid.host which is used by mpirun
|
|
# skip lines starting with # and only consider lines with more than
|
|
# one word in them. Note that the hostfile given to this script
|
|
# has two columns: the host name and the number of shared processes
|
|
# to run on this host. mpirun wants the number of _other_
|
|
# processes to run in addition to the one being run on the machine
|
|
# on which the job is started. hence the $2-1 for fnr == 1.
|
|
if test -f $jid.host
|
|
then
|
|
/bin/rm $jid.host 2> /dev/null
|
|
fi
|
|
if test $MPITYPE = hpmpi -o $MACHINENAME = HP -a $MPITYPE = hardware
|
|
then
|
|
# HPMPI or HP hardware MPI
|
|
grep -v '^#' $host | $AWK -v path=$execpath -v en=$execname -v us=$usersub \
|
|
-v mpihpspecial="$MPIHPSPECIAL" \
|
|
'{if ( NF > 0) {\
|
|
fnr++ ; \
|
|
printf("-h %s -np %s",$1,$2); \
|
|
printf(" %s",mpihpspecial); \
|
|
if ( NF == 2 ) printf(" %s\n",path);\
|
|
if ( NF >= 3 ) printf(" -e MPI_WORKDIR=%s", $3);\
|
|
if ( NF >= 3 ) if (us) printf(" %s/%s\n",$3,en); else printf(" %s\n",path) \
|
|
}\
|
|
}' > $jid.host
|
|
# end HPMPI or HP hardware MPI
|
|
elif test $MACHINENAME = IBM -a $MPITYPE = hardware -a "$MACHINETYPE" = NONSP
|
|
then
|
|
# IBM using hardware MPI (POE)
|
|
MP_HOSTFILE=$jid.host
|
|
grep -v '^#' $host | $AWK '{host=$1;num=$2;for (i=1;i<=num;i++) print host}' > $jid.host
|
|
# end IBM using hardware MPI (POE)
|
|
# for Intel MPI, need to create a machinefile for DMP
|
|
elif test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
|
|
then
|
|
# Intel MPI
|
|
if test -f $jid.mfile
|
|
then
|
|
/bin/rm $jid.mfile 2> /dev/null
|
|
fi
|
|
/bin/cp $host $jid.host
|
|
grep -v '^#' $host | $AWK '{host=$1;num=$2;for (i=1;i<=num;i++) print host}' > $jid.mfile
|
|
# end Intel MPI for DMP
|
|
# for Solaris HPC 7.1, need to create a machinefile for DMP
|
|
elif test $MACHINENAME = "SUN" -a $MPITYPE = "hardware"
|
|
then
|
|
# Solaris HPC 7.1
|
|
if test -f $jid.mfile
|
|
then
|
|
/bin/rm $jid.mfile 2> /dev/null
|
|
fi
|
|
grep -v '^#' $host | $AWK '{host=$1;num=$2;for (i=1;i<=num;i++) print host}' > $jid.mfile
|
|
# end Solaris HPC 7.1 for DMP
|
|
# for Myrinet, construct a configuration file in ~/.gmpi
|
|
# this must be readable by each process
|
|
# format is (hostname) (port number) for each process
|
|
elif test $MPITYPE = "myrinet"
|
|
then
|
|
if test $MPIVERSION = "MPICH-GM1.2.1..7"
|
|
then
|
|
echo $nprocd > ~/.gmpi/$jid.host
|
|
grep -v '^#' $host | $AWK \
|
|
'BEGIN {iport[0] = 2; \
|
|
iport[1] = 4; \
|
|
iport[2] = 5; \
|
|
iport[3] = 6; \
|
|
iport[4] = 7 \
|
|
} \
|
|
{if ( NF > 0 ) \
|
|
for(iproc = 0; iproc < $2; iproc++) printf("%s %d\n",$1,iport[iproc]); \
|
|
}' >> ~/.gmpi/$jid.host
|
|
else
|
|
# this is for mpich-1.2.5 and later, using the -pg option
|
|
# format: host nproc executable user arguments
|
|
# the arguments are added later
|
|
grep -v '^#' $host | $AWK -v path=$execpath -v en=$execname -v us=$usersub -v user=`whoami` \
|
|
'{if ( NF > 0) {\
|
|
fnr++ ; \
|
|
if ( fnr == 1 ) printf("%s %d",$1,$2-1); \
|
|
else printf("%s %s",$1,$2); \
|
|
if ( NF == 2 ) printf(" %s %s\n",path,user);\
|
|
if ( NF == 3 ) if (us) printf(" %s/%s %s\n",$3,en,user); else printf(" %s %s\n",path,user) ;\
|
|
if ( NF == 4 ) if (us) printf(" %s/%s %s\n",$3,en,user); else printf(" %s/bin/%s %s\n",$4,en,user) \
|
|
}\
|
|
}' > $jid.host
|
|
fi
|
|
# end Myrinet
|
|
elif test $MACHINENAME = DEC -a $MPITYPE = hardware
|
|
then
|
|
# Compaq MPI via Memory Channel
|
|
grep -v '^#' $host | $AWK '{if (NF > 0) print $1}' > $jid.host
|
|
# end Compaq MPI
|
|
else
|
|
# MPICH
|
|
grep -v '^#' $host | $AWK -v path=$execpath -v en=$execname -v us=$usersub \
|
|
'{if ( NF > 0) {\
|
|
fnr++ ; \
|
|
if ( fnr == 1 ) printf("%s %d",$1,$2-1); \
|
|
else printf("%s %s",$1,$2); \
|
|
if ( NF == 2 ) printf(" %s\n",path);\
|
|
if ( NF == 3 ) if (us) printf(" %s/%s\n",$3,en); else printf(" %s\n",path) ;\
|
|
if ( NF == 4 ) if (us) printf(" %s/%s\n",$3,en); else printf(" %s/bin/%s\n",$4,en) \
|
|
}\
|
|
}' > $jid.host
|
|
fi
|
|
# define the variable host and host_filt
|
|
# host_filt is used for loops over hosts
|
|
# for Myrinet we need to use a filtered variant of userhost
|
|
# for others we can use $host
|
|
if test $MPITYPE = "myrinet"
|
|
then
|
|
if test $MPIVERSION = "MPICH-GM1.2.1..7"
|
|
then
|
|
host=~/.gmpi/$jid.host
|
|
host_filt=$jid.host_tMp
|
|
grep -v '^#' $userhost | $AWK '{if (NF > 0) print $1}' > $host_filt
|
|
else
|
|
host=$jid.host
|
|
host_filt=$host
|
|
fi
|
|
else
|
|
host=$jid.host
|
|
host_filt=$host
|
|
if test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
|
|
then
|
|
host_filt=$jid.mfile
|
|
fi
|
|
fi
|
|
# figure out if the machines in the hostfile are nfs mounted
|
|
# or distributed and set the variable "dirstatus" accordingly.
|
|
# only perform the check if user subroutine is used
|
|
# or a user subroutine executable is used
|
|
|
|
numfield=1
|
|
if test $MPITYPE = hpmpi -o $MACHINENAME = HP -a $MPITYPE = hardware
|
|
then
|
|
numfield=2
|
|
fi
|
|
DIR1=$DIRJOB
|
|
if test $program = $prog.marc -o -n "$user" -o -n "$objs"
|
|
then
|
|
counter=0
|
|
echo " "
|
|
echo "checking if local or shared directories for host"
|
|
if test "$deletelog" = no
|
|
then
|
|
echo "checking if local or shared directories for host" >> $jid.log
|
|
fi
|
|
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
counter=$((counter+1))
|
|
dirstatus[$counter]="shared"
|
|
$ECHO " $i $ECHOTXT"
|
|
if test "$deletelog" = no
|
|
then
|
|
$ECHO " $i $ECHOTXT" >> $jid.log
|
|
fi
|
|
DIR1=$DIRJOB
|
|
line=`grep -v '^#' $userhost | grep "^$ibase "`
|
|
workdir=`echo $line | $AWK '{print $3}'`
|
|
if test -n "$workdir"
|
|
then
|
|
DIR1=$workdir
|
|
fi
|
|
if test -f $jid.$$
|
|
then
|
|
/bin/rm $jid.$$
|
|
fi
|
|
$RSH $i /bin/touch $DIR1/$jid.$$ 2> tmp.$$
|
|
if test -s tmp.$$
|
|
then
|
|
dirstatus[$counter]="local"
|
|
/bin/rm tmp.$$
|
|
else
|
|
if test ! -f $jid.$$
|
|
then
|
|
dirstatus[$counter]="local"
|
|
$RSH $i /bin/rm $DIR1/$jid.$$
|
|
else
|
|
/bin/rm $jid.$$
|
|
fi
|
|
fi
|
|
if test -f tmp.$$
|
|
then
|
|
/bin/rm tmp.$$
|
|
fi
|
|
if test -f $jid.$$
|
|
then
|
|
/bin/rm $jid.$$
|
|
fi
|
|
echo " ${dirstatus[$counter]}"
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " ${dirstatus[$counter]}" >> $jid.log
|
|
fi
|
|
fi
|
|
done
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
fi
|
|
fi
|
|
|
|
# figure out if this is a compatible set of machines
|
|
# unless explicitly specified with flag -comp
|
|
# only perform the check if user subroutine is used
|
|
# or a user subroutine executable is used
|
|
# Myrinet does not support heterogeneous
|
|
if test $program = $prog.marc -o -n "$user" -o -n "$objs"
|
|
then
|
|
if test $compatible = "unknown"
|
|
then
|
|
thisname=$ARCH
|
|
compatible=yes
|
|
counter=0
|
|
echo "checking if machines are compatible for host"
|
|
if test "$deletelog" = no
|
|
then
|
|
echo "checking if machines are compatible for host" >> $jid.log
|
|
fi
|
|
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
counter=$((counter+1))
|
|
compstatus[$counter]="yes"
|
|
$ECHO " $i $ECHOTXT"
|
|
if test "$deletelog" = no
|
|
then
|
|
$ECHO " $i $ECHOTXT" >> $jid.log
|
|
fi
|
|
othername=`$RSH $i uname -a | cut -f 1 -d " "`
|
|
if test $thisname != $othername
|
|
then
|
|
compatible=no
|
|
compstatus[$counter]="no"
|
|
fi
|
|
fi
|
|
echo " ${compstatus[$counter]}"
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " ${compstatus[$counter]}" >> $jid.log
|
|
fi
|
|
done
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
fi
|
|
else
|
|
counter=0
|
|
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
counter=$((counter+1))
|
|
compstatus[$counter]=$compatible
|
|
fi
|
|
done
|
|
if test $compatible = "no"
|
|
then
|
|
echo "all machines assumed incompatible"
|
|
if test "$deletelog" = no
|
|
then
|
|
echo "all machines assumed incompatible" >> $jid.log
|
|
fi
|
|
else
|
|
echo "all machines compatible"
|
|
if test "$deletelog" = no
|
|
then
|
|
echo "all machines compatible" >> $jid.log
|
|
fi
|
|
fi
|
|
fi
|
|
# error out if user objects or libraries are used on incompatible machines
|
|
if test "$compatible" = "no" -a -n "$objs"
|
|
then
|
|
echo "User object/libraries cannot be used in a parallel job on incompatible machines"
|
|
if test "$deletelog" = no
|
|
then
|
|
echo "User object/libraries cannot be used in a parallel job on incompatible machines" >> $jid.log
|
|
fi
|
|
exit 1
|
|
fi
|
|
# modify new host file if NFS mounted heterogeneous machine
|
|
doit=
|
|
if test $program = $prog.marc
|
|
then
|
|
doit=yes
|
|
fi
|
|
if test "$user"
|
|
then
|
|
doit=yes
|
|
fi
|
|
if test "$doit"
|
|
then
|
|
counter=0
|
|
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
counter=$((counter+1))
|
|
if test ${dirstatus[$counter]} = "shared" -a ${compstatus[$counter]} = "no"
|
|
then
|
|
$AWK -v hst=$i '{fnr++ ; \
|
|
if ($1 ~ hst) {if ( fnr == 1 ) printf("%s\n",$0); else \
|
|
printf("%s %s %s_%s\n",$1,$2,$3,$1) } else print}' $jid.host > $jid.host{$$}
|
|
/bin/mv $jid.host{$$} $jid.host
|
|
host=$jid.host
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
fi # if test $program = $prog.marc -o $user -o $obj
|
|
|
|
else # if test $host
|
|
# assume shared memory machine if no hostfile given and
|
|
# MPITYPE is set to mpich or Myrinet
|
|
# check for Myrinet that the total number of processes is
|
|
# less than number of available user ports, 5
|
|
if test $MPITYPE = "mpich" -o $MPITYPE = "scali"
|
|
then
|
|
numproc=`echo $nprocd | $AWK '{sum=$1-1}; {print sum}'`
|
|
echo `hostname` $numproc $execpath > $jid.host
|
|
host=$jid.host
|
|
elif test $MPITYPE = "myrinet"
|
|
then
|
|
if test $nprocd -gt 5
|
|
then
|
|
echo " "
|
|
echo "error, for Myrinet the number of processes "
|
|
echo "must not exceed 5 for a hostname"
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
echo "error, for Myrinet the number of processes " >> $jid.log
|
|
echo "must not exceed 5 for a hostname" >> $jid.log
|
|
echo " " >> $jid.log
|
|
fi
|
|
exit 1
|
|
fi
|
|
if test $MPIVERSION = "MPICH-GM1.2.1..7"
|
|
then
|
|
echo $nprocd > ~/.gmpi/$jid.host
|
|
echo `hostname` $nprocd | $AWK \
|
|
'BEGIN {iport[0] = 2; \
|
|
iport[1] = 4; \
|
|
iport[2] = 5; \
|
|
iport[3] = 6; \
|
|
iport[4] = 7 \
|
|
} \
|
|
{for(iproc = 0; iproc < $2; iproc++) printf("%s %d\n",$1,iport[iproc])} \
|
|
' >> ~/.gmpi/$jid.host
|
|
host=~/.gmpi/$jid.host
|
|
else
|
|
numproc=`echo $nprocd | $AWK '{sum=$1-1}; {print sum}'`
|
|
echo `hostname` $numproc $execpath > $jid.host
|
|
|
|
fi
|
|
fi # if test myrinet
|
|
|
|
fi # if test $host
|
|
|
|
fi # if test $nprocd -gt 1
|
|
|
|
fi # if test $program = $exefile -o $program = $prog.marc
|
|
|
|
##############################################################################
|
|
# construct run stream (Marc only) #
|
|
##############################################################################
|
|
|
|
# set maximum message length for ddm to a large number
|
|
# for vendor provided mpi
|
|
if test $itree -eq 0 -a $MPITYPE = hardware
|
|
then
|
|
itree=100000000
|
|
if test $MACHINENAME = SGI
|
|
then
|
|
itree=100000001
|
|
fi
|
|
fi
|
|
if test $itree -eq 0 -a $MPITYPE = hpmpi
|
|
then
|
|
itree=100000000
|
|
fi
|
|
if test $itree -eq 0 -a $MPITYPE = myrinet
|
|
then
|
|
itree=100000000
|
|
fi
|
|
if test $itree -eq 0 -a $MPITYPE = nec
|
|
then
|
|
itree=100000000
|
|
fi
|
|
if test $itree -eq 0 -a $MPITYPE = scali
|
|
then
|
|
itree=100000000
|
|
fi
|
|
if test $itree -eq 0 -a $MPITYPE = intelmpi
|
|
then
|
|
itree=100000000
|
|
fi
|
|
if test $nprocdddm -lt 2
|
|
then
|
|
nprocdarg=
|
|
else
|
|
nprocdarg="$nprocdarg $nprocdddm"
|
|
fi
|
|
if test $nsolver -eq 0
|
|
then
|
|
nsolverarg=
|
|
else
|
|
nsolverarg="$nsolverarg $nsolver"
|
|
fi
|
|
if test $nprocdddm -lt 2 -a $nsolver -eq 0
|
|
then
|
|
nprocd=0
|
|
fi
|
|
if test $nprocd -gt 0
|
|
then
|
|
if test "$host"
|
|
then
|
|
if test -z "$RUN_JOB2"
|
|
then
|
|
echo " "
|
|
echo "error: parallel job attempted on non-parallel version,"
|
|
echo " or, if parallel version is installed, the include "
|
|
echo " file is probably corrupted"
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
echo "error: parallel job attempted on non-parallel version," >> $jid.log
|
|
echo " or, if parallel version is installed, the include " >> $jid.log
|
|
echo " file is probably corrupted" >> $jid.log
|
|
echo " " >> $jid.log
|
|
fi
|
|
exit
|
|
fi
|
|
if test $MPITYPE = hpmpi -o $MACHINENAME = HP -a $MPITYPE = hardware
|
|
then
|
|
RUN_JOB="$RUN_JOB2 $host -- -jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
elif test $MACHINENAME = IBM -a $MPITYPE = hardware -a "$MACHINETYPE" = NONSP
|
|
then
|
|
RUN_JOB="$RUN_JOB2 $bd$program -jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
elif test $MPITYPE = "myrinet"
|
|
then
|
|
if test $MPIVERSION = "MPICH-GM1.2.1..7"
|
|
then
|
|
RUN_JOB="$RUN_JOB2 $host $bd$program -jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
else
|
|
RUN_JOB_TMP="$RUN_JOB2 $host $bd$program"
|
|
RUN_JOB=" -jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
fi
|
|
elif test $MACHINENAME = DEC -a $MPITYPE = hardware
|
|
then
|
|
RUN_JOB="$RUN_JOB2 $nprocd -hf $host $bd$program -jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
elif test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
|
|
then
|
|
numhost=`uniq $jid.mfile | wc -l`
|
|
if test "$INTELMPI_VERSION" = "HYDRA"
|
|
then
|
|
RUN_JOB_TMP="$RUN_JOB2 -configfile $jid.cfile"
|
|
else
|
|
export I_MPI_JOB_CONTEXT=$$
|
|
mpdboot -n $numhost -r $RSH -f $jid.mfile
|
|
RUN_JOB_TMP="$RUN_JOB2 $jid.cfile"
|
|
fi
|
|
|
|
# intelmpi uses configfile. format:
|
|
# -host host1 -n n1 executable marcargs
|
|
# one such line per host
|
|
# collect the marcargs in RUN_JOB and construct the config file later
|
|
# collect the run stream in RUN_JOB_TMP
|
|
RUN_JOB="-jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
|
|
|
|
elif test $MACHINENAME = "SUN" -a $MPITYPE = "hardware"
|
|
then
|
|
RUN_JOB="$RUN_JOB2 $jid.mfile -n $nprocd $bd$program -jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
else
|
|
RUN_JOB="$RUN_JOB2 $host $bd$program -jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
fi
|
|
if test "$userhost"
|
|
then
|
|
RUN_JOB="$RUN_JOB -mhost $userhost"
|
|
fi
|
|
if test $MPITYPE = "scali"
|
|
then
|
|
# set default working directory to /tmp to allow
|
|
# different directory names
|
|
SCAMPI_WORKING_DIRECTORY=/tmp
|
|
export SCAMPI_WORKING_DIRECTORY
|
|
fi
|
|
else
|
|
if test -z "$RUN_JOB1"
|
|
then
|
|
echo " "
|
|
echo "error: parallel job attempted on non-parallel version,"
|
|
echo " or, if parallel version is installed, the include "
|
|
echo " file is probably corrupted"
|
|
echo " "
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
echo "error: parallel job attempted on non-parallel version," >> $jid.log
|
|
echo " or, if parallel version is installed, the include " >> $jid.log
|
|
echo " file is probably corrupted" >> $jid.log
|
|
echo " " >> $jid.log
|
|
fi
|
|
exit
|
|
fi
|
|
RUNNPROCD=$nprocd
|
|
if test $MACHINENAME = "IBM" -a $MPITYPE = "hardware"
|
|
then
|
|
RUNNPROCD=
|
|
MP_PROCS=$nprocd
|
|
export MP_PROCS
|
|
fi
|
|
if test $MPITYPE = "myrinet"
|
|
then
|
|
RUN_JOB="$RUN_JOB1 $RUNNPROCD $bd$program -jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
else
|
|
RUN_JOB="$RUN_JOB1 $RUNNPROCD $bd$program -jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
fi
|
|
if test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
|
|
then
|
|
if test "$INTELMPI_VERSION" = "HYDRA"
|
|
then
|
|
echo " " > /dev/null
|
|
else
|
|
export I_MPI_JOB_CONTEXT=$$
|
|
mpdboot -n 1 -f $jid.hosts
|
|
fi
|
|
RUN_JOB="$RUN_JOB1 $RUNNPROCD $bd$program -jid $jid -dirjid $DIRJID \
|
|
$nprocdarg \
|
|
$nsolverarg \
|
|
-maxnum $MAXNUM -itree $itree \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
fi
|
|
fi
|
|
else
|
|
if test $nauto -gt 0 -o $ndcoup -gt 0
|
|
then
|
|
RUN_JOB="$RUN_JOB0 $BINDIR/exe_auto $bd$program -jid $jid -dirjid $DIRJID \
|
|
-maxnum $MAXNUM \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
else
|
|
# this is for a serial job without auto restart:
|
|
RUN_JOB="$RUN_JOB0 $bd$program -jid $jid -dirjid $DIRJID \
|
|
-maxnum $MAXNUM \
|
|
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
|
|
fi
|
|
fi
|
|
if test "$rid"
|
|
then
|
|
RUN_JOB="$RUN_JOB -rid $rid -dirrid $DIRRID"
|
|
fi
|
|
if test "$pid"
|
|
then
|
|
RUN_JOB="$RUN_JOB -pid $pid -dirpid $DIRPID"
|
|
fi
|
|
if test "$sid"
|
|
then
|
|
RUN_JOB="$RUN_JOB -sid $sid -dirsid $DIRSID"
|
|
fi
|
|
if test "$did"
|
|
then
|
|
RUN_JOB="$RUN_JOB -def $did -dirdid $DIRDID"
|
|
fi
|
|
if test "$vid"
|
|
then
|
|
RUN_JOB="$RUN_JOB -vf $vid -dirvid $DIRVID"
|
|
fi
|
|
if test $nauto -gt 0
|
|
then
|
|
RUN_JOB="$RUN_JOB -autorst $nauto "
|
|
fi
|
|
if test $ndcoup -gt 0
|
|
then
|
|
RUN_JOB="$RUN_JOB -dcoup $ndcoup "
|
|
fi
|
|
if test $ndytran -gt 0
|
|
then
|
|
RUN_JOB="$RUN_JOB -dytran $ndytran "
|
|
fi
|
|
if test $mesh -gt 0
|
|
then
|
|
RUN_JOB="$RUN_JOB -me $mesh "
|
|
fi
|
|
if test $noutcore -gt 0
|
|
then
|
|
RUN_JOB="$RUN_JOB -outcore $noutcore "
|
|
fi
|
|
if test "$dllrun" -gt 0
|
|
then
|
|
RUN_JOB="$RUN_JOB -dll $dllrun "
|
|
fi
|
|
if test "$trkrun" -gt 0
|
|
then
|
|
RUN_JOB="$RUN_JOB -trk $trkrun "
|
|
fi
|
|
if test "$iam"
|
|
then
|
|
RUN_JOB="$RUN_JOB -iam $iam "
|
|
fi
|
|
if test "$justlist"
|
|
then
|
|
RUN_JOB="$RUN_JOB -list 1 "
|
|
fi
|
|
if test "$feature"
|
|
then
|
|
RUN_JOB="$RUN_JOB -feature $feature "
|
|
fi
|
|
if test "$memlimit" -ne 0
|
|
then
|
|
RUN_JOB="$RUN_JOB -ml $memlimit "
|
|
fi
|
|
if test "$cpinput"
|
|
then
|
|
RUN_JOB="$RUN_JOB -ci $cpinput "
|
|
fi
|
|
if test "$cpresults"
|
|
then
|
|
RUN_JOB="$RUN_JOB -cr $cpresults "
|
|
fi
|
|
if test "$DIRSCR" != "$DIRJOB"
|
|
then
|
|
RUN_JOB="$RUN_JOB -dirscr $DIRSCR"
|
|
else
|
|
DIRSCR=$DIRJOB
|
|
fi
|
|
if test "$makebdf"
|
|
then
|
|
RUN_JOB="$RUN_JOB -bdf $makebdf "
|
|
fi
|
|
if test $MPITYPE = "myrinet" -a "$host" -a "$MPIVERSION" != "MPICH-GM1.2.1..7"
|
|
then
|
|
# append $RUN_JOB to all lines of the host file
|
|
# and set RUN_JOB
|
|
$AWK -v args="$RUN_JOB" '{print $0,args}' $host > $host.$$
|
|
/bin/mv $host.$$ $host
|
|
RUN_JOB=$RUN_JOB_TMP
|
|
fi
|
|
if test $MPITYPE = "intelmpi" -a "$host"
|
|
then
|
|
# construct config file, append $RUN_JOB to all lines of the config file
|
|
# and set RUN_JOB
|
|
if test "$INTELMPI_VERSION" = "HYDRA"
|
|
then
|
|
grep -v '^#' $host | $AWK -v args="$RUN_JOB" -v path=$execpath -v en=$execname -v us=$usersub \
|
|
'{if ( NF > 0) {\
|
|
printf(" -host %s",$1); \
|
|
printf(" -n %s",$2); \
|
|
if ( NF == 2 ) printf(" %s",path);\
|
|
if ( NF >= 3 ) printf(" -wdir %s ",$3); \
|
|
if ( NF >= 3 ) if (us) printf(" %s/%s",$3,en); else printf(" %s",path); \
|
|
printf(" %s\n",args); \
|
|
}\
|
|
}' > $jid.cfile
|
|
else
|
|
grep -v '^#' $host | $AWK -v args="$RUN_JOB" -v path=$execpath -v en=$execname -v us=$usersub \
|
|
'{if ( NF > 0) {\
|
|
printf("-host %s -n %s",$1,$2); \
|
|
if ( NF == 2 ) printf(" %s",path);\
|
|
if ( NF >= 3 ) printf(" -wdir %s ",$3); \
|
|
if ( NF >= 3 ) if (us) printf(" %s/%s",$3,en); else printf(" %s",path); \
|
|
printf(" %s\n",args); \
|
|
}\
|
|
}' > $jid.cfile
|
|
fi
|
|
RUN_JOB=$RUN_JOB_TMP
|
|
fi
|
|
echo " "
|
|
echo "Final run stream value"
|
|
echo " RUNJOB="$RUN_JOB
|
|
if test "$deletelog" = no
|
|
then
|
|
echo " " >> $jid.log
|
|
echo "Final run stream value" >> $jid.log
|
|
echo " RUNJOB="$RUN_JOB >> $jid.log
|
|
fi
|
|
|
|
|
|
##############################################################################
|
|
# run marc using valgrind #
|
|
##############################################################################
|
|
#RUN_JOB="valgrind $RUN_JOB"
|
|
#RUN_JOB="valgrind --read-var-info=yes --gen-suppressions=yes $RUN_JOB"
|
|
#RUN_JOB="valgrind --gen-suppressions=all -v $RUN_JOB"
|
|
#RUN_JOB="valgrind --gen-suppressions=yes --error-limit=no $RUN_JOB"
|
|
##############################################################################
|
|
|
|
|
|
##############################################################################
|
|
# run the requested program in a queue #
|
|
##############################################################################
|
|
|
|
if test "$deletelog" = yes
|
|
then
|
|
echo
|
|
date
|
|
else
|
|
echo >> $jid.log
|
|
date >> $jid.log
|
|
fi
|
|
if [ $qid = short -o $qid = long -o $qid = verylong -o $qid = at ]
|
|
then
|
|
|
|
/bin/rm -f $jid.runmarcscript
|
|
|
|
|
|
#
|
|
# compile user subroutine if present
|
|
#
|
|
if test "$link"
|
|
then
|
|
if test -z "$FCOMPROOT"; then
|
|
echo "$0: No compiler available"
|
|
echo
|
|
echo " $PRODUCT Exit number 3"
|
|
exit 1
|
|
fi
|
|
echo
|
|
echo "Using compiler from: $FCOMPROOT"
|
|
echo
|
|
if test "$user"
|
|
then
|
|
userobj=$usermoext.o
|
|
fi
|
|
cat > $jid.runmarcscript << END4
|
|
if test "$user"
|
|
then
|
|
if test $MACHINENAME = "CRAY"
|
|
then
|
|
$DFORTLOW $user || \
|
|
{
|
|
echo "$0: compile failed for $user"
|
|
exit 1
|
|
}
|
|
/bin/rm $program 2>/dev/null
|
|
else
|
|
$DFORTLOW $user -o $userobj || \
|
|
{
|
|
echo "$0: compile failed for $user"
|
|
exit 1
|
|
}
|
|
/bin/rm $program 2>/dev/null
|
|
fi
|
|
fi
|
|
|
|
|
|
$LOAD $bd${program} $MARC_LIB/main.o \
|
|
$MARC_LIB/blkdta.o $MARC_LIB/comm?.o \
|
|
${userobj-} \
|
|
$objs \
|
|
$MARC_LIB/srclib.a \
|
|
$MNFLIBS \
|
|
$MDUSER \
|
|
${MUMPSSOLVERLIBS} \
|
|
$MDSRCLIB \
|
|
$MARC_LIB/mcvfit.a \
|
|
$STUBS \
|
|
$SOLVERLIBS \
|
|
$MARCCUDALIBS \
|
|
$TKLIBS \
|
|
$MRCLIBS \
|
|
$METISLIBS \
|
|
$DAMASK \
|
|
$OPENSSL_LIB \
|
|
$SYSLIBS \
|
|
$SFLIB \
|
|
$SECLIBS || \
|
|
{
|
|
echo "$0: link failed for ${user:+$userobj }$objs"
|
|
exit 1
|
|
}
|
|
END4
|
|
else
|
|
prgsav=yes
|
|
fi
|
|
/bin/rm $userobj 2>/dev/null
|
|
/bin/rm $DIRJOB/*.mod 2>/dev/null
|
|
|
|
#
|
|
# run marc
|
|
#
|
|
|
|
cat >> $jid.runmarcscript << END5
|
|
|
|
# Define share library path based on platforms
|
|
# This is required for using the Patran Mesher
|
|
if test $MACHINENAME = "IBM"
|
|
then
|
|
LIBPATH=$MARC_LIB:$MARC_LIB_SHARED:$LIBPATH
|
|
export LIBPATH
|
|
fi
|
|
|
|
# first remove all .out files and incremental restart files
|
|
# the ones for ddm are removed in the code
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
numdom=$nprocdddm
|
|
while test \$numdom -gt 0
|
|
do
|
|
/bin/rm $DIRJOB/$numdom$jid.out 2>/dev/null
|
|
/bin/rm $DIRJOB/$numdom$jid.log 2>/dev/null
|
|
/bin/rm $DIRJOB/$numdom${jid}_i_*.t08 2>/dev/null
|
|
numdom=\`echo \$numdom | $AWK '{sum=\$1-1}; {print sum}'\`
|
|
done
|
|
else
|
|
/bin/rm $DIRJOB/$jid.out 2>/dev/null
|
|
/bin/rm $DIRJOB/${jid}_i_*.t08 2>/dev/null
|
|
fi
|
|
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
$RUN_JOB 2>>$jid.log
|
|
else
|
|
$RUN_JOB 2>>$jid.log
|
|
fi
|
|
|
|
if test $dllrun -eq 0; then
|
|
if test $prgsav = no
|
|
then
|
|
/bin/rm -f $bd$program 2>/dev/null
|
|
fi
|
|
else
|
|
if test $cpdll = yes; then
|
|
filename=$usernoext
|
|
/bin/cp $DIRJOB/$marcdll $DIRJOB/${filename}_$marcdll 2>/dev/null
|
|
fi
|
|
if test $rmdll = yes
|
|
then
|
|
/bin/rm -f $DIRJOB/$marcdll 2>/dev/null
|
|
fi
|
|
fi
|
|
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
numdom=$nprocdddm
|
|
while test \$numdom -gt 0
|
|
do
|
|
/bin/rm $DIRSCR/$numdom$jid.t02 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t03 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t11 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t12 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t13 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t14 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t15 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t22 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t23 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t32 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t33 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t74 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t75 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t76 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t77 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t78 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t79 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.*.t81* 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.*.t82* 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.*.t83* 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t84 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t85 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t86 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t87 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t88 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t90 2>/dev/null
|
|
numdom=\`echo \$numdom | $AWK '{sum=\$1-1}; {print sum}'\`
|
|
done
|
|
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
|
|
else
|
|
/bin/rm $DIRSCR/$jid.t02 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t03 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t11 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t12 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t13 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t14 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t15 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t22 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t23 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t32 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t33 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.*.t81* 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.*.t82* 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.*.t83* 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t84 2>/dev/null
|
|
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
|
|
fi
|
|
END5
|
|
|
|
|
|
# Submit to marc batch queue
|
|
#
|
|
if [ $qid = at ]
|
|
then
|
|
QUENAME=at
|
|
SUBMCMD=
|
|
else
|
|
#
|
|
# Submit to qsub queue
|
|
#
|
|
QUENAME=qsub
|
|
SUBMCMD="-q $qid -o /dev/null -e $jid.batch_err_log -x -r $jid"
|
|
if test "$priority"
|
|
then
|
|
SUBMCMD=$SUBMCMD" -p $priority"
|
|
fi
|
|
if test "$att"
|
|
then
|
|
SUBMCMD=$SUBMCMD" -a $att"
|
|
fi
|
|
if test "$cpu"
|
|
then
|
|
SUBMCMD=$SUBMCMD" -lt $cpu"
|
|
fi
|
|
|
|
fi
|
|
echo $QUENAME $SUBMCMD
|
|
#cat $jid.runmarcscript
|
|
$QUENAME $SUBMCMD < $jid.runmarcscript
|
|
|
|
/bin/rm -f $jid.runmarcscript
|
|
|
|
##############################################################################
|
|
# run the requested program in the background #
|
|
##############################################################################
|
|
|
|
else
|
|
if test $qid = background
|
|
then
|
|
|
|
#
|
|
# first remove all old .out files
|
|
# the ones for ddm are removed in the code
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
numdom=$nprocdddm
|
|
while test $numdom -gt 0
|
|
do
|
|
/bin/rm $DIRJOB/$numdom$jid.out 2>/dev/null
|
|
/bin/rm $DIRJOB/$numdom$jid.log 2>/dev/null
|
|
numdom=`echo $numdom | $AWK '{sum=$1-1}; {print sum}'`
|
|
done
|
|
else
|
|
/bin/rm $DIRJOB/$jid.out 2>/dev/null
|
|
fi
|
|
#
|
|
# compile user subroutine if present
|
|
#
|
|
(
|
|
if test "$link"
|
|
then
|
|
if test -z "$FCOMPROOT"; then
|
|
echo "$0: No compiler available"
|
|
echo
|
|
echo " $PRODUCT Exit number 3"
|
|
exit 1
|
|
fi
|
|
echo
|
|
echo "Using compiler from: $FCOMPROOT"
|
|
echo
|
|
if test "$user"
|
|
then
|
|
# compile and link on other hosts in $host if compstatus=no
|
|
if test $nprocd -gt 1
|
|
then
|
|
if test "$userhost"
|
|
then
|
|
counter=0
|
|
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
counter=$((counter+1))
|
|
if test ${compstatus[$counter]} = "no"
|
|
then
|
|
DIR1=$DIRJOB
|
|
DIR2=$DIR
|
|
line=`grep -v '^#' $userhost | grep "^$ibase "`
|
|
workdir=`echo $line | $AWK '{print $3}'`
|
|
marcdir=`echo $line | $AWK '{print $4}'`
|
|
if test -n "$workdir"
|
|
then
|
|
DIR1=$workdir
|
|
fi
|
|
if test -n "$marcdir"
|
|
then
|
|
DIR2=$marcdir
|
|
fi
|
|
# first copy over the user sub if local directories
|
|
if test ${dirstatus[$counter]} = "local"
|
|
then
|
|
$RCP $user $i:$DIR1/
|
|
fi
|
|
# do the compilation on the other machine
|
|
if test ${dirstatus[$counter]} = "shared"
|
|
then
|
|
hname=_$ibase
|
|
else
|
|
hname=
|
|
fi
|
|
remoteprog=$DIR1/${execname}$hname
|
|
remoteuser=$DIR1/`$BASENAME $user`
|
|
$RSH $i /bin/rm $remoteprog 2> /dev/null
|
|
echo
|
|
$RSH $i $DIR2/tools/comp_damask_l $DIR2 $DIR1 $remoteuser $remoteprog
|
|
# check if successful, the new executable should be there
|
|
line=`$RSH $i /bin/ls $remoteprog 2> /dev/null`
|
|
if test "$line"
|
|
then
|
|
echo compilation and linking successful on host $i
|
|
else
|
|
echo "$0: compile failed for $user on host $i"
|
|
echo " $PRODUCT Exit number 3"
|
|
exit 1
|
|
fi
|
|
# remove the user subroutine on remote machine
|
|
if test ${dirstatus[$counter]} = "local"
|
|
then
|
|
$RSH $i /bin/rm $remoteuser 2> /dev/null
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
if test "$userhost"
|
|
then
|
|
echo
|
|
echo "Compiling and linking user subroutine $user on host `hostname`"
|
|
fi
|
|
userobj=$usernoext.o
|
|
if test $MACHINENAME = "CRAY"
|
|
then
|
|
$DFORTLOW $user || \
|
|
{
|
|
echo "$0: compile failed for $user"
|
|
echo " $PRODUCT Exit number 3"
|
|
exit 1
|
|
}
|
|
/bin/rm $program 2>/dev/null
|
|
else
|
|
$DFORTLOW $user -o $userobj || \
|
|
{
|
|
echo "$0: compile failed for $user"
|
|
echo " $PRODUCT Exit number 3"
|
|
exit 1
|
|
}
|
|
/bin/rm $program 2>/dev/null
|
|
fi
|
|
fi # if test $user
|
|
|
|
|
|
$LOAD $bd${program} $MARC_LIB/main.o \
|
|
$MARC_LIB/blkdta.o $MARC_LIB/comm?.o \
|
|
${userobj-} \
|
|
$objs \
|
|
$MARC_LIB/srclib.a \
|
|
$MNFLIBS \
|
|
$MDUSER \
|
|
${MUMPSSOLVERLIBS} \
|
|
$MDSRCLIB \
|
|
$MARC_LIB/mcvfit.a \
|
|
$STUBS \
|
|
${SOLVERLIBS} \
|
|
${MARCCUDALIBS} \
|
|
$TKLIBS \
|
|
$MRCLIBS \
|
|
$METISLIBS \
|
|
$DAMASK \
|
|
$OPENSSL_LIB \
|
|
$SYSLIBS \
|
|
$SFLIB \
|
|
$SECLIBS || \
|
|
{
|
|
echo "$0: link failed for ${user:+$userobj }$objs"
|
|
echo " $PRODUCT Exit number 3"
|
|
exit 1
|
|
}
|
|
# copy user subroutine executable for hosts using local working dir
|
|
if test $nprocd -gt 1
|
|
then
|
|
if test "$userhost"
|
|
then
|
|
counter=0
|
|
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
counter=$((counter+1))
|
|
if test ${dirstatus[$counter]} = "local" -a ${compstatus[$counter]} = "yes"
|
|
then
|
|
DIR1=$DIRJOB
|
|
line=`grep -v '^#' $userhost | grep "^$ibase "`
|
|
workdir=`echo $line | $AWK '{print $3}'`
|
|
if test -n "$workdir"
|
|
then
|
|
DIR1=$workdir
|
|
fi
|
|
echo "Copying executable to host ${i}"
|
|
$RCP $program ${i}:${DIR1}/
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
else # if test $link
|
|
prgsav=yes
|
|
fi # if test $link
|
|
/bin/rm $userobj 2>/dev/null
|
|
/bin/rm $DIRJOB/*.mod 2>/dev/null
|
|
|
|
#
|
|
# run marc
|
|
|
|
#
|
|
|
|
# Define share library path based on platforms
|
|
# This is required for using the Patran Mesher
|
|
if test $MACHINENAME = "IBM"
|
|
then
|
|
LIBPATH=$MARC_LIB:$MARC_LIB_SHARED:$LIBPATH
|
|
export LIBPATH
|
|
fi
|
|
|
|
# for DDM with ARC support
|
|
|
|
if test $ddm_arc -gt 0; then
|
|
RUN_JOB="$MESHERDIR/sf_exeddm $RUN_JOB -ddm $ddm_arc "
|
|
fi
|
|
|
|
|
|
$RUN_JOB &
|
|
|
|
marcpid=$!
|
|
echo $marcpid > $DIRJOB/$jid.pid
|
|
wait $marcpid
|
|
|
|
if test $nprocd -gt 1
|
|
then
|
|
if test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
|
|
then
|
|
if test "$INTELMPI_VERSION" = "HYDRA"
|
|
then
|
|
if test "$host"
|
|
then
|
|
/bin/rm $jid.mfile 2> /dev/null
|
|
/bin/rm $jid.hosts 2> /dev/null
|
|
/bin/rm $jid.host 2> /dev/null
|
|
/bin/rm $jid.cfile 2> /dev/null
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
|
|
if test $dllrun -eq 0; then
|
|
if test $prgsav = no
|
|
then
|
|
/bin/rm -f $bd$program 2>/dev/null
|
|
# for network run, remove executable on remote machines
|
|
# and executables with modified name
|
|
if test $nprocd -gt 1
|
|
then
|
|
if test "$userhost"
|
|
then
|
|
counter=0
|
|
if test -f "$host_filt"
|
|
then
|
|
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
counter=$((counter+1))
|
|
DIR1=$DIRJOB
|
|
line=`grep -v '^#' $userhost | grep "^$ibase "`
|
|
workdir=`echo $line | $AWK '{print $3}'`
|
|
if test -n "$workdir"
|
|
then
|
|
DIR1=$workdir
|
|
fi
|
|
# if an incompatible host uses shared directory,
|
|
# then the root machine deletes the executable
|
|
if test ${dirstatus[$counter]} = "shared" -a ${compstatus[$counter]} = "no"
|
|
then
|
|
hname=_$ibase
|
|
/bin/rm ${execname}$hname
|
|
fi
|
|
# if local directory used, the remote machine
|
|
# deletes the executable
|
|
if test ${dirstatus[$counter]} = "local"
|
|
then
|
|
$RSH $i /bin/rm $DIR1/${execname} 2>/dev/null
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
else
|
|
#dllrun >0
|
|
if test $cpdll = yes; then
|
|
filename=$usernoext
|
|
/bin/cp $DIRJOB/$marcdll $DIRJOB/${filename}_$marcdll 2>/dev/null
|
|
fi
|
|
if test $rmdll = yes;then
|
|
/bin/rm -f $DIRJOB/$marcdll 2>/dev/null
|
|
fi
|
|
fi
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
numdom=$nprocdddm
|
|
while test $numdom -gt 0
|
|
do
|
|
/bin/rm $DIRSCR/$numdom$jid.t02 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t03 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t11 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t12 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t13 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t14 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t15 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t22 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t23 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t32 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t33 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t74 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t75 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t76 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t77 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t78 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t79 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.*.t81* 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.*.t82* 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.*.t83* 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t84 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t85 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t86 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t87 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t88 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t90 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.sle 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.sin 2>/dev/null
|
|
numdom=`echo $numdom | $AWK '{sum=$1-1}; {print sum}'`
|
|
done
|
|
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
|
|
if test $MPITYPE = "myrinet"
|
|
then
|
|
if test -f "$host_filt"
|
|
then
|
|
/bin/rm $host_filt
|
|
fi
|
|
fi
|
|
else
|
|
/bin/rm $DIRSCR/$jid.t02 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t03 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t11 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t12 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t13 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t14 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t15 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t22 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t23 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t32 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t33 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.*.t81* 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.*.t82* 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.*.t83* 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t84 2>/dev/null
|
|
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
|
|
/bin/rm $DIRJOB/$jid.sle 2>/dev/null
|
|
/bin/rm $DIRJOB/$jid.sin 2>/dev/null
|
|
fi
|
|
) 1>>$jid.log 2>&1 &
|
|
|
|
|
|
##############################################################################
|
|
# run the requested program in the foreground #
|
|
##############################################################################
|
|
|
|
else
|
|
|
|
#
|
|
# compile user subroutine if present
|
|
#
|
|
if test "$link"
|
|
then
|
|
if test -z "$FCOMPROOT"; then
|
|
echo "$0: No compiler available"
|
|
echo
|
|
echo " $PRODUCT Exit number 3"
|
|
exit 1
|
|
fi
|
|
echo
|
|
echo "Using compiler from: $FCOMPROOT"
|
|
echo
|
|
if test "$user"
|
|
then
|
|
# compile and link on other hosts in $host if compstatus=no
|
|
if test $nprocd -gt 1
|
|
then
|
|
if test "$userhost"
|
|
then
|
|
counter=0
|
|
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
counter=$((counter+1))
|
|
if test ${compstatus[$counter]} = "no"
|
|
then
|
|
DIR1=$DIRJOB
|
|
DIR2=$DIR
|
|
line=`grep -v '^#' $userhost | grep "^$ibase "`
|
|
workdir=`echo $line | $AWK '{print $3}'`
|
|
marcdir=`echo $line | $AWK '{print $4}'`
|
|
if test -n "$workdir"
|
|
then
|
|
DIR1=$workdir
|
|
fi
|
|
if test -n "$marcdir"
|
|
then
|
|
DIR2=$marcdir
|
|
fi
|
|
# first copy over the user sub if local directories
|
|
if test ${dirstatus[$counter]} = "local"
|
|
then
|
|
$RCP $user $i:$DIR1/
|
|
fi
|
|
# do the compilation on the other machine
|
|
if test ${dirstatus[$counter]} = "shared"
|
|
then
|
|
hname=_$ibase
|
|
else
|
|
hname=
|
|
fi
|
|
remoteprog=$DIR1/${execname}$hname
|
|
remoteuser=$DIR1/`$BASENAME $user`
|
|
$RSH $i /bin/rm $remoteprog 2> /dev/null
|
|
echo
|
|
$RSH $i $DIR2/tools/comp_damask_l $DIR2 $DIR1 $remoteuser $remoteprog
|
|
# check if successful, the new executable should be there
|
|
line=`$RSH $i /bin/ls $remoteprog 2> /dev/null`
|
|
if test "$line"
|
|
then
|
|
echo compilation and linking successful on host $i
|
|
else
|
|
echo "$0: compile failed for $user on host $i"
|
|
exit 1
|
|
fi
|
|
# remove the user subroutine on remote machine
|
|
if test ${dirstatus[$counter]} = "local"
|
|
then
|
|
$RSH $i /bin/rm $remoteuser 2> /dev/null
|
|
fi
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
if test "$userhost"
|
|
then
|
|
echo
|
|
echo "Compiling and linking user subroutine $user on host `hostname`"
|
|
fi
|
|
userobj=$usernoext.o
|
|
if test $MACHINENAME = "CRAY"
|
|
then
|
|
$DFORTLOW $user || \
|
|
{
|
|
echo "$0: compile failed for $user"
|
|
exit 1
|
|
}
|
|
/bin/rm $program 2>/dev/null
|
|
else
|
|
$DFORTLOW $user -o $userobj || \
|
|
{
|
|
echo "$0: compile failed for $user"
|
|
exit 1
|
|
}
|
|
/bin/rm $program 2>/dev/null
|
|
fi
|
|
fi # if test $user
|
|
|
|
|
|
$LOAD $bd${program} $MARC_LIB/main.o \
|
|
$MARC_LIB/blkdta.o $MARC_LIB/comm?.o \
|
|
${userobj-} \
|
|
$objs \
|
|
$MARC_LIB/srclib.a \
|
|
$MNFLIBS \
|
|
$MDUSER \
|
|
${MUMPSSOLVERLIBS} \
|
|
$MDSRCLIB \
|
|
$MARC_LIB/mcvfit.a \
|
|
$STUBS \
|
|
${SOLVERLIBS} \
|
|
${MARCCUDALIBS} \
|
|
$TKLIBS \
|
|
$MRCLIBS \
|
|
$METISLIBS \
|
|
$DAMASK \
|
|
$OPENSSL_LIB \
|
|
$SYSLIBS \
|
|
$SFLIB \
|
|
$SECLIBS || \
|
|
{
|
|
echo "$0: link failed for ${user:+$userobj }$objs"
|
|
exit 1
|
|
}
|
|
# copy user subroutine executable for hosts using local working dir
|
|
if test $nprocd -gt 1
|
|
then
|
|
if test "$userhost"
|
|
then
|
|
counter=0
|
|
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
counter=$((counter+1))
|
|
if test ${dirstatus[$counter]} = "local" -a ${compstatus[$counter]} = "yes"
|
|
then
|
|
DIR1=$DIRJOB
|
|
line=`grep -v '^#' $userhost | grep "^$ibase "`
|
|
workdir=`echo $line | $AWK '{print $3}'`
|
|
if test -n "$workdir"
|
|
then
|
|
DIR1=$workdir
|
|
fi
|
|
echo "Copying executable to host ${i}"
|
|
$RCP $program ${i}:${DIR1}/
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
else # if test $link
|
|
prgsav=yes
|
|
fi # if test $link
|
|
/bin/rm $userobj 2>/dev/null
|
|
/bin/rm $DIRJOB/*.mod 2>/dev/null
|
|
# done if no job id given
|
|
if test -z "$jid"
|
|
then
|
|
echo
|
|
echo only compilation requested
|
|
echo
|
|
exit
|
|
fi
|
|
#
|
|
# run marc
|
|
#
|
|
# Define share library path based on platforms
|
|
# This is required for using the Patran Mesher
|
|
if test $MACHINENAME = "IBM"
|
|
then
|
|
LIBPATH=$MARC_LIB:$MARC_LIB_SHARED:$LIBPATH
|
|
export LIBPATH
|
|
fi
|
|
# first remove all .out files
|
|
# the ones for ddm are removed in the code
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
numdom=$nprocdddm
|
|
while test $numdom -gt 0
|
|
do
|
|
/bin/rm $DIRJOB/$numdom$jid.out 2>/dev/null
|
|
/bin/rm $DIRJOB/$numdom$jid.log 2>/dev/null
|
|
numdom=`echo $numdom | $AWK '{sum=$1-1}; {print sum}'`
|
|
done
|
|
else
|
|
/bin/rm $DIRJOB/$jid.out 2>/dev/null
|
|
fi
|
|
|
|
# for DDM with ARC support
|
|
|
|
if test $ddm_arc -gt 0; then
|
|
RUN_JOB="$MESHERDIR/sf_exeddm $RUN_JOB -ddm $ddm_arc "
|
|
fi
|
|
|
|
$RUN_JOB
|
|
|
|
if test $nprocd -gt 1
|
|
then
|
|
if test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
|
|
then
|
|
if test "$INTELMPI_VERSION" = "HYDRA"
|
|
then
|
|
if test "$host"
|
|
then
|
|
/bin/rm $jid.mfile 2> /dev/null
|
|
/bin/rm $jid.hosts 2> /dev/null
|
|
/bin/rm $jid.host 2> /dev/null
|
|
/bin/rm $jid.cfile 2> /dev/null
|
|
else
|
|
echo " " > /dev/null
|
|
fi
|
|
else
|
|
if test "$host"
|
|
then
|
|
mpdcleanup -a -f $jid.mfile
|
|
/bin/rm $jid.host 2> /dev/null
|
|
/bin/rm $jid.mfile 2> /dev/null
|
|
else
|
|
mpdcleanup -a -f $jid.hosts
|
|
/bin/rm $jid.hosts 2> /dev/null
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if test $dllrun -eq 0; then
|
|
if test $prgsav = no
|
|
then
|
|
/bin/rm -f $bd$program 2>/dev/null
|
|
# for network run, remove executable on remote machines
|
|
# and executables with modified name
|
|
if test $nprocd -gt 1
|
|
then
|
|
if test "$userhost"
|
|
then
|
|
counter=0
|
|
if test -f "$host_filt"
|
|
then
|
|
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
|
|
do
|
|
ibase=${i%%.*}
|
|
if test $ibase != $thishost
|
|
then
|
|
counter=$((counter+1))
|
|
DIR1=$DIRJOB
|
|
line=`grep -v '^#' $userhost | grep "^$ibase "`
|
|
workdir=`echo $line | $AWK '{print $3}'`
|
|
if test -n "$workdir"
|
|
then
|
|
DIR1=$workdir
|
|
fi
|
|
# if an incompatible host uses shared directory,
|
|
# then the root machine deletes the executable
|
|
if test ${dirstatus[$counter]} = "shared" -a ${compstatus[$counter]} = "no"
|
|
then
|
|
hname=_$ibase
|
|
/bin/rm ${execname}$hname
|
|
fi
|
|
# if local directory used, the remote machine
|
|
# deletes the executable
|
|
if test ${dirstatus[$counter]} = "local"
|
|
then
|
|
$RSH $i /bin/rm $DIR1/${execname} 2>/dev/null
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
else
|
|
#dllrun >0
|
|
if test $cpdll = yes; then
|
|
filename=$usernoext
|
|
/bin/cp $DIRJOB/$marcdll $DIRJOB/${filename}_$marcdll 2>/dev/null
|
|
fi
|
|
if test $rmdll = yes;then
|
|
/bin/rm -f $DIRJOB/$marcdll 2>/dev/null
|
|
fi
|
|
fi
|
|
|
|
if test $nprocdddm -gt 1
|
|
then
|
|
numdom=$nprocdddm
|
|
while test $numdom -gt 0
|
|
do
|
|
/bin/rm $DIRSCR/$numdom$jid.t02 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t03 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t11 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t12 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t13 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t14 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t15 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t22 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t23 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t32 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t33 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t74 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t75 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t76 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t77 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t78 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t79 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.*.t81* 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.*.t82* 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.*.t83* 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t84 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t85 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t86 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t87 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t88 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.t90 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.sle 2>/dev/null
|
|
/bin/rm $DIRSCR/$numdom$jid.sin 2>/dev/null
|
|
numdom=`echo $numdom | $AWK '{sum=$1-1}; {print sum}'`
|
|
done
|
|
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
|
|
if test $MPITYPE = "myrinet"
|
|
then
|
|
if test -f "$host_filt"
|
|
then
|
|
/bin/rm $host_filt
|
|
fi
|
|
fi
|
|
else
|
|
/bin/rm $DIRSCR/$jid.t02 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t03 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t11 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t12 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t13 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t14 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t15 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t22 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t23 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t32 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t33 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.*.t81* 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.*.t82* 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.*.t83* 2>/dev/null
|
|
/bin/rm $DIRSCR/$jid.t84 2>/dev/null
|
|
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
|
|
/bin/rm $DIRJOB/$jid.sle 2>/dev/null
|
|
/bin/rm $DIRJOB/$jid.sin 2>/dev/null
|
|
fi
|
|
|
|
|
|
fi
|
|
fi
|