#!/bin/ksh
##############################################################################
# #
# run_marc - run a marc job #
# ------------------------- #
# #
# usage: run_marc -j jid { options } #
# #
# where standard options are: required: defaults: #
# -------------------------- #
# #
# -j* jid job id number. ** YES ** . #
# -pr* prog program name. . marc #
# -v* y|n do or do not verify inputs. . yes #
# -q* s|l|v|b|f batch queue name or background, . short #
# foreground. #
# -b* as alternative to option -q* #
# #
# ( batch queues only : #
# -pq* intra queue priority. . . #
# -at DATE/TIME delay start of job. . . #
# format : January,1,1990,12:31 #
# or : today,5pm #
# -cpu* secs job CPU limit . . ) #
# #
# -r* rid restart file job id. . . #
# -si* sid substructure file id. . . #
# -pi* post post file job id. . . #
# -de* did defaults file . no #
# -vf vid viewfactor . no #
# #
# -u* user user subroutine. . . #
# -obj obj user objects or libraries. . . #
# -sa* y|n do or do not save load module. . no #
# -me manual remeshing control . no #
# -ml memory limit in Mbyte #
# -mo This option is deprecated. As of Marc 2015, only #
# the integer*8 version is available. #
# -mpi selects MPI version #
# each platform has a default MPI version and some #
# have an alternative version. see the include file #
# for the respective platform #
# MPI_DEFAULT defines the default MPI version #
# MPI_OTHER defines versions one can switch to #
# -dcoup for contact decoupling #
# currently not supported #
# -dir directory where the job i/o should take place. #
# defaults to current directory. #
# -sdir directory where scratch files are created #
# defaults to current directory. #
# #
# -alloc only perform memory allocation test, no analysis #
# -list y only list options in the input file, no analysis #
# -fe num set feature number "num" for the run. only one allowed #
# -dytran flag to switch from Dytran to Marc #
# dytran = 0, program will run w/o Marc-Dytran Switch #
# = 1, program will restart Marc after Dytran run #
# >= 2, Not supported yet. #
# currently not supported #
# -ou force analysis to use out-of-core control #
# =0, not used #
# =1, element storage out-of-core #
# -dll run marc using shared library libmarc.so and exe_marc #
# =1, used #
# =2, do not free streaming input memory #
# =3, run with marc input deck #
# -trk run marc for post-tracking #
# -gpuid run marc using GPGPU capability #
# specify gpuid on to be used in the analysis. Multiple #
# IDs may be assigned for DDM runs. #
# Separate a list of IDs with a colon. Each DMP #
# process will be assigned a GPU ID in round robin fastion#
# = 0 #
# = 0:1 etc... #
# #
# where parallel options are: #
# -------------------------- #
# #
# itree, host, and comp options are available for the domain #
# decomposition only. #
# MARC_NUMBER_OF_THREADS, nthread, and dir options always available. #
# #
# #
# -nprocd number of domains. #
# defaults to single domain solution. #
# -nprocds number of domains if single input file. #
# defaults to single domain solution. #
# -nps same as -nprocds. #
# -nsolver number of solver tasks for solver types 12 and 13 #
# these are distributed tasks operating via MPI #
# -nthread_elem number of threads for element assembly and recovery #
# = 0: use defaults. #
# defaults to 1 for single domain solution. #
# defaults to number of domains for multi-domain #
# solution. #
# > 1: number of threads to be used by element assembly #
# recovery. #
# Also can be set through MARC_NUMBER_OF_THREADS #
# environment variable. #
# if both specified, -nthread_elem option will be used. #
# defaults if neither MARC_NUMBER_OF_THREADS environment #
# variable set nor -nthread_elem specified. #
# -nthread_solver number of threads for solver types 6, 8, and 11 #
# = 0: use defaults. #
# defaults to 1 for single domain solution. #
# defaults to number of domains for multi-domain #
# solution. #
# > 1: number of threads to be used by 6, 8, and 11 #
# Also can be set through MARC_NUMBER_OF_THREADS #
# environment variable. #
# if both specified, -nthread_solver option will be used. #
# defaults if neither MARC_NUMBER_OF_THREADS environment #
# variable set nor -nthread_solver specified. #
# -nthread Same as -nthread_solver. #
# -itree message passing tree type for domain decomposition. #
# for debugging purposes; should not normally be used. #
# -host hostfile name for distributed execution on network. #
# defaults to no hostfile, unless jobid.defhost exists. #
# if jobid.defhost exists, only -np(s) necessary #
# -comp* y|n to be used with user routines on a network of #
# incompatible machines. #
# if set to no, a separate executable will be created #
# for each machine on the network. #
# if set to yes, the executable located on the machine #
# from which marc is started will be used on all machines.#
# defaults to no if O/S versions different on machines. #
# #
# -ci y|n copy input files to remote hosts (default: yes) #
# if "yes", input files are automatically copied to #
# remote hosts for a network run if necessary. #
# -cr y|n copy post files from remote hosts (default: yes) #
# if "yes", post files are automatically copied back from #
# remote hosts for a network run if necessary. #
##############################################################################
# set DIR to the directory in which this script is
REALCOM="`/bin/ls -l $0 |awk '{ print $NF; }'`"
DIR=`dirname $REALCOM`
# make sure DIR has an absolute path
case $DIR in
\/*)
;;
*)
DIR=`pwd`/$DIR
;;
esac
DIRSCRIPT=$DIR
AWK=awk
ARCH=`uname -a | cut -f 1 -d " "`
# Sun has a bad awk, use nawk instead
if test $ARCH = "SunOS"
then
AWK=nawk
fi
BASENAME=basename
# Sun has an incorrect /bin/basename, check if /usr/ucb/basename exists
if test $ARCH = "SunOS"
then
if test -x /usr/ucb/basename
then
BASENAME=/usr/ucb/basename
fi
fi
# echo command line in the case of ECHO_COMMAND is true
if test "$ECHO_COMMAND" = true ; then
echo command "$0" "$@"
fi
#
# "mode" selects version, i4 or i8
# default is i4
# this can be changed by a file run_marc_defaults
# located in the tools directory of the Marc installation
# or in the user's home directory
# format:
# MARC_MODE i8
# it can also be set by the environmental variable MARC_INTEGER_SIZE
# and by the command line option "-mo"
#
mode=
modeerror=
modeoption=
if test -f $DIRSCRIPT/run_marc_defaults; then
line=`$AWK '{if ($1 == "MARC_MODE") {print $1}}' $DIRSCRIPT/run_marc_defaults`
if test "$line" = "MARC_MODE"; then
echo
echo warning: the option MARC_MODE is deprecated, as of Marc 2015, only the integer*8 version is available
echo
line=
fi
line=`$AWK '{if ($1 == "MARC_MODE") {print $2}}' $DIRSCRIPT/run_marc_defaults`
line=`echo $line | $AWK '{print $NF}'`
if test "$line" = "i4"; then
modeerror="defaults file $DIRSCRIPT/run_marc_defaults used mode $line ; this must be i8"
modeoption=error
echo $modeerror
fi
if test "$line" = "i8"; then
mode=i8
fi
fi
if test -f $HOME/run_marc_defaults; then
line=`$AWK '{if ($1 == "MARC_MODE") {print $1}}' $HOME/run_marc_defaults`
if test "$line" = "MARC_MODE"; then
echo
echo warning: the option MARC_MODE is deprecated, as of Marc 2015, only the integer*8 version is available
echo
line=
fi
line=`$AWK '{if ($1 == "MARC_MODE") {print $2}}' $HOME/run_marc_defaults`
line=`echo $line | $AWK '{print $NF}'`
if test "$line" = "i4"; then
modeerror="defaults file $HOME/run_marc_defaults used mode $line ; this must be i8"
modeoption=error
echo $modeerror
fi
if test "$line" = "i8"; then
mode=i8
fi
fi
if test -n "$MARC_INTEGER_SIZE" ; then
mode=$MARC_INTEGER_SIZE
fi
if test -z "$mode" ; then
mode=i8
fi
case $mode in
i4)
modeerror="bad value for MARC_INTEGER_SIZE variable; only i8 is supported."
modeoption=error
echo $modeerror
;;
i8)
MARC_INTEGER_SIZE=i8
export MARC_INTEGER_SIZE
;;
*)
echo "bad value for MARC_INTEGER_SIZE variable; only i8 is supported."
exit
;;
esac
setmode=false
for arg in $* ; do
if $setmode ; then
mode=$arg
case $mode in
i4)
modeerror="bad value for mode option; only i8 is supported."
modeoption=error
echo
echo $modeerror
echo
;;
i8)
MARC_INTEGER_SIZE=i8
export MARC_INTEGER_SIZE
;;
*)
echo " "
echo "error, version mode must be i8"
echo " "
echo " use -mo i8 "
echo " "
exit
;;
esac
setmode=false
fi
if [ ${arg}X = -moX -o ${arg}X = -MOX ] ; then
echo
echo warning: the option -mo is deprecated, as of Marc 2015, only the integer*8 version is available
echo
setmode=true
fi
if [ ${arg}X = -i8X -o ${arg}X = -I8X ] ; then
MARC_INTEGER_SIZE=i8
export MARC_INTEGER_SIZE
fi
if [ ${arg}X = -i4X -o ${arg}X = -I4X ] ; then
modeerror="bad value for mode option; only i8 is supported."
modeoption=error
echo
echo $modeerror
echo
fi
done
# set to i4 version for 32 bit Linux
if test "`uname -s`" = "Linux"; then
if test "`uname -m`" = "i686"; then
mode=i4
MARC_INTEGER_SIZE=i4
export MARC_INTEGER_SIZE
fi
fi
. "$DIR/getarch"
# getting user subroutine file name
found=0
for i in "$@"; do
if test $found = 1; then
DAMASK_USER=$i
found=0
fi
case $i in
-u* | -U*)
found=1
;;
esac
done
# sourcing include_linux64 (needs DAMASK_USER to be set)
. $MARC_INCLUDE
#
#
# Dynamically determine the echo syntax
#
case "`echo '\c'`" in
'\c')
ECHO='echo -n'
ECHOTXT=' '
;;
*)
ECHO='echo'
ECHOTXT=' \c'
;;
esac
#
# Variables for the MARC environment
#
PRODUCT="Marc"
EXITMSG=$MARC_TOOLS/MESSAGES
export EXITMSG
FLEXDIR=$DIR/../flexlm/licenses
export FLEXDIR
TIMCHK=3600
export TIMCHK
BINDIR=$MARC_BIN
export BINDIR
AFMATDAT=$MARC_RUNTIME/AF_flowmat/
export AFMATDAT
export MESHERDIR
MSC_LICENSE_FINPROC=0
export MSC_LICENSE_FINPROC
#
# define directory path to global unified material database
#
MATFILE=
export MATFILE
#
# define memory limit
# first set to MEMLIMIT from include
# -ml option overrules if specified
memlimit=$MEMLIMIT
#
# Define share library path based on platforms
# This is required for using the Patran Mesher
#
if test $MACHINENAME = "HP"
then
SHLIB_PATH=$MARC_LIB:$MARC_LIB_SHARED:$SHLIB_PATH
export SHLIB_PATH
fi
# the one for IBM is defined futher down
LD_LIBRARY_PATH=$MARC_LIB_SHARED:$LD_LIBRARY_PATH:$SFMATDIR
if test -f "/etc/redhat-release"; then
ver=`cat /etc/redhat-release | sed 's/.* release \([0-9]\).\([0-9]\+\) .*/\1\2/'`
case "$ver" in
6*) LD_LIBRARY_PATH="${MARC_LIB_SHARED}rh67:$LD_LIBRARY_PATH" ;;
esac
fi
LD_LIBRARY_PATH=$MARC_LIB:$LD_LIBRARY_PATH
LD_LIBRARY_PATH=$MESHERDIR:$LD_LIBRARY_PATH
LD_LIBRARY_PATH=$LD_LIBRARY_PATH
LD_LIBRARY64_PATH=$MARC_LIB:$LD_LIBRARY64_PATH
LD_LIBRARYN32_PATH=$MARC_LIB:$LD_LIBRARYN32_PATH
export LD_LIBRARY_PATH
export LD_LIBRARY64_PATH
export LD_LIBRARYN32_PATH
atexit() {
kill -15 $$
#
if test $MPITYPE = "myrinet"
then
if test -f "$host_filt"
then
/bin/rm $host_filt
fi
fi
}
trap "atexit" 2
#
# defaults
#
prog=marc
exefile=marc
jid=
rid=
pid=
sid=
did=
vid=
user=
usernoext=
objs=
qid=background
cpu=
priority=
att=
trk=
verify=yes
prgsav=no
rmdll=no
cpdll=no
progdll=
pathdll=
error=
nprocd=0
nprocdddm=1
nprocdddmprint=
icreated=0
nprocdarg=
nsolver=0
nsolverarg=-ns
if test $nprocds
then
if test $nprocds -gt 1
then
nprocdddm=$nprocds
nprocdddmprint=$nprocds
icreated=1
nprocdarg=-nprocds
fi
fi
ntprint=0
nt=-1
nte=-1
nts=-1
ntarg=-nt
ntearg=-nte
ntsarg=-nts
nteprint=
ntsprint=
oversubscribe=false
gpuids=
nauto=
ndcoup=0
ndytran=0
noutcore=0
dllrun=0
mesh=0
itree=0
iam=
ddm_arc=0
link=
trkrun=0
DIRJOB=`pwd`
DIRSCR=$DIRJOB
DIRSCRSET=
autoforge=0
dotdat=.dat
dotdefhost=.defhost
host=
numhost=
mfile=
userhost=
makebdf=
cpinput=yes
cpresults=yes
marcdll=libmarc.$EXT_DLL
# define hostname and strip off extensions (alpha.aaa.com)
thishost=`hostname`
thishost=${thishost%%.*}
compatible=unknown
numfield=1
justlist=
feature=
mpioption=false
iprintsimufact=
SRCLIB=$MARC_LIB/srclib.a
MDSRCLIB=$MARC_LIB/mdsrc.a
#
# check run_marc_defaults file for default MPI setting
# located in the tools directory of the Marc installation
# or in the user's home directory
# format:
# MARC_MPI <mpiversion>
#
value=
file=
if test -f $DIRSCRIPT/run_marc_defaults; then
value=`$AWK '{if ($1 == "MARC_MPI") {print $2}}' $DIRSCRIPT/run_marc_defaults`
value=`echo $value | $AWK '{print $NF}'`
if test -n "$value"; then
file=$DIRSCRIPT/run_marc_defaults
fi
fi
if test -f $HOME/run_marc_defaults; then
value=`$AWK '{if ($1 == "MARC_MPI") {print $2}}' $HOME/run_marc_defaults`
value=`echo $value | $AWK '{print $NF}'`
if test -n "$value"; then
file=$HOME/run_marc_defaults
fi
fi
if test -n "$value"; then
MARC_MPITYPE=$value
notok=true
for i in "$MPI_OTHER"; do
if test "$MARC_MPITYPE" = "$i"; then
notok=false
fi
done
if test "$MARC_MPITYPE" = "$MPI_DEFAULT"; then
notok=false
fi
if $notok; then
echo " "
echo " error, incorrect option for MARC_MPI"
echo " defined in $file: $MARC_MPITYPE"
echo " valid options: $MPI_OTHER $MPI_DEFAULT"
echo " "
exit
fi
if test "$value" != "$MPI_DEFAULT"; then
exefile=marc_$value
. $MARC_INCLUDE
MDSRCLIB=$MARC_LIB/mdsrc.a_$value
if test "$MUMPSSOLVER" = MUMPS; then
MUMPSSOLVERLIBS="$MUMPSLIB_DIR/libmumps_$value.a"
fi
fi
fi
#
#
# allow scratch directory to be specified with environmental variable
# MARCSCRATCH
if test $MARCSCRATCH
then
if test -d $MARCSCRATCH
then
DIRSCR=$MARCSCRATCH
else
echo "error, scratch directory '$MARCSCRATCH'"
echo " specified via environmental variable MARCSCRATCH does not exist"
exit
fi
fi
#
##############################################################################
# parse input - arguments always come in pairs #
##############################################################################
arg=$1
if [ ${arg}X = -i8X -o ${arg}X = -I8X ] ; then
shift
arg=$1
fi
while [ -n "$arg" ]
do
shift
value=$1
case $arg in
-al* | -AL*)
LD_LIBRARY_PATH=$CUDALIB1:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH
$MARC_BIN/marc -alloc 1
exit
;;
-li* | -LI*)
justlist=yes
;;
-fe* | -FE*)
feature=$value
;;
-pr* | -PR*)
if test `dirname $value` = '.'
then
prog=`$BASENAME $value .marc`
progdll=`$BASENAME $value`
else
prog=`dirname $value`/`$BASENAME $value .marc`
progdll=`dirname $value`/`$BASENAME $value`
fi
prdir=`dirname $value`
case $prdir in
\/*)
;;
*)
prog=`pwd`/$prdir/$prog
;;
esac
;;
-j* | -J*)
jid=`$BASENAME $value $dotdat`
DIRJID=`dirname $value`
case $DIRJID in
\/*)
;;
*)
DIRJID=`pwd`/$DIRJID
;;
esac
;;
-r* | -R*)
rid=`$BASENAME $value .t08`
DIRRID=`dirname $value`
case $DIRRID in
\/*)
;;
*)
DIRRID=`pwd`/$DIRRID
;;
esac
;;
-si* | -SI*)
sid=$value
DIRSID=`dirname $value`
case $DIRSID in
\/*)
;;
*)
DIRSID=`pwd`/$DIRSID
;;
esac
;;
-pi* | -PI*)
if test -f $value.t19
then
pid=`$BASENAME $value .t19`
else
pid=`$BASENAME $value .t16`
fi
DIRPID=`dirname $value`
case $DIRPID in
\/*)
;;
*)
DIRPID=`pwd`/$DIRPID
;;
esac
;;
-bdf | -BDF)
makebdf=1
;;
-de* | -DE*)
did=`$BASENAME $value $dotdat`
DIRDID=`dirname $value`
case $DIRDID in
\/*)
;;
*)
DIRDID=`pwd`/$DIRDID
;;
esac
;;
-vf | -VF)
vid=`$BASENAME $value .vfs`
DIRVID=`dirname $value`
case $DIRVID in
\/*)
;;
*)
DIRVID=`pwd`/$DIRVID
;;
esac
;;
-u* | -U*)
user=$value
case $user in
\/*)
;;
*)
user=`pwd`/$user
;;
esac
usernoext=$user
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .f`
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .F`
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .for`
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .f90`
;;
-obj | -OBJ)
objs="$value"
;;
-q* | -Q*)
qid=$value
;;
-b* | -B*)
case $value in
y* | Y*)
qid=background
;;
n* | N*)
qid=foreground
;;
*)
;;
esac
;;
-at | -AT)
att=$value
;;
-cpu* | -CPU*)
cpu=$value
;;
-pq | -PQ*)
priority=$value
;;
-v* | -V*)
verify=$value
;;
-sa* | -SA*)
prgsav=$value
;;
-np* | -NP*)
nprocdddm=$value
nprocdddmprint=$value
case $arg in
-nps* | -NPS* | -nprocds* | -NPROCDS*)
icreated=1
nprocdarg=-nprocds
;;
esac
case $arg in
-np | -NP | -nprocd | -NPROCD)
icreated=0
nprocdarg=-nprocd
;;
esac
;;
-ns* | -NS*)
nsolver=$value
;;
-nt* | -NT*)
case $arg in
-nte | -NTE | -nthread_e* | -NTHREAD_E*)
nte=$value
;;
esac
case $arg in
-nts | -NTS | -nthread_s* | -NTHREAD_S*)
nts=$value
;;
esac
case $arg in
-nt | -NT | -nth* | -NTH* | -nthread* | -NTHREAD*)
nt=$value
;;
esac
;;
-gp* | -GP*)
gpuids=$value
;;
-it* | -IT*)
itree=$value
;;
-iam | -IAM)
iam=$value
case $value in
sfg | sfm | sim)
iprintsimufact=true
;;
esac
;;
-au* | -AU*)
nauto=$value
echo
echo warning: the option -au is no longer supported and will be ignored
echo
;;
-dc* | -DC*)
ndcoup=$value
;;
-dy* | -DY*)
ndytran=$value
;;
-ou* | -OU*)
noutcore=$value
;;
-dll | -DLL)
dllrun=$value
;;
-trk | -TRK)
trkrun=$value
;;
-ddm | -DDM)
ddm_arc=$value
;;
-me | -ME )
mesh=$value
;;
-ml | -ML )
memlimit=$value
;;
-mo | -MO )
;;
-mpi | -MPI )
mpioption=true
MARC_MPITYPE=$value
if test "$value" != "$MPI_DEFAULT"; then
exefile=marc_$value
. $MARC_INCLUDE
MDSRCLIB=$MARC_LIB/mdsrc.a_$value
if test "$MUMPSSOLVER" = MUMPS; then
MUMPSSOLVERLIBS="$MUMPSLIB_DIR/libmumps_$value.a"
fi
else
exefile=marc
. $MARC_INCLUDE
MDSRCLIB=$MARC_LIB/mdsrc.a
if test "$MUMPSSOLVER" = MUMPS; then
MUMPSSOLVERLIBS="$MUMPSLIB_DIR/libmumps_intelmpi.a"
fi
fi
;;
-dir* | -DIR*)
DIRJOB=$value
case $DIRJOB in
\/*)
;;
*)
DIRJOB=`pwd`/$DIRJOB
;;
esac
if test -z "$DIRSCRSET"
then
DIRSCR=$DIRJOB
fi
;;
-sd* | -SD*)
DIRSCR=$value
DIRSCRSET=yes
case $DIRSCR in
\/*)
;;
*)
DIRSCR=`pwd`/$DIRSCR
;;
esac
;;
-ho* | -HO*)
host=$value
;;
-co* | -CO*)
compatible=$value
;;
-ci* | -CI*)
cpinput=$value
;;
-cr* | -CR*)
cpresults=$value
;;
*)
error="$error
$arg: invalid option"
break
;;
esac
case $value in
-*)
error="$error
$arg: invalid name $value"
break
;;
esac
shift
arg=$1
if [ ${arg}X = -i8X -o ${arg}X = -I8X -o ${arg}X = -i4X -o ${arg}X = -I4X ] ; then
shift
arg=$1
fi
done
argc=`expr $# % 2`
if test $argc -eq 1
then
#
# odd number of arguments
#
error="$error
argument list incomplete"
fi
if test $nprocdddm -gt 0
then
nprocd=$nprocdddm
fi
if test $nsolver -gt 0
then
if test $nsolver -gt $nprocd
then
nprocd=$nsolver
fi
fi
# oversubscribe case
if test $nprocd -gt $(/usr/bin/nproc); then
if test "$ALLOWMPIOVERSUBSCRIBE" != "true"; then
error="$error
The requested number of domains ($nprocd) is larger than the number
of processors ($(/usr/bin/nproc)). As such an oversubscription may result in
very poor performance, the analysis will not be started.
Running an analysis in case of oversubscription can be forced
by defining an environment variable \"ALLOWMPIOVERSUBSCRIBE\"
with the value \"true\"."
oversubscribe=true
fi
fi
# Set defaults
if test $nt -eq -1
then
nt=${MARC_NUMBER_OF_THREADS:-0}
fi
if test $nt -lt 0
then
nt=0
fi
if test $nte -eq -1
then
nte=${MARC_NUMBER_OF_THREADS:-0}
fi
if test $nte -lt 0
then
nte=0
fi
if test $nts -eq -1
then
nts=${MARC_NUMBER_OF_THREADS:-0}
fi
if test $nts -lt 0
then
nts=0
fi
#
# set number of element loop threads
#
ntprint=$nt
nteprint=$nte
# copy from -nprocd[s]
if test $nprocdddm -gt 1
then
nteprint=$nprocdddm
fi
# override with -nthread_elem option
if test $nte -ne 0
then
nteprint=$nte
fi
# check for minimum 1 threads per processes for DDM
if test $nprocdddm -gt 1
then
if test $nteprint -lt $nprocdddm
then
nteprint=$nprocdddm
fi
fi
nte=$nteprint
#
# set number of Solver threads
#
ntsprint=$nts
# copy from -nthread or -nprocd[s]
if test $ntprint -ne 0
then
ntsprint=$ntprint
else
if test $nprocdddm -gt 1
then
ntsprint=$nprocdddm
fi
fi
# override with -nthread_solver option
if test $nts -ne 0
then
ntsprint=$nts
fi
# check for minimum 1 threads per solver process.
if test $nsolver -lt $nprocdddm
then
if test $ntsprint -lt $nsolver
then
ntsprint=$nsolver
fi
else
if test $ntsprint -lt $nprocdddm
then
ntsprint=$nprocdddm
fi
fi
if test $ntsprint -eq 1
then
set ntsprint=0
fi
nts=$ntsprint
# set stack size for multi-threading.
export KMP_MONITOR_STACKSIZE=7M
export OMP_STACKSIZE=7M
#
# deprecate -nthread option at arugment of marc
nt=0
# Reset nprocdddmm, nsolver and threads if not given.
if test $nprocdddm -eq 0
then
nprocdarg=
fi
if test $nprocdddm -eq 0
then
nprocdddmprint=
fi
if test $nprocdddm -eq 0
then
nprocdddm=
fi
nsolverprint=$nsolver
if test $nsolver -eq 0
then
nsolverprint=
fi
# end of threads setting.
gpuoption=
if test "$gpuids" = "" ; then
gpuoption=
else
gpuoption="-gp $gpuids"
fi
if test "$gpuids" = "" ; then
export LD_LIBRARY_PATH=$CUDALIB1:$LD_LIBRARY_PATH
else
MARCCUDALIBS=$MARCCUDALIBS2
export LD_LIBRARY_PATH=$CUDALIB2:$LD_LIBRARY_PATH
fi
# Linux 64 + HPMPI, Below code is taken from include_linux64
if test $MPITYPE = hpmpi -a "$ARCHITECTURE" = "linux_amd64"
then
export MPIHPSPECIAL="$MPIHPSPECIAL -e LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
fi
if test "$iam" = sim ; then
SFLIB="-L$SFMATDIR -lMBA_Grain"
fi
if test $nprocd -gt 1; then
if test -f $jid$dotdefhost; then
if test "$host" = ""; then
host=$jid$dotdefhost
fi
fi
if test -f hostfile_qa_$nprocd; then
if test "$host" = ""; then
host=hostfile_qa_$nprocd
fi
fi
fi
if test "$dllrun" -gt 0; then
exefile=exe_marc
prog=exe_marc
program=$exefile
bd=$MARC_BIN/
if test "$dllrun" -eq 1 || test "$dllrun" -eq 2; then
dotdat=.inp
fi
if test "$progdll"; then
/bin/cp ${progdll}_$marcdll $DIRJOB/$marcdll
rmdll=yes
pathdll=yes
progdll=${progdll}_$marcdll
else
progdll=$marcdll
fi
if test "$user"; then
. $MARC_TOOLS/make_marc_user_dll $DIRJOB $user
user=
if test $prgsav = no; then
rmdll=yes
fi
if test $prgsav = yes; then
cpdll=yes
rmdll=yes
fi
pathdll=yes
fi
fi
##############################################################################
# check parameter validity #
##############################################################################
while test forever; do
#
# check for input file existence
#
if test $nprocdddm -gt 1 -a $icreated -eq 0; then
if test ! -f $DIRJID/1$jid$dotdat; then
if test "$jid" != "" ; then
error="$error
input file $DIRJID/1$jid$dotdat not accessible"
fi
fi
else
if test ! -f $DIRJID/$jid$dotdat; then
if test "$jid" != "" ; then
error="$error
input file $DIRJID/$jid$dotdat not accessible"
fi
fi
fi
if test $nprocd -gt 1; then
if test "$host" ; then
if test ! -f $host; then
error="$error
host name file $host not accessible"
fi
fi
fi
#
# check if the job is already running in the background
#
if test -f $DIRJOB/$jid.pid; then
error="$error
job is already running (the file $jid.pid exists)"
fi
#
# if the program name is other than marc, then
# assume that this is a program in the users local directory
#
bd=$MARC_BIN/
case $prog in
marc | MARC | $exefile)
program=$exefile
if test "$rid"
then
if test ! -f $DIRRID/$rid.t08
then
error="$error
restart file $DIRRID/$rid.t08 not accessible"
fi
fi
if test "$pid"
then
if test ! -f $DIRPID/$pid.t16
then
if test ! -f $DIRPID/$pid.t19
then
error="$error
post file $DIRPID/$pid.t16 or $DIRPID/$pid.t19 not accessible"
fi
fi
fi
if test "$user"
then
if test ! -f $user
then
error="$error
user subroutine file $user not accessible"
fi
fi
if test "$objs"
then
missingobjs=
for o in $objs
do
if test ! -f "$o"
then
if test -z "$missingobjs"
then
missingobjs="$o"
else
missingobjs="$missingobjs $o"
fi
fi
done
if test -n "$missingobjs"
then
error="$error
user object/library file(s) $missingobjs not accessible"
fi
fi
if test "$did"
then
if test $nprocdddm -gt 1 -a $icreated -eq 0
then
if test ! -f $DIRDID/1$did$dotdat
then
error="$error
defaults file $DIRDID/1$did$dotdat not accessible"
fi
else
if test ! -f $DIRDID/$did$dotdat
then
error="$error
defaults file $DIRDID/$did$dotdat not accessible"
fi
fi
fi
if test "$vid"
then
if test $nprocdddm -gt 1 -a $icreated -eq 0
then
if test ! -f $DIRVID/1$vid.vfs
then
error="$error
view factor file $DIRVID/1$vid.vfs not accessible"
fi
else
if test ! -f $DIRVID/$vid.vfs
then
error="$error
view factor file $DIRVID/$vid.vfs not accessible"
fi
fi
fi
if $mpioption
then
notok=true
for i in "$MPI_OTHER"; do
if test "$MARC_MPITYPE" = "$i"; then
notok=false
fi
done
if test "$MARC_MPITYPE" = "$MPI_DEFAULT"; then
notok=false
fi
if $notok; then
error="$error
incorrect option for -mpi option: $MARC_MPITYPE (valid: $MPI_OTHER $MPI_DEFAULT)"
fi
fi
;;
*)
program=$prog.marc
case $prog in
\/* | \.\/*)
bd=
;;
*)
bd=`pwd`/
;;
esac
if test "$rid"
then
if test ! -f $DIRRID/$rid.t08
then
error="$error
restart file $DIRRID/$rid.t08 not accessible"
fi
fi
if test "$pid"
then
if test ! -f $DIRPID/$pid.t16
then
if test ! -f $DIRPID/$pid.t19
then
error="$error
post file $DIRPID/$pid.t16 and $DIRPID/$pid.t19 not accessible"
fi
fi
fi
if test "$user"
then
error="$error
program option may not be used with user subroutine"
fi
if test "$objs"
then
error="$error
program option may not be used with user objects or libraries"
fi
if test "$did"
then
if test $nprocdddm -gt 1 -a $icreated -eq 0
then
if test ! -f $DIRDID/1$did$dotdat
then
error="$error
defaults file $DIRDID/1$did$dotdat not accessible"
fi
else
if test ! -f $DIRDID/$did$dotdat
then
error="$error
defaults file $DIRDID/$did$dotdat not accessible"
fi
fi
fi
if test "$ndcoup"
then
if test $ndcoup -gt 3
then
error="$error
incorrect option for contact decoupling "
fi
fi
if test "$ndytran"
then
if test $ndytran -gt 1
then
error="$error
incorrect option for Marc-Dytran Switch "
fi
fi
if $mpioption
then
if test ! -x $MARC_BIN/$exefile
then
error="$error
incorrect option for -mpi option: $MARC_MPITYPE "
fi
fi
;;
esac
##############################################################################
# check argument integrity #
##############################################################################
if test "$jid"
then
:
else
if test "$user"
then
# allow user sub without giving job id
qid=foreground
verify=no
else
error="$error
job id required"
fi
fi
case $qid in
S* | s*)
qid=short
;;
L* | l*)
qid=long
;;
V* | v*)
qid=verylong
;;
B* | b*)
qid=background
;;
F* | f*)
qid=foreground
;;
A* | a*)
qid=at
;;
*)
error="$error
bad value for queue_id option"
;;
esac
case $prgsav in
N* | n*)
prgsav=no
;;
Y* | y*)
prgsav=yes
;;
*)
error="$error
bad value for save option"
;;
esac
case $verify in
N* | n*)
verify=no
;;
Y* | y*)
verify=yes
;;
*)
error="$error
bad value for verify option"
;;
esac
case $nprocdddm in
-* )
error="$error
bad value for nprocd option"
;;
esac
case $nt in
-* )
error="$error
bad value for nt option"
;;
esac
case $itree in
-* )
error="$error
bad value for itree option"
;;
esac
case $iam in
-* )
error="$error
bad value for iam option"
;;
esac
case $compatible in
N* | n*)
compatible=no
;;
Y* | y*)
compatible=yes
;;
unknown)
;;
*)
error="$error
bad value for comp option"
;;
esac
case $cpinput in
N* | n*)
cpinput=no
;;
Y* | y*)
cpinput=yes
;;
*)
error="$error
bad value for copy input option"
;;
esac
case $cpresults in
N* | n*)
cpresults=no
;;
Y* | y*)
cpresults=yes
;;
*)
error="$error
bad value for copy results option"
;;
esac
#
# check for external file to run
#
if test -f $MARC_TOOLS/run_marc_check
then
. $MARC_TOOLS/run_marc_check
fi
##############################################################################
# interact with the user to get the required information to run marc or #
# other marc system program #
##############################################################################
deletelog=yes
if test $qid = background -a $verify = no
then
echo \
"
Program name : $prog
Marc shared lib : $progdll
Version type : $mode
Job ID : $DIRJID/$jid
User subroutine name : $user
User objects/libs : $objs
Restart file job ID : $rid
Substructure file ID : $sid
Post file job ID : $pid
Defaults file ID : $did
View Factor file ID : $vid
Save generated module: $prgsav
MPI library : $MPITYPE
DDM processes : $nprocdddmprint
Element loop threads : $nteprint
Solver processes : $nsolverprint
Solver threads : $ntsprint
GPGPU option : $gpuids
Host file name : $host" > $jid.log
if test "$iprintsimufact" = true ; then
echo "DDM with ARC Mapper : $ddm_arc" >> $jid.log
fi
echo \
"Message passing type : $itree
Run job in queue : $qid
Run directory : $DIRJOB
Scratch directory : $DIRSCR
Memory limit in Mbyte: $memlimit" >> $jid.log
deletelog=no
fi
echo \
"
Program name : $prog
Marc shared lib : $progdll
Version type : $mode
Job ID : $DIRJID/$jid
User subroutine name : $user
User objects/libs : $objs
Restart file job ID : $rid
Substructure file ID : $sid
Post file job ID : $pid
Defaults file ID : $did
View Factor file ID : $vid
Save generated module: $prgsav
MPI library : $MPITYPE
DDM processes : $nprocdddmprint
Element loop threads : $nteprint
Solver processes : $nsolverprint
Solver threads : $ntsprint"
if test "$iprintsimufact" = true ; then
echo "DDM with ARC Mapper : $ddm_arc"
fi
echo \
"GPGPU option : $gpuids
Host file name : $host
Message passing type : $itree
Run job in queue : $qid
Run directory : $DIRJOB
Scratch directory : $DIRSCR
Memory limit in Mbyte: $memlimit"
case $qid in
s* | S* | l* | L* | v* | V* )
echo \
"Queue priority : $priority
Queue CPU limit : $cpu
Queue start time : $att"
;;
# * )
# echo \
#" "
# ;;
esac
if test "$modeoption"
then
error=$modeerror
fi
if test "$error"
then
if test $verify = yes
then
$ECHO "$error
Please correct or quit(correct,quit,<correct>): $ECHOTXT"
error=
read answer
case $answer in
q* | Q*)
answer=quit
;;
*)
answer=correct
;;
esac
else
$ECHO "$error
$ECHOTXT"
echo " "
if test "$deletelog" = no
then
$ECHO "$error
$ECHOTXT" >> $jid.log
echo " " >> $jid.log
fi
answer=quit
fi
else
if test $verify = yes
then
$ECHO "
Are these parameters correct (yes,no,quit,<no>)? $ECHOTXT"
read answer
case $answer in
q* | Q*)
answer=quit
;;
y* | Y*)
answer=yes
;;
*)
answer=no
;;
esac
else
answer=yes
fi
fi
case $answer in
no | correct)
##############################################################################
# prompt for each value #
##############################################################################
$ECHO "
Program name ($prog)? $ECHOTXT"
read value
if test "$value"
then
prog=$value
fi
$ECHO "Job ID ($jid)? $ECHOTXT"
read value
if test "$value"
then
jid=`$BASENAME $value $dotdat`
DIRJID=`dirname $value`
case $DIRJID in
\/*)
;;
*)
DIRJID=`pwd`/$DIRJID
;;
esac
fi
$ECHO "User subroutine name ($user)? $ECHOTXT"
read value
if test "$value"
then
case $value in
-*)
user=
;;
*)
user=$value
case $user in
\/*)
;;
*)
user=`pwd`/$user
;;
esac
usernoext=$user
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .f`
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .F`
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .for`
usernoext=`dirname $usernoext`/`$BASENAME $usernoext .f90`
;;
esac
fi
$ECHO "User objects or libraries ($objs)? $ECHOTXT"
read value
if test "$value"
then
case $value in
-*)
objs=
;;
*)
objs="$value"
;;
esac
fi
$ECHO "Restart File Job ID ($rid)? $ECHOTXT"
read value
if test "$value"
then
case $value in
-*)
rid=
;;
*)
rid=`$BASENAME $value .t08`
DIRRID=`dirname $value`
case $DIRRID in
\/*)
;;
*)
DIRRID=`pwd`/$DIRRID
;;
esac
;;
esac
fi
$ECHO "Substructure File ID ($sid)? $ECHOTXT"
read value
if test "$value"
then
case $value in
-*)
sid=
;;
*)
sid=$value
DIRSID=`dirname $value`
case $DIRSID in
\/*)
;;
*)
DIRSID=`pwd`/$DIRSID
;;
esac
;;
esac
fi
$ECHO "Post File Job ID ($pid)? $ECHOTXT"
read value
if test "$value"
then
case $value in
-*)
pid=
;;
*)
pid=$value
DIRPID=`dirname $value`
case $DIRPID in
\/*)
;;
*)
DIRPID=`pwd`/$DIRPID
;;
esac
;;
esac
fi
$ECHO "Defaults File ID ($did)? $ECHOTXT"
read value
if test "$value"
then
case $value in
-*)
did=
;;
*)
did=`$BASENAME $value $dotdat`
DIRDID=`dirname $value`
case $DIRDID in
\/*)
;;
*)
DIRDID=`pwd`/$DIRDID
;;
esac
;;
esac
fi
$ECHO "View Factor File ID ($vid)? $ECHOTXT"
read value
if test "$value"
then
case $value in
-*)
vid=
;;
*)
vid=`$BASENAME $value .vfs`
DIRVID=`dirname $value`
case $DIRVID in
\/*)
;;
*)
DIRVID=`pwd`/$DIRVID
;;
esac
;;
esac
fi
$ECHO "Save generated module ($prgsav)? $ECHOTXT"
read value
if test "$value"
then
prgsav=$value
fi
$ECHO "Run on tasks ($nprocdddm) tasks? $ECHOTXT"
read value
if test "$value"
then
nprocdddm=$value
nprocdddmprint=$value
fi
$ECHO "Run on ($nte) Element loop threads ? $ECHOTXT"
read value
if test "$value"
then
nte=$value
fi
$ECHO "Run on ($nsolver) solvers ? $ECHOTXT"
read value
if test "$value"
then
nsolver=$value
fi
$ECHO "Run on ($nts) Solver threads ? $ECHOTXT"
read value
if test "$value"
then
nts=$value
fi
#
if test $nprocdddm -gt 0
then
nprocd=$nprocdddm
fi
if test $nsolver -gt 0
then
if test $nsolver -gt $nprocd
then
nprocd=$nsolver
fi
fi
# Element loop threads.
if test $nte -eq -1
then
nte=${MARC_NUMBER_OF_THREADS:-0}
fi
if test $nte -lt 0
then
nte=0
fi
nteprint=$nte
# Copy from ddm
if test $nprocdddm -gt 1
then
nteprint=$nprocdddm
fi
# override with -nthread_elem option
if test $nte -ne 0
then
nteprint=$nte
fi
# check for minimum 1 threads per processes for DDM
if test $nprocdddm -ne 0
then
if test $nteprint -lt $nprocdddm
then
nteprint=$nprocdddm
fi
fi
nte=$nteprint
# Solver threads.
if test $nts -eq -1
then
nts=${MARC_NUMBER_OF_THREADS:-0}
fi
if test $nts -lt 0
then
nts=0
fi
ntsprint=$nts
# Copy from ddm
if test $nprocdddm -gt 1
then
ntsprint=$nprocdddm
fi
# override with -nthread_solver option
if test $nts -ne 0
then
ntsprint=$nts
fi
# check for minimum 1 threads per solver process.
if test $nsolver -lt $nprocdddm
then
if test $ntsprint -lt $nsolver
then
ntsprint=$nsolver
fi
else
if test $ntsprint -lt $nprocdddm
then
ntsprint=$nprocdddm
fi
fi
if test $ntsprint -eq 1
then
set ntsprint=0
fi
nts=$ntsprint
# Update print variable for -nsolver option
nsolverprint=$nsolver
if test $nsolver -eq 0
then
nsolverprint=
fi
$ECHO "GPGPU id option ($gpuids)? $ECHOTXT"
read value
if test "$value"
then
gpuids=$value
fi
if test "$gpuids" = "" ; then
gpuoption=
else
gpuoption="-gp $gpuids"
fi
if test "$gpuids" = "" ; then
export LD_LIBRARY_PATH=$CUDALIB1:$LD_LIBRARY_PATH
else
MARCCUDALIBS=$MARCCUDALIBS2
export LD_LIBRARY_PATH=$CUDALIB2:$LD_LIBRARY_PATH
fi
if test $MPITYPE = hpmpi -a "$ARCHITECTURE" = "linux_amd64"
then
export MPIHPSPECIAL="$MPIHPSPECIAL -e LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
fi
#
if test $nprocd -gt 1
then
$ECHO "Message passing type ($itree)? $ECHOTXT"
read value
if test "$value"
then
itree=$value
fi
$ECHO "Host file name ($host)? $ECHOTXT"
read value
if test "$value"
then
host=$value
fi
if test $nprocdddm -gt 1
then
$ECHO "Single input file? $ECHOTXT"
read value
case $value in
y* | Y*)
icreated=1
nprocdarg=-nprocds
;;
esac
$ECHO "Compatible machines for DDM ($compatible)? $ECHOTXT"
read value
if test "$value"
then
compatible=$value
fi
$ECHO "Copy input files to remote hosts ($cpinput)? $ECHOTXT"
read value
if test "$value"
then
cpinput=$value
fi
$ECHO "Copy post files from remote hosts ($cpresults)? $ECHOTXT"
read value
if test "$value"
then
cpresults=$value
fi
fi
fi
$ECHO "Run the job in the queue ($qid)? $ECHOTXT"
read value
if test "$value"
then
qid=$value
fi
case $qid in
s* | S* | l* | L* | v* | V* )
$ECHO "Queue priority ($priority)? $ECHOTXT"
read value
if test "$value"
then
priority=$value
fi
$ECHO "Job starts at ($att)? $ECHOTXT"
read value
if test "$value"
then
att=$value
fi
$ECHO "Queue CPU limit ($cpu)? $ECHOTXT"
read value
if test "$value"
then
cpu=$value
fi
;;
* )
;;
esac
$ECHO "Run directory ($DIRJOB)? $ECHOTXT"
read value
if test "$value"
then
DIRJOB=$value
DIRSCR=$DIRJOB
fi
$ECHO "Scratch directory ($DIRSCR)? $ECHOTXT"
read value
if test "$value"
then
DIRSCR=$value
fi
;;
quit)
if test "$deletelog" = no -a $oversubscribe = true
then
echo " " >> $jid.log
echo " Exit number 8" >> $jid.log
echo " " >> $jid.log
fi
exit 1
;;
*)
break
;;
esac
if test $nt -eq -1
then
nt=${MARC_NUMBER_OF_THREADS:-0}
fi
if test $nt -lt 0
then
nt=0
fi
done
#
if test $nt -eq 0
then
ntarg=
fi
if test $nt -eq 0
then
ntprint=
fi
if test $nt -eq 0
then
nt=
fi
if test $nte -eq 0
then
ntearg=
fi
if test $nte -eq 0
then
nteprint=
fi
if test $nte -eq 0
then
nte=
fi
if test $nts -eq 0
then
ntsarg=
fi
if test $nts -eq 0
then
ntsprint=
fi
if test $nts -eq 0
then
nts=
fi
#
if test "$dllrun" -gt 0; then
exefile=exe_marc
prog=exe_marc
program=$exefile
bd=$MARC_BIN/
if test "$user"; then
. $MARC_TOOLS/make_marc_user_dll $DIRJOB $user
user=
pathdll=yes
if test $prgsav = no; then
rmdll=yes
fi
if test $prgsav = yes; then
cpdll=yes
rmdll=yes
fi
fi
if test "$pathdll"; then
#
# reset share lib path
#
if test $MACHINENAME = "HP"
then
SHLIB_PATH=$DIRJOB:$SHLIB_PATH
export SHLIB_PATH
fi
if test $MACHINENAME = "IBM"
then
LIBPATH=$DIRJOB:$LIBPATH
export LIBPATH
fi
#
LD_LIBRARY_PATH=$DIRJOB:$LD_LIBRARY_PATH
LD_LIBRARY64_PATH=$DIRJOB:$LD_LIBRARY64_PATH
LD_LIBRARYN32_PATH=$DIRJOB:$LD_LIBRARYN32_PATH
export LD_LIBRARY_PATH
export LD_LIBRARY64_PATH
export LD_LIBRARYN32_PATH
fi
fi
# end of dllrun>0
if test $program = $exefile -o $program = $prog.marc
then
# delete the old .log file unless we run in the background
if test "$deletelog" = yes
then
if test "$jid"
then
/bin/rm $jid.log 2>/dev/null
fi
else
echo
echo running the job in the background, see $jid.log
echo
fi
#
# check if this is an autoforge or rezoning or radiation job
#
if test $nprocd -eq 1 -a "$jid"
then
line=`$AWK '/^[eE][nN][dD]/ {exit} ; {print}' $DIRJID/${jid}$dotdat | grep -i "^autoforge"`
if test "$line"
then
autoforge=1
fi
line=`$AWK '/^[eE][nN][dD]/ {exit} ; {print}' $DIRJID/${jid}$dotdat | grep -i "^rezoning"`
if test "$line"
then
autoforge=1
fi
line=`$AWK '/^[eE][nN][dD]/ {exit} ; {print}' $DIRJID/${jid}$dotdat | grep -i "^radiation"`
if test "$line"
then
autoforge=1
fi
fi
#
# check that jobname for restarted run is not the same
# as restart file basename
#
if test "$rid"
then
if test "$jid" = "$rid"
then
echo " "
echo "ERROR: job name of current run is the same as job name"
echo " of the restarted job"
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
echo "ERROR: job name of current run is the same as job name" >> $jid.log
echo " of the restarted job" >> $jid.log
echo " " >> $jid.log
echo " Exit number 8" >> $jid.log
echo " " >> $jid.log
fi
exit 1
fi
fi
#
# user objects/libraries used
#
if test "$objs"
then
program="$DIRJOB/$jid.marc"
case $program in
\/* | \.\/*)
bd=
;;
*)
bd=`pwd`/
;;
esac
link=yes
fi
#
# user subroutine used
#
# add DAMASK options for linking
DAMASK="-lstdc++"
if test "$user"
then
program=$usernoext.marc
case $program in
\/* | \.\/*)
bd=
;;
*)
bd=`pwd`/
;;
esac
link=yes
fi
#
# Special case for IBM using POE but not an SP machine
# in this case we always need a host file, also for serial jobs.
#
if test $MACHINENAME = IBM -a $MPITYPE = hardware -a "$MACHINETYPE" = NONSP
then
MP_HOSTFILE=${jid}.host
if test -f $jid.host
then
/bin/rm $jid.host 2> /dev/null
fi
if test $nprocd -gt 1
then
numdom=$nprocd
while test $numdom -gt 0
do
hostname -s >> $MP_HOSTFILE
numdom=`echo $numdom | $AWK '{sum=$1-1}; {print sum}'`
done
else
hostname -s > $MP_HOSTFILE
fi
fi
#
# check ssh for all hosts in host file
#
if test $nprocd -gt 1
then
if test $MPITYPE = "intelmpi" -a "$INTELMPI_VERSION" = "HYDRA"
then
# get host list
if test "$host"
then
line=`grep -v '^#' $host | $AWK '{host=$1;num=$2;for (i=1;i<=num;i++) print host}' | uniq`
# count failing hosts
counter=0
for i in $line
do
$RSH -o BatchMode=yes -o ConnectTimeout=10 $i uname -n
status=$?
if [[ $status != 0 ]] ; then
counter=$((counter+1))
if [ "$counter" = "1" ]; then
echo " "
echo " error - connection test failed... "
echo " "
fi
echo " "
echo " connection test with ssh failed on host $i"
echo " check the following command: ssh $i uname -n "
echo " "
fi
done
# echo error message and quit
if test $counter -ne 0
then
echo " "
echo " A parallel job using IntelMPI cannot be started. "
echo " The ssh command must be working correctly between "
echo " the computers used in the analysis. Furthermore, "
echo " it must be set up such that it does not prompt the "
echo " user for a password. "
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
echo " A parallel job using IntelMPI cannot be started. ">> $jid.log
echo " The ssh command must be working correctly between ">> $jid.log
echo " the computers used in the analysis. Furthermore, ">> $jid.log
echo " it must be set up such that it does not prompt the ">> $jid.log
echo " user for a password. ">> $jid.log
echo " " >> $jid.log
echo " Exit number 8" >> $jid.log
echo " " >> $jid.log
fi
exit 1
fi
fi
fi
fi
#
# check correctness of host file; fix for user sub
#
if test $nprocd -gt 1
then
# construct the path name to the executable (execpath)
execpath=$MARC_BIN/$exefile
usersub=0
if test $program = $prog.marc
then
execpath=$prog.marc
usersub=1
fi
if test "$objs"
then
execpath="$DIRJOB/$jid.marc"
usersub=1
fi
if test "$user"
then
execpath=$usernoext.marc
usersub=1
fi
export execpath
execname=`$BASENAME $execpath`
if test "$host"
then
userhost=$host
case $userhost in
\/* | \.\/*)
;;
*)
userhost=`pwd`/$userhost
;;
esac
# check that the number of processes specified in the hostfile is
# equal to nprocd specified by -nprocd.
numproc=`grep -v '^#' $host | $AWK -v sum=0 '{sum=sum+$2}; END {print sum}'`
if test $nprocd -ne $numproc
then
echo " "
echo "error, the number of processes specified in the host file"
echo "must be equal to the number of processes given by -nprocd/-nsolver"
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
echo "error, the number of processes specified in the host file" >> $jid.log
echo "must be equal to the number of processes given by -nprocd/-nsolver" >> $jid.log
echo " " >> $jid.log
fi
exit 1
fi
# check for Myrinet that the number of processes per host is
# less than number of available user ports, 5
# .gmpi directory must exist in user's home directory
# and must have write permission from remote hosts
if test $MPITYPE = "myrinet"
then
numproc=`grep -v '^#' $host | $AWK -v sum=1 '{if( $2 > 5) sum=6}; END {print sum}'`
if test $numproc -gt 5
then
echo " "
echo "error, for Myrinet the number of processes specified "
echo "in the hostfile must not exceed 5 for a hostname"
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
echo "error, for Myrinet the number of processes specified " >> $jid.log
echo "in the hostfile must not exceed 5 for a hostname" >> $jid.log
echo " " >> $jid.log
fi
exit 1
fi
if test $MPIVERSION = "MPICH-GM1.2.1..7"
then
if test ! -d ~/.gmpi
then
echo " "
echo "error, for Myrinet a .gmpi directory must exist "
echo "under the user's home directory"
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
echo "error, for Myrinet a .gmpi directory must exist " >> $jid.log
echo "under the user's home directory" >> $jid.log
echo " " >> $jid.log
fi
exit 1
fi
fi
if test $MPIVERSION = "MPICH-GM1.2.1..7"
then
homedir=`echo ~`
for i in `grep -v '^#' $host | $AWK '{if (NF > 0) print $1}'`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
$RSH $i /bin/touch $homedir/.gmpi/$jid.$$ 2> tmp.$$
if test -s tmp.$$
then
echo " "
echo "error, for Myrinet a shared .gmpi directory must exist "
echo "under the user's home directory "
echo "with remote write permission"
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
echo "error, for Myrinet a shared .gmpi directory must exist " >> $jid.log
echo "under the user's home directory " >> $jid.log
echo "with remote write permission" >> $jid.log
echo " " >> $jid.log
fi
exit 1
else
/bin/rm tmp.$$
if test -f $jid.$$
then
/bin/rm $jid.$$
fi
fi
fi
done
fi
fi
# construct the host file $jid.host which is used by mpirun
# skip lines starting with # and only consider lines with more than
# one word in them. Note that the hostfile given to this script
# has two columns: the host name and the number of shared processes
# to run on this host. mpirun wants the number of _other_
# processes to run in addition to the one being run on the machine
# on which the job is started. hence the $2-1 for fnr == 1.
if test -f $jid.host
then
/bin/rm $jid.host 2> /dev/null
fi
if test $MPITYPE = hpmpi -o $MACHINENAME = HP -a $MPITYPE = hardware
then
# HPMPI or HP hardware MPI
grep -v '^#' $host | $AWK -v path=$execpath -v en=$execname -v us=$usersub \
-v mpihpspecial="$MPIHPSPECIAL" \
'{if ( NF > 0) {\
fnr++ ; \
printf("-h %s -np %s",$1,$2); \
printf(" %s",mpihpspecial); \
if ( NF == 2 ) printf(" %s\n",path);\
if ( NF >= 3 ) printf(" -e MPI_WORKDIR=%s", $3);\
if ( NF >= 3 ) if (us) printf(" %s/%s\n",$3,en); else printf(" %s\n",path) \
}\
}' > $jid.host
# end HPMPI or HP hardware MPI
elif test $MACHINENAME = IBM -a $MPITYPE = hardware -a "$MACHINETYPE" = NONSP
then
# IBM using hardware MPI (POE)
MP_HOSTFILE=$jid.host
grep -v '^#' $host | $AWK '{host=$1;num=$2;for (i=1;i<=num;i++) print host}' > $jid.host
# end IBM using hardware MPI (POE)
# for Intel MPI, need to create a machinefile for DMP
elif test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
then
# Intel MPI
if test -f $jid.mfile
then
/bin/rm $jid.mfile 2> /dev/null
fi
/bin/cp $host $jid.host
grep -v '^#' $host | $AWK '{host=$1;num=$2;for (i=1;i<=num;i++) print host}' > $jid.mfile
# end Intel MPI for DMP
# for Solaris HPC 7.1, need to create a machinefile for DMP
elif test $MACHINENAME = "SUN" -a $MPITYPE = "hardware"
then
# Solaris HPC 7.1
if test -f $jid.mfile
then
/bin/rm $jid.mfile 2> /dev/null
fi
grep -v '^#' $host | $AWK '{host=$1;num=$2;for (i=1;i<=num;i++) print host}' > $jid.mfile
# end Solaris HPC 7.1 for DMP
# for Myrinet, construct a configuration file in ~/.gmpi
# this must be readable by each process
# format is (hostname) (port number) for each process
elif test $MPITYPE = "myrinet"
then
if test $MPIVERSION = "MPICH-GM1.2.1..7"
then
echo $nprocd > ~/.gmpi/$jid.host
grep -v '^#' $host | $AWK \
'BEGIN {iport[0] = 2; \
iport[1] = 4; \
iport[2] = 5; \
iport[3] = 6; \
iport[4] = 7 \
} \
{if ( NF > 0 ) \
for(iproc = 0; iproc < $2; iproc++) printf("%s %d\n",$1,iport[iproc]); \
}' >> ~/.gmpi/$jid.host
else
# this is for mpich-1.2.5 and later, using the -pg option
# format: host nproc executable user arguments
# the arguments are added later
grep -v '^#' $host | $AWK -v path=$execpath -v en=$execname -v us=$usersub -v user=`whoami` \
'{if ( NF > 0) {\
fnr++ ; \
if ( fnr == 1 ) printf("%s %d",$1,$2-1); \
else printf("%s %s",$1,$2); \
if ( NF == 2 ) printf(" %s %s\n",path,user);\
if ( NF == 3 ) if (us) printf(" %s/%s %s\n",$3,en,user); else printf(" %s %s\n",path,user) ;\
if ( NF == 4 ) if (us) printf(" %s/%s %s\n",$3,en,user); else printf(" %s/bin/%s %s\n",$4,en,user) \
}\
}' > $jid.host
fi
# end Myrinet
elif test $MACHINENAME = DEC -a $MPITYPE = hardware
then
# Compaq MPI via Memory Channel
grep -v '^#' $host | $AWK '{if (NF > 0) print $1}' > $jid.host
# end Compaq MPI
else
# MPICH
grep -v '^#' $host | $AWK -v path=$execpath -v en=$execname -v us=$usersub \
'{if ( NF > 0) {\
fnr++ ; \
if ( fnr == 1 ) printf("%s %d",$1,$2-1); \
else printf("%s %s",$1,$2); \
if ( NF == 2 ) printf(" %s\n",path);\
if ( NF == 3 ) if (us) printf(" %s/%s\n",$3,en); else printf(" %s\n",path) ;\
if ( NF == 4 ) if (us) printf(" %s/%s\n",$3,en); else printf(" %s/bin/%s\n",$4,en) \
}\
}' > $jid.host
fi
# define the variable host and host_filt
# host_filt is used for loops over hosts
# for Myrinet we need to use a filtered variant of userhost
# for others we can use $host
if test $MPITYPE = "myrinet"
then
if test $MPIVERSION = "MPICH-GM1.2.1..7"
then
host=~/.gmpi/$jid.host
host_filt=$jid.host_tMp
grep -v '^#' $userhost | $AWK '{if (NF > 0) print $1}' > $host_filt
else
host=$jid.host
host_filt=$host
fi
else
host=$jid.host
host_filt=$host
if test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
then
host_filt=$jid.mfile
fi
fi
# figure out if the machines in the hostfile are nfs mounted
# or distributed and set the variable "dirstatus" accordingly.
# only perform the check if user subroutine is used
# or a user subroutine executable is used
numfield=1
if test $MPITYPE = hpmpi -o $MACHINENAME = HP -a $MPITYPE = hardware
then
numfield=2
fi
DIR1=$DIRJOB
if test $program = $prog.marc -o -n "$user" -o -n "$objs"
then
counter=0
echo " "
echo "checking if local or shared directories for host"
if test "$deletelog" = no
then
echo "checking if local or shared directories for host" >> $jid.log
fi
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
counter=$((counter+1))
dirstatus[$counter]="shared"
$ECHO " $i $ECHOTXT"
if test "$deletelog" = no
then
$ECHO " $i $ECHOTXT" >> $jid.log
fi
DIR1=$DIRJOB
line=`grep -v '^#' $userhost | grep "^$ibase "`
workdir=`echo $line | $AWK '{print $3}'`
if test -n "$workdir"
then
DIR1=$workdir
fi
if test -f $jid.$$
then
/bin/rm $jid.$$
fi
$RSH $i /bin/touch $DIR1/$jid.$$ 2> tmp.$$
if test -s tmp.$$
then
dirstatus[$counter]="local"
/bin/rm tmp.$$
else
if test ! -f $jid.$$
then
dirstatus[$counter]="local"
$RSH $i /bin/rm $DIR1/$jid.$$
else
/bin/rm $jid.$$
fi
fi
if test -f tmp.$$
then
/bin/rm tmp.$$
fi
if test -f $jid.$$
then
/bin/rm $jid.$$
fi
echo " ${dirstatus[$counter]}"
if test "$deletelog" = no
then
echo " ${dirstatus[$counter]}" >> $jid.log
fi
fi
done
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
fi
fi
# figure out if this is a compatible set of machines
# unless explicitly specified with flag -comp
# only perform the check if user subroutine is used
# or a user subroutine executable is used
# Myrinet does not support heterogeneous
if test $program = $prog.marc -o -n "$user" -o -n "$objs"
then
if test $compatible = "unknown"
then
thisname=$ARCH
compatible=yes
counter=0
echo "checking if machines are compatible for host"
if test "$deletelog" = no
then
echo "checking if machines are compatible for host" >> $jid.log
fi
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
counter=$((counter+1))
compstatus[$counter]="yes"
$ECHO " $i $ECHOTXT"
if test "$deletelog" = no
then
$ECHO " $i $ECHOTXT" >> $jid.log
fi
othername=`$RSH $i uname -a | cut -f 1 -d " "`
if test $thisname != $othername
then
compatible=no
compstatus[$counter]="no"
fi
fi
echo " ${compstatus[$counter]}"
if test "$deletelog" = no
then
echo " ${compstatus[$counter]}" >> $jid.log
fi
done
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
fi
else
counter=0
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
counter=$((counter+1))
compstatus[$counter]=$compatible
fi
done
if test $compatible = "no"
then
echo "all machines assumed incompatible"
if test "$deletelog" = no
then
echo "all machines assumed incompatible" >> $jid.log
fi
else
echo "all machines compatible"
if test "$deletelog" = no
then
echo "all machines compatible" >> $jid.log
fi
fi
fi
# error out if user objects or libraries are used on incompatible machines
if test "$compatible" = "no" -a -n "$objs"
then
echo "User object/libraries cannot be used in a parallel job on incompatible machines"
if test "$deletelog" = no
then
echo "User object/libraries cannot be used in a parallel job on incompatible machines" >> $jid.log
fi
exit 1
fi
# modify new host file if NFS mounted heterogeneous machine
doit=
if test $program = $prog.marc
then
doit=yes
fi
if test "$user"
then
doit=yes
fi
if test "$doit"
then
counter=0
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
counter=$((counter+1))
if test ${dirstatus[$counter]} = "shared" -a ${compstatus[$counter]} = "no"
then
$AWK -v hst=$i '{fnr++ ; \
if ($1 ~ hst) {if ( fnr == 1 ) printf("%s\n",$0); else \
printf("%s %s %s_%s\n",$1,$2,$3,$1) } else print}' $jid.host > $jid.host{$$}
/bin/mv $jid.host{$$} $jid.host
host=$jid.host
fi
fi
done
fi
fi # if test $program = $prog.marc -o $user -o $obj
else # if test $host
# assume shared memory machine if no hostfile given and
# MPITYPE is set to mpich or Myrinet
# check for Myrinet that the total number of processes is
# less than number of available user ports, 5
if test $MPITYPE = "mpich" -o $MPITYPE = "scali"
then
numproc=`echo $nprocd | $AWK '{sum=$1-1}; {print sum}'`
echo `hostname` $numproc $execpath > $jid.host
host=$jid.host
elif test $MPITYPE = "myrinet"
then
if test $nprocd -gt 5
then
echo " "
echo "error, for Myrinet the number of processes "
echo "must not exceed 5 for a hostname"
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
echo "error, for Myrinet the number of processes " >> $jid.log
echo "must not exceed 5 for a hostname" >> $jid.log
echo " " >> $jid.log
fi
exit 1
fi
if test $MPIVERSION = "MPICH-GM1.2.1..7"
then
echo $nprocd > ~/.gmpi/$jid.host
echo `hostname` $nprocd | $AWK \
'BEGIN {iport[0] = 2; \
iport[1] = 4; \
iport[2] = 5; \
iport[3] = 6; \
iport[4] = 7 \
} \
{for(iproc = 0; iproc < $2; iproc++) printf("%s %d\n",$1,iport[iproc])} \
' >> ~/.gmpi/$jid.host
host=~/.gmpi/$jid.host
else
numproc=`echo $nprocd | $AWK '{sum=$1-1}; {print sum}'`
echo `hostname` $numproc $execpath > $jid.host
fi
fi # if test myrinet
fi # if test $host
fi # if test $nprocd -gt 1
fi # if test $program = $exefile -o $program = $prog.marc
##############################################################################
# construct run stream (Marc only) #
##############################################################################
# set maximum message length for ddm to a large number
# for vendor provided mpi
if test $itree -eq 0 -a $MPITYPE = hardware
then
itree=100000000
if test $MACHINENAME = SGI
then
itree=100000001
fi
fi
if test $itree -eq 0 -a $MPITYPE = hpmpi
then
itree=100000000
fi
if test $itree -eq 0 -a $MPITYPE = myrinet
then
itree=100000000
fi
if test $itree -eq 0 -a $MPITYPE = nec
then
itree=100000000
fi
if test $itree -eq 0 -a $MPITYPE = scali
then
itree=100000000
fi
if test $itree -eq 0 -a $MPITYPE = intelmpi
then
itree=100000000
fi
if test $nprocdddm -lt 2
then
nprocdarg=
else
nprocdarg="$nprocdarg $nprocdddm"
fi
if test $nsolver -eq 0
then
nsolverarg=
else
nsolverarg="$nsolverarg $nsolver"
fi
if test $nprocdddm -lt 2 -a $nsolver -eq 0
then
nprocd=0
fi
if test $nprocd -gt 0
then
if test "$host"
then
if test -z "$RUN_JOB2"
then
echo " "
echo "error: parallel job attempted on non-parallel version,"
echo " or, if parallel version is installed, the include "
echo " file is probably corrupted"
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
echo "error: parallel job attempted on non-parallel version," >> $jid.log
echo " or, if parallel version is installed, the include " >> $jid.log
echo " file is probably corrupted" >> $jid.log
echo " " >> $jid.log
fi
exit
fi
if test $MPITYPE = hpmpi -o $MACHINENAME = HP -a $MPITYPE = hardware
then
RUN_JOB="$RUN_JOB2 $host -- -jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
elif test $MACHINENAME = IBM -a $MPITYPE = hardware -a "$MACHINETYPE" = NONSP
then
RUN_JOB="$RUN_JOB2 $bd$program -jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
elif test $MPITYPE = "myrinet"
then
if test $MPIVERSION = "MPICH-GM1.2.1..7"
then
RUN_JOB="$RUN_JOB2 $host $bd$program -jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
else
RUN_JOB_TMP="$RUN_JOB2 $host $bd$program"
RUN_JOB=" -jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
fi
elif test $MACHINENAME = DEC -a $MPITYPE = hardware
then
RUN_JOB="$RUN_JOB2 $nprocd -hf $host $bd$program -jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
elif test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
then
numhost=`uniq $jid.mfile | wc -l`
if test "$INTELMPI_VERSION" = "HYDRA"
then
RUN_JOB_TMP="$RUN_JOB2 -configfile $jid.cfile"
else
export I_MPI_JOB_CONTEXT=$$
mpdboot -n $numhost -r $RSH -f $jid.mfile
RUN_JOB_TMP="$RUN_JOB2 $jid.cfile"
fi
# intelmpi uses configfile. format:
# -host host1 -n n1 executable marcargs
# one such line per host
# collect the marcargs in RUN_JOB and construct the config file later
# collect the run stream in RUN_JOB_TMP
RUN_JOB="-jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
elif test $MACHINENAME = "SUN" -a $MPITYPE = "hardware"
then
RUN_JOB="$RUN_JOB2 $jid.mfile -n $nprocd $bd$program -jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
else
RUN_JOB="$RUN_JOB2 $host $bd$program -jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
fi
if test "$userhost"
then
RUN_JOB="$RUN_JOB -mhost $userhost"
fi
if test $MPITYPE = "scali"
then
# set default working directory to /tmp to allow
# different directory names
SCAMPI_WORKING_DIRECTORY=/tmp
export SCAMPI_WORKING_DIRECTORY
fi
else
if test -z "$RUN_JOB1"
then
echo " "
echo "error: parallel job attempted on non-parallel version,"
echo " or, if parallel version is installed, the include "
echo " file is probably corrupted"
echo " "
if test "$deletelog" = no
then
echo " " >> $jid.log
echo "error: parallel job attempted on non-parallel version," >> $jid.log
echo " or, if parallel version is installed, the include " >> $jid.log
echo " file is probably corrupted" >> $jid.log
echo " " >> $jid.log
fi
exit
fi
RUNNPROCD=$nprocd
if test $MACHINENAME = "IBM" -a $MPITYPE = "hardware"
then
RUNNPROCD=
MP_PROCS=$nprocd
export MP_PROCS
fi
if test $MPITYPE = "myrinet"
then
RUN_JOB="$RUN_JOB1 $RUNNPROCD $bd$program -jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
else
RUN_JOB="$RUN_JOB1 $RUNNPROCD $bd$program -jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
fi
if test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
then
if test "$INTELMPI_VERSION" = "HYDRA"
then
echo " " > /dev/null
else
export I_MPI_JOB_CONTEXT=$$
mpdboot -n 1 -f $jid.hosts
fi
RUN_JOB="$RUN_JOB1 $RUNNPROCD $bd$program -jid $jid -dirjid $DIRJID \
$nprocdarg \
$nsolverarg \
-maxnum $MAXNUM -itree $itree \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
fi
fi
else
if test $ndcoup -gt 0
then
RUN_JOB="$RUN_JOB0 $BINDIR/exe_auto $bd$program -jid $jid -dirjid $DIRJID \
-maxnum $MAXNUM \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
else
# this is for a serial job without auto restart:
RUN_JOB="$RUN_JOB0 $bd$program -jid $jid -dirjid $DIRJID \
-maxnum $MAXNUM \
$ntearg $nte $ntsarg $nts $gpuoption -dirjob $DIRJOB "
fi
fi
if test "$rid"
then
RUN_JOB="$RUN_JOB -rid $rid -dirrid $DIRRID"
fi
if test "$pid"
then
RUN_JOB="$RUN_JOB -pid $pid -dirpid $DIRPID"
fi
if test "$sid"
then
RUN_JOB="$RUN_JOB -sid $sid -dirsid $DIRSID"
fi
if test "$did"
then
RUN_JOB="$RUN_JOB -def $did -dirdid $DIRDID"
fi
if test "$vid"
then
RUN_JOB="$RUN_JOB -vf $vid -dirvid $DIRVID"
fi
if test $ndcoup -gt 0
then
RUN_JOB="$RUN_JOB -dcoup $ndcoup "
fi
if test $ndytran -gt 0
then
RUN_JOB="$RUN_JOB -dytran $ndytran "
fi
if test $mesh -gt 0
then
RUN_JOB="$RUN_JOB -me $mesh "
fi
if test $noutcore -gt 0
then
RUN_JOB="$RUN_JOB -outcore $noutcore "
fi
if test "$dllrun" -gt 0
then
RUN_JOB="$RUN_JOB -dll $dllrun "
fi
if test "$trkrun" -gt 0
then
RUN_JOB="$RUN_JOB -trk $trkrun "
fi
if test "$iam"
then
RUN_JOB="$RUN_JOB -iam $iam "
fi
if test "$justlist"
then
RUN_JOB="$RUN_JOB -list 1 "
fi
if test "$feature"
then
RUN_JOB="$RUN_JOB -feature $feature "
fi
if test "$memlimit" -ne 0
then
RUN_JOB="$RUN_JOB -ml $memlimit "
fi
if test "$cpinput"
then
RUN_JOB="$RUN_JOB -ci $cpinput "
fi
if test "$cpresults"
then
RUN_JOB="$RUN_JOB -cr $cpresults "
fi
if test "$DIRSCR" != "$DIRJOB"
then
RUN_JOB="$RUN_JOB -dirscr $DIRSCR"
else
DIRSCR=$DIRJOB
fi
if test "$makebdf"
then
RUN_JOB="$RUN_JOB -bdf $makebdf "
fi
if test $MPITYPE = "myrinet" -a "$host" -a "$MPIVERSION" != "MPICH-GM1.2.1..7"
then
# append $RUN_JOB to all lines of the host file
# and set RUN_JOB
$AWK -v args="$RUN_JOB" '{print $0,args}' $host > $host.$$
/bin/mv $host.$$ $host
RUN_JOB=$RUN_JOB_TMP
fi
if test $MPITYPE = "intelmpi" -a "$host"
then
# construct config file, append $RUN_JOB to all lines of the config file
# and set RUN_JOB
if test "$INTELMPI_VERSION" = "HYDRA"
then
grep -v '^#' $host | $AWK -v args="$RUN_JOB" -v path=$execpath -v en=$execname -v us=$usersub \
'{if ( NF > 0) {\
printf(" -host %s",$1); \
printf(" -n %s",$2); \
if ( NF == 2 ) printf(" %s",path);\
if ( NF >= 3 ) printf(" -wdir %s ",$3); \
if ( NF >= 3 ) if (us) printf(" %s/%s",$3,en); else printf(" %s",path); \
printf(" %s\n",args); \
}\
}' > $jid.cfile
else
grep -v '^#' $host | $AWK -v args="$RUN_JOB" -v path=$execpath -v en=$execname -v us=$usersub \
'{if ( NF > 0) {\
printf("-host %s -n %s",$1,$2); \
if ( NF == 2 ) printf(" %s",path);\
if ( NF >= 3 ) printf(" -wdir %s ",$3); \
if ( NF >= 3 ) if (us) printf(" %s/%s",$3,en); else printf(" %s",path); \
printf(" %s\n",args); \
}\
}' > $jid.cfile
fi
RUN_JOB=$RUN_JOB_TMP
fi
echo " "
echo "Final run stream value"
echo " RUNJOB="$RUN_JOB
if test "$deletelog" = no
then
echo " " >> $jid.log
echo "Final run stream value" >> $jid.log
echo " RUNJOB="$RUN_JOB >> $jid.log
fi
#
# check for external file to run using valgrind
#
if test -f $MARC_TOOLS/run_marc_valgrind
then
. $MARC_TOOLS/run_marc_valgrind
fi
##############################################################################
# run the requested program in a queue #
##############################################################################
if test "$deletelog" = yes
then
echo
date
else
echo >> $jid.log
date >> $jid.log
fi
if [ $qid = short -o $qid = long -o $qid = verylong -o $qid = at ]
then
/bin/rm -f $jid.runmarcscript
#
# compile user subroutine if present
#
if test "$link"
then
if test -z "$FCOMPROOT"; then
echo "$0: No compiler available"
echo
echo " $PRODUCT Exit number 3"
exit 1
fi
echo
echo "Using compiler from: $FCOMPROOT"
echo
if test "$user"
then
userobj=$usermoext.o
fi
cat > $jid.runmarcscript << END4
if test "$user"
then
if test $MACHINENAME = "CRAY"
then
$DFORTRANMP $user || \
{
echo "$0: compile failed for $user"
exit 1
}
/bin/rm $program 2>/dev/null
else
$DFORTRANMP $user -o $userobj || \
{
echo "$0: compile failed for $user"
exit 1
}
/bin/rm $program 2>/dev/null
fi
fi
$LOAD $bd${program} $MARC_LIB/main.o \
$MARC_LIB/blkdta.o $MARC_LIB/comm?.o \
${userobj-} \
$objs \
$SRCLIB \
$MNFLIBS \
$MDUSER \
${MUMPSSOLVERLIBS} \
$MDSRCLIB \
$MARC_LIB/mcvfit.a \
$STUBS \
$SOLVERLIBS \
$MARCCUDALIBS \
$TKLIBS \
$MRCLIBS \
$METISLIBS \
$DAMASK \
$SFLIB \
$OPENSSL_LIB \
$SYSLIBS \
$SECLIBS || \
{
echo "$0: link failed for ${user:+$userobj }$objs"
exit 1
}
END4
else
prgsav=yes
fi
/bin/rm $userobj 2>/dev/null
/bin/rm $DIRJOB/*.mod 2>/dev/null
/bin/rm $DIRJOB/*.smod 2>/dev/null
#
# run marc
#
cat >> $jid.runmarcscript << END5
# Define share library path based on platforms
# This is required for using the Patran Mesher
if test $MACHINENAME = "IBM"
then
LIBPATH=$MARC_LIB:$MARC_LIB_SHARED:$LIBPATH
export LIBPATH
fi
# first remove all .out files and incremental restart files
# the ones for ddm are removed in the code
if test $nprocdddm -gt 1
then
numdom=$nprocdddm
while test \$numdom -gt 0
do
/bin/rm $DIRJOB/$numdom$jid.out 2>/dev/null
/bin/rm $DIRJOB/$numdom$jid.log 2>/dev/null
/bin/rm $DIRJOB/$numdom${jid}_i_*.t08 2>/dev/null
numdom=\`echo \$numdom | $AWK '{sum=\$1-1}; {print sum}'\`
done
else
/bin/rm $DIRJOB/$jid.out 2>/dev/null
/bin/rm $DIRJOB/${jid}_i_*.t08 2>/dev/null
fi
if test $nprocdddm -gt 1
then
$RUN_JOB 2>>$jid.log
else
$RUN_JOB 2>>$jid.log
fi
if test $dllrun -eq 0; then
if test $prgsav = no
then
/bin/rm -f $bd$program 2>/dev/null
fi
else
if test $cpdll = yes; then
filename=$usernoext
/bin/cp $DIRJOB/$marcdll $DIRJOB/${filename}_$marcdll 2>/dev/null
fi
if test $rmdll = yes
then
/bin/rm -f $DIRJOB/$marcdll 2>/dev/null
fi
fi
if test $nprocdddm -gt 1
then
numdom=$nprocdddm
while test \$numdom -gt 0
do
/bin/rm $DIRSCR/$numdom$jid.t02 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t03 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t11 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t12 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t13 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t14 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t15 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t22 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t23 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t32 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t33 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t74 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t75 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t76 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t77 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t78 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t79 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.*.t81* 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.*.t82* 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.*.t83* 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t84 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t85 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t86 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t87 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t88 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t90 2>/dev/null
numdom=\`echo \$numdom | $AWK '{sum=\$1-1}; {print sum}'\`
done
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
else
/bin/rm $DIRSCR/$jid.t02 2>/dev/null
/bin/rm $DIRSCR/$jid.t03 2>/dev/null
/bin/rm $DIRSCR/$jid.t11 2>/dev/null
/bin/rm $DIRSCR/$jid.t12 2>/dev/null
/bin/rm $DIRSCR/$jid.t13 2>/dev/null
/bin/rm $DIRSCR/$jid.t14 2>/dev/null
/bin/rm $DIRSCR/$jid.t15 2>/dev/null
/bin/rm $DIRSCR/$jid.t22 2>/dev/null
/bin/rm $DIRSCR/$jid.t23 2>/dev/null
/bin/rm $DIRSCR/$jid.t32 2>/dev/null
/bin/rm $DIRSCR/$jid.t33 2>/dev/null
/bin/rm $DIRSCR/$jid.*.t81* 2>/dev/null
/bin/rm $DIRSCR/$jid.*.t82* 2>/dev/null
/bin/rm $DIRSCR/$jid.*.t83* 2>/dev/null
/bin/rm $DIRSCR/$jid.t84 2>/dev/null
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
fi
END5
# Submit to marc batch queue
#
if [ $qid = at ]
then
QUENAME=at
SUBMCMD=
else
#
# Submit to qsub queue
#
QUENAME=qsub
SUBMCMD="-q $qid -o /dev/null -e $jid.batch_err_log -x -r $jid"
if test "$priority"
then
SUBMCMD=$SUBMCMD" -p $priority"
fi
if test "$att"
then
SUBMCMD=$SUBMCMD" -a $att"
fi
if test "$cpu"
then
SUBMCMD=$SUBMCMD" -lt $cpu"
fi
fi
echo $QUENAME $SUBMCMD
#cat $jid.runmarcscript
$QUENAME $SUBMCMD < $jid.runmarcscript
/bin/rm -f $jid.runmarcscript
##############################################################################
# run the requested program in the background #
##############################################################################
else
if test $qid = background
then
#
# first remove all old .out files
# the ones for ddm are removed in the code
if test $nprocdddm -gt 1
then
numdom=$nprocdddm
while test $numdom -gt 0
do
/bin/rm $DIRJOB/$numdom$jid.out 2>/dev/null
/bin/rm $DIRJOB/$numdom$jid.log 2>/dev/null
numdom=`echo $numdom | $AWK '{sum=$1-1}; {print sum}'`
done
else
/bin/rm $DIRJOB/$jid.out 2>/dev/null
fi
#
# compile user subroutine if present
#
(
if test "$link"
then
if test -z "$FCOMPROOT"; then
echo "$0: No compiler available"
echo
echo " $PRODUCT Exit number 3"
exit 1
fi
echo
echo "Using compiler from: $FCOMPROOT"
echo
if test "$user"
then
# compile and link on other hosts in $host if compstatus=no
if test $nprocd -gt 1
then
if test "$userhost"
then
counter=0
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
counter=$((counter+1))
if test ${compstatus[$counter]} = "no"
then
DIR1=$DIRJOB
DIR2=$DIR
line=`grep -v '^#' $userhost | grep "^$ibase "`
workdir=`echo $line | $AWK '{print $3}'`
marcdir=`echo $line | $AWK '{print $4}'`
if test -n "$workdir"
then
DIR1=$workdir
fi
if test -n "$marcdir"
then
DIR2=$marcdir
fi
# first copy over the user sub if local directories
if test ${dirstatus[$counter]} = "local"
then
$RCP $user $i:$DIR1/
fi
# do the compilation on the other machine
if test ${dirstatus[$counter]} = "shared"
then
hname=_$ibase
else
hname=
fi
remoteprog=$DIR1/${execname}$hname
remoteuser=$DIR1/`$BASENAME $user`
$RSH $i /bin/rm $remoteprog 2> /dev/null
echo
$RSH $i $DIR2/tools/comp_damask_mp $DIR2 $DIR1 $remoteuser $remoteprog
# check if successful, the new executable should be there
line=`$RSH $i /bin/ls $remoteprog 2> /dev/null`
if test "$line"
then
echo compilation and linking successful on host $i
else
echo "$0: compile failed for $user on host $i"
echo " $PRODUCT Exit number 3"
exit 1
fi
# remove the user subroutine on remote machine
if test ${dirstatus[$counter]} = "local"
then
$RSH $i /bin/rm $remoteuser 2> /dev/null
fi
fi
fi
done
fi
fi
if test "$userhost"
then
echo
echo "Compiling and linking user subroutine $user on host `hostname`"
fi
userobj=$usernoext.o
if test $MACHINENAME = "CRAY"
then
$DFORTRANMP $user || \
{
echo "$0: compile failed for $user"
echo " $PRODUCT Exit number 3"
exit 1
}
/bin/rm $program 2>/dev/null
else
$DFORTRANMP $user -o $userobj || \
{
echo "$0: compile failed for $user"
echo " $PRODUCT Exit number 3"
exit 1
}
/bin/rm $program 2>/dev/null
fi
fi # if test $user
$LOAD $bd${program} $MARC_LIB/main.o \
$MARC_LIB/blkdta.o $MARC_LIB/comm?.o \
${userobj-} \
$objs \
$SRCLIB \
$MNFLIBS \
$MDUSER \
${MUMPSSOLVERLIBS} \
$MDSRCLIB \
$MARC_LIB/mcvfit.a \
$STUBS \
${SOLVERLIBS} \
${MARCCUDALIBS} \
$TKLIBS \
$MRCLIBS \
$METISLIBS \
$DAMASK \
$SFLIB \
$OPENSSL_LIB \
$SYSLIBS \
$SECLIBS || \
{
echo "$0: link failed for ${user:+$userobj }$objs"
echo " $PRODUCT Exit number 3"
exit 1
}
# copy user subroutine executable for hosts using local working dir
if test $nprocd -gt 1
then
if test "$userhost"
then
counter=0
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
counter=$((counter+1))
if test ${dirstatus[$counter]} = "local" -a ${compstatus[$counter]} = "yes"
then
DIR1=$DIRJOB
line=`grep -v '^#' $userhost | grep "^$ibase "`
workdir=`echo $line | $AWK '{print $3}'`
if test -n "$workdir"
then
DIR1=$workdir
fi
echo "Copying executable to host ${i}"
$RCP $program ${i}:${DIR1}/
fi
fi
done
fi
fi
else # if test $link
prgsav=yes
fi # if test $link
/bin/rm $userobj 2>/dev/null
/bin/rm $DIRJOB/*.mod 2>/dev/null
/bin/rm $DIRJOB/*.smod 2>/dev/null
#
# run marc
#
# Define share library path based on platforms
# This is required for using the Patran Mesher
if test $MACHINENAME = "IBM"
then
LIBPATH=$MARC_LIB:$MARC_LIB_SHARED:$LIBPATH
export LIBPATH
fi
# for DDM with ARC support
if test $ddm_arc -gt 0; then
RUN_JOB="$BINDIR/exeddm $RUN_JOB -ddm $ddm_arc "
fi
$RUN_JOB &
marcpid=$!
echo $marcpid > $DIRJOB/$jid.pid
wait $marcpid
if test $nprocd -gt 1
then
if test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
then
if test "$INTELMPI_VERSION" = "HYDRA"
then
if test "$host"
then
/bin/rm $jid.mfile 2> /dev/null
/bin/rm $jid.hosts 2> /dev/null
/bin/rm $jid.host 2> /dev/null
/bin/rm $jid.cfile 2> /dev/null
fi
fi
fi
fi
if test $dllrun -eq 0; then
if test $prgsav = no
then
/bin/rm -f $bd$program 2>/dev/null
# for network run, remove executable on remote machines
# and executables with modified name
if test $nprocd -gt 1
then
if test "$userhost"
then
counter=0
if test -f "$host_filt"
then
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
counter=$((counter+1))
DIR1=$DIRJOB
line=`grep -v '^#' $userhost | grep "^$ibase "`
workdir=`echo $line | $AWK '{print $3}'`
if test -n "$workdir"
then
DIR1=$workdir
fi
# if an incompatible host uses shared directory,
# then the root machine deletes the executable
if test ${dirstatus[$counter]} = "shared" -a ${compstatus[$counter]} = "no"
then
hname=_$ibase
/bin/rm ${execname}$hname
fi
# if local directory used, the remote machine
# deletes the executable
if test ${dirstatus[$counter]} = "local"
then
$RSH $i /bin/rm $DIR1/${execname} 2>/dev/null
fi
fi
done
fi
fi
fi
fi
else
#dllrun >0
if test $cpdll = yes; then
filename=$usernoext
/bin/cp $DIRJOB/$marcdll $DIRJOB/${filename}_$marcdll 2>/dev/null
fi
if test $rmdll = yes;then
/bin/rm -f $DIRJOB/$marcdll 2>/dev/null
fi
fi
if test $nprocdddm -gt 1
then
numdom=$nprocdddm
while test $numdom -gt 0
do
/bin/rm $DIRSCR/$numdom$jid.t02 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t03 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t11 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t12 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t13 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t14 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t15 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t22 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t23 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t32 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t33 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t74 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t75 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t76 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t77 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t78 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t79 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.*.t81* 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.*.t82* 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.*.t83* 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t84 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t85 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t86 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t87 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t88 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t90 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.sle 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.sin 2>/dev/null
numdom=`echo $numdom | $AWK '{sum=$1-1}; {print sum}'`
done
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
if test $MPITYPE = "myrinet"
then
if test -f "$host_filt"
then
/bin/rm $host_filt
fi
fi
else
/bin/rm $DIRSCR/$jid.t02 2>/dev/null
/bin/rm $DIRSCR/$jid.t03 2>/dev/null
/bin/rm $DIRSCR/$jid.t11 2>/dev/null
/bin/rm $DIRSCR/$jid.t12 2>/dev/null
/bin/rm $DIRSCR/$jid.t13 2>/dev/null
/bin/rm $DIRSCR/$jid.t14 2>/dev/null
/bin/rm $DIRSCR/$jid.t15 2>/dev/null
/bin/rm $DIRSCR/$jid.t22 2>/dev/null
/bin/rm $DIRSCR/$jid.t23 2>/dev/null
/bin/rm $DIRSCR/$jid.t32 2>/dev/null
/bin/rm $DIRSCR/$jid.t33 2>/dev/null
/bin/rm $DIRSCR/$jid.*.t81* 2>/dev/null
/bin/rm $DIRSCR/$jid.*.t82* 2>/dev/null
/bin/rm $DIRSCR/$jid.*.t83* 2>/dev/null
/bin/rm $DIRSCR/$jid.t84 2>/dev/null
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
/bin/rm $DIRJOB/$jid.sle 2>/dev/null
/bin/rm $DIRJOB/$jid.sin 2>/dev/null
fi
) 1>>$jid.log 2>&1 &
##############################################################################
# run the requested program in the foreground #
##############################################################################
else
#
# compile user subroutine if present
#
if test "$link"
then
if test -z "$FCOMPROOT"; then
echo "$0: No compiler available"
echo
echo " $PRODUCT Exit number 3"
exit 1
fi
echo
echo "Using compiler from: $FCOMPROOT"
echo
if test "$user"
then
# compile and link on other hosts in $host if compstatus=no
if test $nprocd -gt 1
then
if test "$userhost"
then
counter=0
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
counter=$((counter+1))
if test ${compstatus[$counter]} = "no"
then
DIR1=$DIRJOB
DIR2=$DIR
line=`grep -v '^#' $userhost | grep "^$ibase "`
workdir=`echo $line | $AWK '{print $3}'`
marcdir=`echo $line | $AWK '{print $4}'`
if test -n "$workdir"
then
DIR1=$workdir
fi
if test -n "$marcdir"
then
DIR2=$marcdir
fi
# first copy over the user sub if local directories
if test ${dirstatus[$counter]} = "local"
then
$RCP $user $i:$DIR1/
fi
# do the compilation on the other machine
if test ${dirstatus[$counter]} = "shared"
then
hname=_$ibase
else
hname=
fi
remoteprog=$DIR1/${execname}$hname
remoteuser=$DIR1/`$BASENAME $user`
$RSH $i /bin/rm $remoteprog 2> /dev/null
echo
$RSH $i $DIR2/tools/comp_damask_mp $DIR2 $DIR1 $remoteuser $remoteprog
# check if successful, the new executable should be there
line=`$RSH $i /bin/ls $remoteprog 2> /dev/null`
if test "$line"
then
echo compilation and linking successful on host $i
else
echo "$0: compile failed for $user on host $i"
exit 1
fi
# remove the user subroutine on remote machine
if test ${dirstatus[$counter]} = "local"
then
$RSH $i /bin/rm $remoteuser 2> /dev/null
fi
fi
fi
done
fi
fi
if test "$userhost"
then
echo
echo "Compiling and linking user subroutine $user on host `hostname`"
fi
userobj=$usernoext.o
if test $MACHINENAME = "CRAY"
then
$DFORTRANMP $user || \
{
echo "$0: compile failed for $user"
exit 1
}
/bin/rm $program 2>/dev/null
else
$DFORTRANMP $user -o $userobj || \
{
echo "$0: compile failed for $user"
exit 1
}
/bin/rm $program 2>/dev/null
fi
fi # if test $user
$LOAD $bd${program} $MARC_LIB/main.o \
$MARC_LIB/blkdta.o $MARC_LIB/comm?.o \
${userobj-} \
$objs \
$SRCLIB \
$MNFLIBS \
$MDUSER \
${MUMPSSOLVERLIBS} \
$MDSRCLIB \
$MARC_LIB/mcvfit.a \
$STUBS \
${SOLVERLIBS} \
${MARCCUDALIBS} \
$TKLIBS \
$MRCLIBS \
$METISLIBS \
$DAMASK \
$SFLIB \
$OPENSSL_LIB \
$SYSLIBS \
$SECLIBS || \
{
echo "$0: link failed for ${user:+$userobj }$objs"
exit 1
}
# copy user subroutine executable for hosts using local working dir
if test $nprocd -gt 1
then
if test "$userhost"
then
counter=0
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
counter=$((counter+1))
if test ${dirstatus[$counter]} = "local" -a ${compstatus[$counter]} = "yes"
then
DIR1=$DIRJOB
line=`grep -v '^#' $userhost | grep "^$ibase "`
workdir=`echo $line | $AWK '{print $3}'`
if test -n "$workdir"
then
DIR1=$workdir
fi
echo "Copying executable to host ${i}"
$RCP $program ${i}:${DIR1}/
fi
fi
done
fi
fi
else # if test $link
prgsav=yes
fi # if test $link
/bin/rm $userobj 2>/dev/null
/bin/rm $DIRJOB/*.mod 2>/dev/null
/bin/rm $DIRJOB/*.smod 2>/dev/null
# done if no job id given
if test -z "$jid"
then
echo
echo only compilation requested
echo
exit
fi
#
# run marc
#
# Define share library path based on platforms
# This is required for using the Patran Mesher
if test $MACHINENAME = "IBM"
then
LIBPATH=$MARC_LIB:$MARC_LIB_SHARED:$LIBPATH
export LIBPATH
fi
# first remove all .out files
# the ones for ddm are removed in the code
if test $nprocdddm -gt 1
then
numdom=$nprocdddm
while test $numdom -gt 0
do
/bin/rm $DIRJOB/$numdom$jid.out 2>/dev/null
/bin/rm $DIRJOB/$numdom$jid.log 2>/dev/null
numdom=`echo $numdom | $AWK '{sum=$1-1}; {print sum}'`
done
else
/bin/rm $DIRJOB/$jid.out 2>/dev/null
fi
# for DDM with ARC support
if test $ddm_arc -gt 0; then
RUN_JOB="$BINDIR/exeddm $RUN_JOB -ddm $ddm_arc "
fi
$RUN_JOB
if test $nprocd -gt 1
then
if test $MACHINENAME = "LINUX" -a $MPITYPE = "intelmpi"
then
if test "$INTELMPI_VERSION" = "HYDRA"
then
if test "$host"
then
/bin/rm $jid.mfile 2> /dev/null
/bin/rm $jid.hosts 2> /dev/null
/bin/rm $jid.host 2> /dev/null
/bin/rm $jid.cfile 2> /dev/null
else
echo " " > /dev/null
fi
else
if test "$host"
then
mpdcleanup -a -f $jid.mfile
/bin/rm $jid.host 2> /dev/null
/bin/rm $jid.mfile 2> /dev/null
else
mpdcleanup -a -f $jid.hosts
/bin/rm $jid.hosts 2> /dev/null
fi
fi
fi
fi
if test $dllrun -eq 0; then
if test $prgsav = no
then
/bin/rm -f $bd$program 2>/dev/null
# for network run, remove executable on remote machines
# and executables with modified name
if test $nprocd -gt 1
then
if test "$userhost"
then
counter=0
if test -f "$host_filt"
then
for i in `$AWK -v n=$numfield '{print $n}' $host_filt`
do
ibase=${i%%.*}
if test $ibase != $thishost
then
counter=$((counter+1))
DIR1=$DIRJOB
line=`grep -v '^#' $userhost | grep "^$ibase "`
workdir=`echo $line | $AWK '{print $3}'`
if test -n "$workdir"
then
DIR1=$workdir
fi
# if an incompatible host uses shared directory,
# then the root machine deletes the executable
if test ${dirstatus[$counter]} = "shared" -a ${compstatus[$counter]} = "no"
then
hname=_$ibase
/bin/rm ${execname}$hname
fi
# if local directory used, the remote machine
# deletes the executable
if test ${dirstatus[$counter]} = "local"
then
$RSH $i /bin/rm $DIR1/${execname} 2>/dev/null
fi
fi
done
fi
fi
fi
fi
else
#dllrun >0
if test $cpdll = yes; then
filename=$usernoext
/bin/cp $DIRJOB/$marcdll $DIRJOB/${filename}_$marcdll 2>/dev/null
fi
if test $rmdll = yes;then
/bin/rm -f $DIRJOB/$marcdll 2>/dev/null
fi
fi
if test $nprocdddm -gt 1
then
numdom=$nprocdddm
while test $numdom -gt 0
do
/bin/rm $DIRSCR/$numdom$jid.t02 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t03 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t11 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t12 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t13 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t14 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t15 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t22 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t23 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t32 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t33 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t74 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t75 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t76 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t77 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t78 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t79 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.*.t81* 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.*.t82* 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.*.t83* 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t84 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t85 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t86 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t87 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t88 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.t90 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.sle 2>/dev/null
/bin/rm $DIRSCR/$numdom$jid.sin 2>/dev/null
numdom=`echo $numdom | $AWK '{sum=$1-1}; {print sum}'`
done
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
if test $MPITYPE = "myrinet"
then
if test -f "$host_filt"
then
/bin/rm $host_filt
fi
fi
else
/bin/rm $DIRSCR/$jid.t02 2>/dev/null
/bin/rm $DIRSCR/$jid.t03 2>/dev/null
/bin/rm $DIRSCR/$jid.t11 2>/dev/null
/bin/rm $DIRSCR/$jid.t12 2>/dev/null
/bin/rm $DIRSCR/$jid.t13 2>/dev/null
/bin/rm $DIRSCR/$jid.t14 2>/dev/null
/bin/rm $DIRSCR/$jid.t15 2>/dev/null
/bin/rm $DIRSCR/$jid.t22 2>/dev/null
/bin/rm $DIRSCR/$jid.t23 2>/dev/null
/bin/rm $DIRSCR/$jid.t32 2>/dev/null
/bin/rm $DIRSCR/$jid.t33 2>/dev/null
/bin/rm $DIRSCR/$jid.*.t81* 2>/dev/null
/bin/rm $DIRSCR/$jid.*.t82* 2>/dev/null
/bin/rm $DIRSCR/$jid.*.t83* 2>/dev/null
/bin/rm $DIRSCR/$jid.t84 2>/dev/null
/bin/rm $DIRJOB/$jid.pid 2>/dev/null
/bin/rm $DIRJOB/$jid.sle 2>/dev/null
/bin/rm $DIRJOB/$jid.sin 2>/dev/null
fi
fi
fi