THIS IS A TEST INSTANCE. ALL YOUR CHANGES WILL BE LOST!!!!
Apache Airavata
#!/bin/sh
# This is the main script that creates the queue-submit script
# PLEASE EXECUTE THE SCRIPT WITH -? TO SEE IMPORTANT INFO
#
# Approximate Organization:
# - Input Processing (after init of input parameters)
# - Local Processing (after init of local parameters)
# - Resource scheduler Parameters
# - Write qsub-file
# - Submit job
# - Produce the required output (eg jobid)
# 2006/11/08 skk Added qmcpack by merging Sudhakar's comments
# helpful variables and functions
null="null"
### 2006/03/23 moved here from futher below since it may be needed very early
# check existence and create if needed $HOME
if [ ${HOME:-null} = "$null" ] ; then
HOME=`pwd`
echo "WARNING: the HOME DIR variable IS NOT DEFINED - DEFINED IT AS HOME=$HOME as a simple variable just for this script"
fi
# 2008/09/12 add path for sendmail (for queenbee mostly but anyway)
PATH=$PATH:/usr/sbin
# 2006/04/09 skk - todo - better to remove all the inits below - and just "argument check" later
# DEFAULTS FOR INPUT ARGUMENTS
Account=default # default
Queue=UNKNOWN
MachineName=UNKNOWN
ResName=UNKNOWN
NProc_arg=4 # 2006/05/25 if gaussian, then this is outdated since we will deduce it from the %nproclinda and %nprocshared di
rectives
SwapMem_arg=1700 # based on Chona's scripts 2005/06/23
CPU_Time_arg="" # 2005/12/20 CURRENTLY NOT REALLY NEEDED SINCE WE ALWAYS SEND IN THE walltime_arg??
TEMPDIR1=UNKNOWN # 2006/05/24 Dir that holds the input file BUT ALSO it defines OutDir (=long-term scratch) by .... see la
ter on
EmailOfUser=UNKNOWN
ResearchProject=UNKNOWN
GridchemUserid=UNKNOWN # Actually this should be named the GridchemUsername eg kyriacou - 2005/11/30
CommandToRun=UNKNOWN
SetVerbose="" # this string will be evaluated within this file, and also included in the batch-script
LindaIsAvailable=0
# 2007/12/07 skk useful functions
exit_with_message () {
echo;
echo ERROR: $* ;
exit 1
}
# 2006/05/16 IMPORTANT: If "set -x" is used, the client seems to try to submit the job 3 times, failing each time
#set -x #TEMP 2005/12/20
#2008/07/21 mjs added -u option for database-retrieved user email. defaults to kyriacou@osc.edu
# INPUT ARGUMENTS
while getopts a:q:m:r:n:M:c:O:e:u:p:g:C:xW:l:j: option
do
# look at each option one at a time checking for validity
case $option in
a ) Account=$OPTARG ;;
q ) Queue=$OPTARG ;;
m ) MachineName=$OPTARG ;; #NOT USED -- remove?? 2005/05/16
r ) ResName=$OPTARG ;;
n ) NProc_arg=$OPTARG ;;
M ) SwapMem_arg=$OPTARG ;;
W ) walltime_arg=$OPTARG;;
c ) CPU_Time_arg=$OPTARG;; #
O ) TEMPDIR1=$OPTARG;;
e ) EmailOfUser=$OPTARG;;
u ) userMail=$OPTARG ;;
p ) ResearchProject=$OPTARG;;
g ) GridchemUserid=$OPTARG;;
C ) CommandToRun=$OPTARG;; # g09 g03l rungms nwchem molpro qmcpack amber tinker ddscat
x ) SetVerbose="set -x" ;;
l ) LindaIsAvailable=$OPTARG ;;
j ) jobName=$OPTARG ;;
\? ) cat << ENDCAT1
>! Usage: $0 [-a account name] !<
>! [-q queue name] !<
>! [-m machine name] !<
>! [-r resource name] !<
>! [-n number of threads/processors] !<
>! [-M peak swap memory] !<
>! [-W walltime Limit [hh:mm:ss] !<
>! [-c CPU time Limit [hh:mm] !<
>! [-O Output Directory] !<
>! [-x Bourne Shell echo] !<
>! [-e EmailOfUser] !<
>| [-u User email address from database |<
>! [-p ResearchProject] !<
>! [-g GridchemUserid] !<
>! [-C CommandToRun [eg g03l or rungms] !<
>! [inputfile (2008/04/09 OR DUMMY? for amber multi-input method) ] !<
OPTIONS BY DECREASING PRIORITY
(2006/04/24 Per Sudhakar, priority is first the script and then the arguments - so i ll switch them and fix the scripts)
- Specific-program-script directives (eg the gaussian input %nproclinda guideline)
- Command-line arguments of the ccgresources scripts (eg -n nproc )
- Defaults in the submit scripts
- ??WE WILL NOT CHECK FOR ENV vars.
USAGE EXAMPLE: assuming we have a queue=debug and we want to run on 4 procs a g03l (g03 2with linda) input g03l_job.inp
with output in the current dir for 30 min?
$0 -C g03l -n4 -q debug -c 00:30 -O \$PWD g03l_job.inp
UPDATE 2008/04/09 : Maybe a better example is this
~/bin/unified_scripts_DEVEL/gridchem_common -C g03l -a kts -q gauss -W 30:00 -n 2 -g kyriacou -p kyriacou_proj -O /scratch/ccg
user/kyriacou/tempdirs/trash -l 0 /scratch/ccguser/kyriacou/tempdirs/trash/default_test.inp
ENDCAT1
exit 2 ;;
esac
done
# mjs 2008/07/21 added userMail argument.
##userMail_cc="$userMail kyriacou@osc.edu spamidig@ncsa.uiuc.edu";
userMail_cc="spamidig@ncsa.uiuc.edu"; # people to cc the userMail mail 2008/09/26 skk
# 2005/08/17 TODO: this might need improvement - SEE ALSO bkrmk20051220
if [ -n "$walltime_arg" ] ; then
echo "INFO: Currently the walltime is translated to cputime directly "; echo
if [ -n "$CPU_Time_arg" ] ; then
echo "WARNING: $CPU_Time can NOT be specified together with walltime - ignoring CPU_Time in favor of walltime"; echo
fi
CPU_Time_arg=$walltime_arg
fi
# Echo back command line arguments
# 2006/05/25 for gaussian the NProc_arg is not really used
echo "INFO: $0 Job to be submitted using $NProc_arg processors for Hrs:Min _${CPU_Time_arg}_ in queue _${Queue}_ with PSN _${A
ccount}_ " ; echo
eval "$SetVerbose"
# SOME USEFUL VARIABLES
PROGRAM_DIR=`dirname $0` # this so as to avoid hardcoding the path of sourced files - assumed in same dir as the PROGRAM
crtn=" \\c"
SLASH="\\" # to improve fontification of scripts in emacs etc
# just a sanity check
#2008/07/17 mjs todo iterate over an array instead?
if [ ${CommandToRun:-null} = "g03l" ]; then
PROGRAM_TO_RUN="g03l" # 2007/11/30 should i remove the variable PROGRAM_TO_RUN and just use CommandToRun? todo:
elif [ ${CommandToRun:-null} = "g09" ]; then
PROGRAM_TO_RUN="g09"
elif [ ${CommandToRun:-null} = "Gamess" ]; then
PROGRAM_TO_RUN="rungms"
elif [ ${CommandToRun:-null} = "NWChem" ]; then
PROGRAM_TO_RUN="nwchem"
elif [ ${CommandToRun:-null} = "Molpro" ]; then
PROGRAM_TO_RUN="molpro"
elif [ ${CommandToRun:-null} = "qmcpack" ]; then
PROGRAM_TO_RUN="qmcpack"
elif [ ${CommandToRun:-null} = "Adf_adf" ]; then
PROGRAM_TO_RUN="adf"
elif [ ${CommandToRun:-null} = "adf_quild" ]; then
# 2007/11/30 joohyan
PROGRAM_TO_RUN="adf_quild"
elif [ ${CommandToRun:-null} = "wien2k" ]; then
PROGRAM_TO_RUN="wien2k"
exit_with_message "wien2k STILL NEEDS TO BE WORKED ON"
# 2007/11/29 joohyan
elif [ ${CommandToRun:-null} = "aces3" ]; then
PROGRAM_TO_RUN="aces3"
elif [ ${CommandToRun:-null} = "amber_sander" ]; then
PROGRAM_TO_RUN="amber_sander"
elif [ ${CommandToRun:-null} = "DMol3" ]; then
PROGRAM_TO_RUN="dmol3"
elif [ ${CommandToRun:-null} = "CASTEP" ]; then
PROGRAM_TO_RUN="castep"
elif [ ${CommandToRun:-null} = "ddscat-v7_1" ]; then
PROGRAM_TO_RUN="ddscat-v7_1"
elif [ ${CommandToRun:-null} = "ddscat-v7_2" ]; then
PROGRAM_TO_RUN="ddscat-v7_2"
elif [ ${CommandToRun:-null} = "lammps_lmp" ]; then
PROGRAM_TO_RUN="lammps"
elif [ ${CommandToRun:-null} = "lammps_ds" ]; then
PROGRAM_TO_RUN="lammps_ds"
elif [ ${CommandToRun:-null} = "tinker_monte" ]; then
PROGRAM_TO_RUN="tinker_monte"
elif [ ${CommandToRun:-null} = "dftb+" ]; then
PROGRAM_TO_RUN="dftb+"
elif [ ${CommandToRun:-null} = "Charmm" ]; then
PROGRAM_TO_RUN="charmm"
elif [ ${CommandToRun:-null} = "abaqus" ]; then
PROGRAM_TO_RUN="abaqus"
elif [ ${CommandToRun:-null} = "mdrun_mpi" ]; then
PROGRAM_TO_RUN="mdrun_mpi"
else
exit_with_message "PROGRAM_TO_RUN is $CommandToRun"
fi
# find the DOMAIN: used as part of the scriptname gridchem_local_$DOMAIN
# sdx etc does not have a dnsdomainname
# ORDER IS IMPORTANT BELOW
# 2008/09/24 skk todo: We should automate this so that it comes from the database ?
if [ `hostname` = "bcxadmin" ] ; then
DOMAIN=ccs_bcx
elif hostname|grep -q '^champ' ; then
DOMAIN="tacc"
elif hostname|grep -q '^lslogin' ; then
DOMAIN="tacc_lonestar"
# 2005/12/15 elif hostname | grep -q '^Cu..$' ; then
elif hostname | grep -q '^cu..$' ; then
DOMAIN="ncsa_copper"
# 2006/06/08 For teragrid cobalt
elif hostname | grep -q '^co-login' ; then # 2006/06/09 co-login1.ncsa.uiuc.edu
DOMAIN="ncsa_cobalt"
# Jayeeta added ember
elif hostname | grep -q '^ember' ; then # 2010/10/20 ember.ncsa.illinois.edu
DOMAIN="ncsa_ember"
elif hostname | grep -q 'stampede' ; then
DOMAIN="tacc_stampede"
elif hostname | grep -q '^trestles' ; then # Added by Ye 2011/06/06 trestles.sdsc.teragrid.org
DOMAIN="sdsc_trestles"
elif hostname | grep -q '^gordon' ; then # Added by Ye 2011/06/06 gordon.sdsc.teragrid.org
DOMAIN="sdsc_gordon"
elif dnsdomainname| grep -q 'ncsa.teragrid.org' ; then # 2007/04/27 skk - mercury
DOMAIN="ncsa_mercury"
elif [ `dnsdomainname` = "dim" ] ; then # 2007/05/29 skk change from grep to [ = ] - 2007/5/18 jk bigred should be changed lat
er
DOMAIN="iu_bigred"
elif dnsdomainname|grep -q osc.edu ; then
DOMAIN="osc"
elif dnsdomainname|grep -q uky.edu ; then
DOMAIN="ccs"
elif dnsdomainname|grep -q lsu.edu ; then
DOMAIN="cct"
elif hostname | grep -q loni.org ; then
DOMAIN='cct_queenbee'
elif hostname | grep -q '^honest..ncsa.uiuc.edu' ; then
DOMAIN='ncsa_abe'
# 20080812 skk - i think this should be last
elif dnsdomainname|grep -q ncsa.uiuc.edu ; then
DOMAIN="ncsa"
elif dnsdomainname|grep pople.psc.teragrid.org; then
DOMAIN="pople"
else
exit_with_message "dnsdomainname is $dnsdomainname -- not in available sites" ;
fi
# 2006/09/29 skk - this is useful when i look in the glsublog files
echo "INFO: we are in DOMAIN $DOMAIN"
# SOURCE LOCAL STUFF
# ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
# defaults for local stuff
EXTRASLASH="" #to be used with uberftp command to ease remote execution
local_qsub_extra="" # for the bsub to send a " < "
local_gauss_source_command=""
local_gms_source_command=""
export LOGIN=`/usr/bin/whoami` # 2006/04/09 todo move this at the begin of script
GMS_VERNO="00" # - version number for rungms
# 2006/10/06 chona suggestion - Check if SYSTEM_LOG is already defined
[ -n "$SYSTEM_LOG" ] || SYSTEM_LOG="$HOME/GridChem_Logs/$PROGRAM_TO_RUN" # 2006/03/30 skk this seems to not be useful - mayb
e remove? todo-
grid_proxy_info="/opt/apps/xsede/gsi-openssh-5.8p2/bin/grid-proxy-info"
PBS_V_FLAG=" -V" # TEMP the -V flag in pbs was giving me problems in AGT cluster
TACC_SPECIFIC_MEMORY="" # needed only in TACC according to Chona - default is empty
TUNGSTEN_SPAN_PTILE_1="" # default needs to be empty
NWCHEM_LOCAL_OPTIONS_TO_EVAL="" # default needs to be empty
NWCHEM_LAPI_NETWORK="" # default needs to be empty
NWCHEM_LOCAL_VARS="" # default needs to be empty
ADF_LOCAL_CU_OPTION=""
ACES3_LOCAL_OPTION="" # 2007/11/29 joohyun
BigRed_LOCAL_OPTION=""
LL_LOCAL_OPTIONS_TO_EVAL="" # TACC specific??
LL_LOCAL_OPTIONS=""
local_preamble_part1="" # to add some local directives - eg vmem at OSC itanium (2007/03/20 skk - todo - this does not seem
right, we want the vmem for all not just OSC itanimum, although it is not so important for clusters, for SMP like cobalt it i
s important i think )
MPIRUN_COMMAND="" # default needs to be empty
NPROC_GAUSS_OSC_ITANIUM=""; # to be able to change num of procs to 2 if osc itanium - we dont have linda there. - 2007/05/18 w
e probably can get rid of this var - now we use altix if we ask for more than 2 and gaussian todo:
NPROC_GAUSS_NCSA_MERCURY=""
LOCAL_LL_CONSUMABLE_CPUS=1 # this is good for champ
# 2005/12/22 only needed for loadleverer - since copper is using a different shell
#####- it does need this funny format with attached newline
LL_SHELL_TO_USE="#!/bin/sh"
###2007/05/29 -- OLDER>>-- 2006/01/25 mike4 does not allow ppn=1 so we ll create this variable to handle it - DEFAULT HAS TO
BE 1 I BELIEVE - also used for nprocshared directive - (2006/05/25 also will use in tg-login1 since no linda and better two th
an one)
LOCAL_PROCS_PER_NODE=1 # 2007/05/29 renames from PBS_PROCS_PER_NODE to LOCAL_PROCS_PER_NODE since now it is used in other sch
edulers eg ll in bigred
OSC_CCG_DO_NOT_USE_DEBUG_QUEUE="" # 2006/02/06 - 2007/07/13 skk i think this is outdated (since outdated ccg-login)
SCRATCH_ROOT="$HOME/scratch"; # 2006/03/31 Is this needed?? we could use TEMPDIR1 which is already localized in specifications
.cfg todo++
clustername="UNKNOWN_CLUSTERNAME" # clustername is the same as machine-name in specifications.cfg - eventually i need t
o get it from server - skk 2005/10/17
local_uname_arch_command="uname -p " # this does not work in .... so it is localized
SCRATCH_BATCH_DIR_IS_ACCESSIBLE=YES # 2006/06/08
local_no_nnodes_allowed="" # if empty we use nodes=N:ppn=2 directive (in pbs) - otherwise (altix) that uses ncpus=N instead
OUTPUT_DIR_IS_ACCESSIBLE="YES" # if true="non empty", it means that ouput dir is visible from the login node (eg not true for
sdx for now 2006/07/14 )
# 2007/07/30 skk added this REMOTE_EXEC="ssh" as default - i think it is pretty safe and a default is always a good idea
REMOTE_EXEC="ssh"
tmp="$PROGRAM_DIR/gridchem_local_$DOMAIN"
echo ; echo "INFO: SOURCING $tmp (LOCAL STUFF ie variables that change from site to site)"; echo
. $tmp
# ))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))
echo "DEBUG: DATE:`date` - COMMAND_RECEIVED: $0 $* " ; echo ###| tee -a $SYSTEM_LOG ; echo ; # DEBUG
# POSTPROCESS LOCAL STUFF
# 2006/02/06
# 2006/10/06 This was buggy and changed all tg-login queues to serial - fixed now
if [ -n "$OSC_CCG_DO_NOT_USE_DEBUG_QUEUE" ] ; then
if [ "$Queue" = "debug" ]; then
echo "WARNING: The requested Queue $Queue WOULD NOT WORK (this is a bug in the client 2006/02/06) - changing queue to corr
ect $OSC_CCG_DO_NOT_USE_DEBUG_QUEUE"; echo;
Queue=$OSC_CCG_DO_NOT_USE_DEBUG_QUEUE ;
fi
fi
# 2005/08/09 since we dont have linda part - if gaussian do queue to serial
if [ -n "$NPROC_GAUSS_OSC_ITANIUM" ] ; then # this condition is separate than next two since there is not short-circuit in ba
sh if i put all 3 on same command line (-)
if [ "$PROGRAM_TO_RUN" = "g03l" ] ; then
# 2006/05/25 moved a few lines elsewhere due to Nproc_Arg not used if gaussian
# 2006/05/25 shouldn't this better go into the osc local script?
echo "DEBUG: Queue is $Queue"; echo;
if [ "$Queue" = "parallel" ] ; then
echo "WARNING: (we do not have linda at OSC-itanium) queue is set to serial instead of parallel (otherwise job will fail
to run due to PBS restrictions)"
Queue="serial"
fi
fi
fi
# check to make sure that $SYSTEM_LOG file exists
if [ ! -w $SYSTEM_LOG ] ; then
###2007/04/11 15:31skif touch $SYSTEM_LOG 2>&1 >/dev/null ; then
if touch $SYSTEM_LOG >/dev/null 2>&1 ; then
echo "INFO: Created file $SYSTEM_LOG"
else
# 2005/11/23 changed WARNING into INFO - it is not important after all
echo "INFO: SYSTEM_LOG is $SYSTEM_LOG and is not writable - Temporarily set to $SCRATCH_ROOT/temp_system_log" ;
echo
### 2006/03/31 SYSTEM_LOG="$SCRATCH_ROOT/temp_system_log"
SYSTEM_LOG="$TEMPDIR1/../../../temp_system_log"
fi
fi
# EVAL THE LOCAL_MODULE_ENV_COMMAND TO PICK UP SUITABLE ENV
# 2005/06/01 skk DO WE REALLY NEED THIS?
# 2007/07/30 skk This is a problem for bcxadmin since not gaussian module on it - so i ll remove and hopefully nothing will br
eak
# 2008/09/24 skk - uncommented it since it is needed in pople and for now bcx is not available - (todo: eventually we should r
emove the similar command at approx line 1200?)
# DEFAULTS FOR INPUT ARGUMENTS THAT ARE LOCALIZED
if [ ${Queue:-null} = $null -o ${Queue:-null} = "UNKNOWN" ] ; then Queue=$LOCAL_QUEUE; fi # 2007/07/13 skk i dont think th
is is needed - we always define Queue - so i can remove the LOCAL_QUEUE variable
# 2005/05/23 TODO- use (buildin) pwd rather than /bin/pwd so as to pick up the canonical /home/.... instead of the actual moun
ted dir
pwd="pwd"
Currdir=`$pwd` # the home dir if remotely executed
# TODO rename this APPLICATION_MEM?? 2005/06/23
if [ $PROGRAM_TO_RUN = "g03l" ] ; then
# G98 defaults - please set these numbers based on the Default.Route file in GAUSS_EXEDIR
NumProc="1"
G98_mem="1000000000"
elif [ $PROGRAM_TO_RUN = "g09" ] ; then
NumProc="1"
G98_mem="1000000000"
elif [ $PROGRAM_TO_RUN = "rungms" ] ; then
GAMESS_mem="64000000"
GAMESS_memM="64"
elif [ $PROGRAM_TO_RUN = "abaqus" ] ; then
abqversion="6.10-2"
abaqus_executable=abq692.exe
fi
setup_X509_credentials () {
# uberftp needs credentials to work and the compute nodes have a different /tmp than login node
# need to transfer the certificate from /tmp to scratch area so that it ll be available on the nodes too
# set up the X509_USER_PROXY if there is a valid cert
local tmp=`$grid_proxy_info`
# 2006/09/19 skk SANITY_CHECK
if [ $? -ne 0 ] ; then
exit_with_message "$grid_proxy_info failed with this output $tmp. Maybe you need to make sure that you do not overwrite LD_L
IBRARY_PATH or PATH etc from your init scripts eg .cshrc"
fi
local valid_certif_exists=1
# quotes are important in the next few lines in "$tmp" otherwise $tmp will become one long line
echo "$tmp" | grep -q timeleft || (echo "WARNING: problem with grid-proxy-info having no timeleft line = $tmp"; valid_certif_e
xists=0)
# ASSUMPTION: line is "timeleft : 0:00:00" if no timeleft
echo "$tmp" | grep timeleft | grep -q "0:00:00" && (echo "WARNING: problem with grid-proxy-info having ZERO timeleft line = $t
mp"; valid_certif_exists=0)
local tmp2=`echo "$tmp" |grep 'path :'| sed 's/path : //'`
##echo "$tmp" $tmp2
if [ $valid_certif_exists -eq 1 ] ; then
X509_USER_PROXY=$tmp2 ;
echo "INFO: Temporarily setting the X509_USER_PROXY to $X509_USER_PROXY based on grid-proxy-info"
fi
if [[ -n "$X509_USER_PROXY" ]] ; then
if echo $X509_USER_PROXY |egrep '^/tmp/' >/dev/null ; then
if [ ! -d $TEMPDIR1 ]; then
exit_with_message "Directory $TEMPDIR1 should have been created by calling program"
fi
/bin/cp -fp $X509_USER_PROXY $TEMPDIR1
# use "#" because $TEMPDIR1 has "/" in it
export X509_USER_PROXY=$(echo $X509_USER_PROXY | sed "s#\/tmp#$TEMPDIR1#")
echo "INFO: setting the X509_USER_PROXY to $X509_USER_PROXY and copied the cert there from /tmp"
fi
else
echo "NOTE There is no X509_USER_PROXY so i assume you are running the script locally"
echo " Please set this variable to your credential file (eg export X509_USER_PROXY=..../x509up_u{YOUR_USERID} ) and reru
n"
echo " if you want to do uberftp files back to mss -- (BUT IF YOU USE THE SSH AUTHENTICATION THIS IS NOT NECESSARY)"
#exit 1
fi
}
setup_X509_credentials
# 2005/10/14 TODO- move these two vars below into the submit-script since they are not used here
# create a unique subdir structure
# 2005/10/21 i add the date to the jobdir and remove the step
# 2005/10/21 remotedir="$GridchemUserid/$ResearchProject/\$jobdir"
# the topdir_name is used in mss and mw server as the top level dir for saving files underneath
# 2006/10/03 IMPORTANT - THIS IS ALSO HARDCODED IN SERVER SCRIPTS
# 2007/05/21 adding -o "$LOGIN" = "gridchem" for sdsc and -o "$LOGIN" = "tg-gridchem" for bigred - TODO: this is temp - the s
olution should be to get a flag from webservice that this is $internal=1
if [ "$LOGIN" = "ccguser" -o "$LOGIN" = "gridchem" -o "$LOGIN" = "tg-gridchem" ] ; then
topdir_name="internal"
else
#topdir_name="external"
topdir_name="internal"
fi
remotedir_common="$GridchemUserid/$ResearchProject/\$jobdir" # this is the common part of OutDir and remotedir
###2006/03/31 OutDir="${SCRATCH_ROOT}/$remotedir_common" # 2006/03/31 OutDir should be renamed to OutDir
# 2008/05/21 skk (+) todo - make it: tmp=`basename $TEMPDIR1`; tmp=`basename $tmp`; OutDir="$tmp/$ResearchProject/\$jobdir"
OutDir=`readlink -f $TEMPDIR1/../../`
`mkdir -p $OutDir/$ResearchProject`
OutDir="$OutDir/$ResearchProject/\$jobdir"
#OutDir="$TEMPDIR1/../../$ResearchProject/\$jobdir" # 2006/03/31 eg .../kyriacou/tempdirs/tempdirXXXXXX/../../$ResearchProjec
t/\$jobdir
remotedir="$topdir_name/$remotedir_common"
# 2006/03/31 skk - this needs to be the new batch-scratch - eg for copper and tungsten it will be $SCR , for OSC it will be $T
MPDIR etc
# 2006/03/31 this is where the job will be running - eventually it will be a different dir that $OutDir eg $OutDir/batch_scrat
ch -todo: start using this in all places it should be
###2006/03/31 SCRATCH_BATCH=$SCRATCH0;
# 2007/05/03 skk Added cobalt to use the $SCR env variable as the SCRATCH_BATCH directory
# 2010/10/21 Jayeeta added ember
if [ "$clustername" = "stampede.tacc.utexas.edu" ]; then
#SCRATCH_BATCH="/scratch1/\$USER/\$PBS_JOBID"
SCRATCH_BATCH="/scratch/00421/$USER/\$SLURM_JOBID"
#mkdir -p $SCRATCH_BATCH
fi
# PROCESS INPUT FILE
if [ $# -ge 1 ]
then
shift `expr $OPTIND - 1`
inputfile=$1
fi
# 2006/03/31 this was part of an interactive part - possibly part of it is not needed
V1=`echo $inputfile`
Diskinputfile=`/bin/basename $inputfile`
Diskinputdir=`dirname $inputfile`
if [ $PROGRAM_TO_RUN = 'lammps' -o $PROGRAM_TO_RUN = "lammps_ds" ] ; then
inputFileArray=(`echo $inputfile | tr "," "\n"`)
inputfile="${inputFileArray[1]}"
if [[ ! -z ${inputFileArray[1]} ]] ; then
isMultipleInput=1;
else
isMultipleInput=0;
fi
inputfile="${inputFileArray[0]}"
Diskinputfile=`basename $inputfile`
Diskinputdir=`dirname $inputfile`
fi
if [ $PROGRAM_TO_RUN = 'dftb+' ] ; then
inputFileArray=(`echo $inputfile | tr "," "\n"`)
inputfile="${inputFileArray[1]}"
if [[ ! -z ${inputFileArray[1]} ]] ; then
isMultipleInput=1;
else
isMultipleInput=0;
fi
inputfile="${inputFileArray[0]}"
# extract arguments from inputfile
# xyzfile; keyfile; steps[1000]; Cartesian/Torsional space[C]; stepsize(ang[C][3.0]/deg[T][180]); Temp[500K]; RMSgrad_conv
_criterion[0.01];
# while read line
# do
# read xyzf; read keyf; read stps; read CTc; read stpsz; read temp; read Rconv;
# #echo $xyzfile $keyfile $steps $CTchoice $stepsize $temp $RMSconv
# done < $inputfile
Diskinputfile=`basename $inputfile`
Diskinputdir=`dirname $inputfile`
fi
if [ $PROGRAM_TO_RUN = 'tinker_monte' ] ; then
inputFileArray=(`echo $inputfile | tr "," "\n"`)
inputfile="${inputFileArray[1]}"
if [[ ! -z ${inputFileArray[1]} ]] ; then
isMultipleInput=1;
else
isMultipleInput=0;
fi
inputfile="${inputFileArray[0]}"
# extract arguments from inputfile
# xyzfile; keyfile; steps[1000]; Cartesian/Torsional space[C]; stepsize(ang[C][3.0]/deg[T][180]); Temp[500K]; RMSgrad_conv
_criterion[0.01];
while read line
do
read xyzf; read keyf; read stps; read CTc; read stpsz; read temp; read Rconv;
#echo $xyzfile $keyfile $steps $CTchoice $stepsize $temp $RMSconv
done < $inputfile
Diskinputfile=`basename $inputfile`
Diskinputdir=`dirname $inputfile`
fi
if [ $PROGRAM_TO_RUN = "ddscat-v7_1" -o $PROGRAM_TO_RUN = "ddscat-v7_2" ]; then
# Get the input directory
inputdir=`dirname $inputfile`
#inputFileArray=$(echo $inputfile | tr "," "\n")
#inputfile="${inputFileArray[0]}"
IFS=","
for file in $inputfile; do
Diskinputfile=$file;
break
done
# 2005/04/01 skk -- if the filename is not fully qualified, fix both the inputdir and the inputfile
# note that inputdir? outputs a "." if the file is in the current dir
if echo $inputdir |egrep '^\.' ; then
exit_with_message "the inputfile should be fully qualified name"
fi
cd $inputdir
Diskinputdir=`$pwd` # 2006/03/31 is this the inputdir with no soft links in it??
Diskinputfile=`/bin/basename $Diskinputfile` # this is just the inputfile without the dirname
cd $Currdir
fi
# if the Diskinputdir has /nfs in it, cut it out
# - causes problem particularly on idun
#
Diskinputdir=`echo $Diskinputdir | /bin/sed 's/\/nfs//'`
#
# 2008/05/19 skk new Rion scripts send the jobName as a flag
if [ -n "$jobName" ]; then
job=$jobName;
else
job=`/bin/basename $V1`
dummy=`expr $job : '.*\(\..*\)'` # gets suffix
job=`/bin/basename $job $dummy` # removes suffix - a better for this would be inputname_nosuffix? todo
fi
jobname_length=`echo "$job"|wc -c` # 2006/08/31 if over 15 chars or if it starts with not alphabetic it fails in mike4 - TO
DO implement
# Number of nodes (nnodes) used in PBS - and number of CPUS (cpus) used in LSF and LL (LL uses it through variables LL_CONSUMA
BLE_CPUS and ....)
cpus=$NProc_arg
# 2006/04/24
if [ $PROGRAM_TO_RUN = "g03l" -o $PROGRAM_TO_RUN = "g09" ] ; then # we dont care about NProc_arg in this case - per Sudhakar
2006/04/24
# Input file directive %nproc* is always used now 2006/05/26
###2008/01/02 OUTDATED COMMENT # (2007/12/20 skk i think i am already doing that ) 2006/05/25 TODO++ if there are both npro
clinda and nprocshared directives i should have cpus be the multiplication of the two
nproclinda_max=`perl -e '$nproclinda_max=0; while (<>) { if (/^\s*\%nprocl\w*\s*=(\d+)/i) {$nproclinda_max=$1 if ($1 > $npro
clinda_max)}; }; print "$nproclinda_max\n";' $inputfile `
nprocshared_max=`perl -e '$nprocshared_max=1; while (<>) { if (/^\s*\%nprocs\w*\s*=(\d+)/i) {$nprocshared_max=$1 if ($1 > $n
procshared_max)}; }; print "$nprocshared_max\n";' $inputfile `
nproc_max=`perl -e '$nproc_max=1; while (<>) { if (/^\s*\%nproc\s*=(\d+)/i) {$nproc_max=$1 if ($1 > $nproc_max)}; }; print
"$nproc_max\n";' $inputfile `
echo "DEBUG: nprocshared (max) from input $inputfile is $nprocshared_max"
# 2008/01/02 skk - per Sudhakar if all three %nproc* are set, let us make nproc_max=1 since it is depreciated; give a warnin
g
if [ $nproclinda_max -ne 0 ] ; then
if [ $nprocshared_max -gt 1 ] ; then
if [ $nproc_max -gt 1 ] ; then
echo "WARNING: Seems we found all three keywords %nproclinda, %nprocshared, %nproc - Ignoring the %nproc ";
nproc_max=1
fi
fi
fi
if [ "$LindaIsAvailable" = "1" ] ; then
if [ $nproclinda_max -ne 0 ] ; then
cpus=$(echo "$nproclinda_max*$nprocshared_max"|bc)
elif [ $nproc_max -gt 1 ] ; then
cpus=$nproc_max
echo "DEBUG: nproc from input is $cpus "
elif [ $nprocshared_max -gt 1 ] ; then
cpus=$nprocshared_max
echo "DEBUG: nprocshared from input is $cpus "
else
echo "DEBUG: could not find any nproclinda; and nproc or nprocshared did not exist or were equal to 1 - DEFAULTING TO cp
us=1"
cpus=1
fi
else # IE IF NO LINDA
[ $nproc_max -gt 1 ] && cpus=$nproc_max
[ $nprocshared_max -gt 1 ] && cpus=$nprocshared_max
echo "DEBUG: nprocshared from input is $cpus "
if [ $nproc_max -eq 1 -a $nprocshared_max -eq 1 ] ; then
# 2006/08/09 skk - let us handle the case when they used nproclinda=N instead of nprocshared=N or nproc=N
if [ $nproclinda_max -ne 0 ] ; then
echo "WARNING: seems that we found no %nprocshared or %nproc directives (this is an SMP machine). Instead we found an
%nproclinda directive- we will pretend that the user meant %nprocshared, and proceed accordingly"
# Note that the actual file directive will be changed later to %nproshared. see bkmrk200608091745
cpus=$nproclinda_max
else
cpus=1 ;
fi
fi
# 2008/01/02 skk - if they give both nproclinda and nprocshared, let us multiply them
if [ $nproclinda_max -ne 0 -a $nprocshared_max -gt 1 ] ; then
cpus=$(echo "$nproclinda_max*$nprocshared_max"|bc)
fi
fi
if [ -n "$NPROC_GAUSS_NCSA_MERCURY" ] ; then
if [ $cpus -gt $NPROC_GAUSS_NCSA_MERCURY ] ; then
echo "WARNING: The number of processors has been reduced from $cpus to $NPROC_GAUSS_NCSA_MERCURY - we do not have linda
at mercury "
cpus_old=$cpus
cpus=$NPROC_GAUSS_NCSA_MERCURY;
fi
fi
fi
# 2006/06/02 round up the number of cpus to be multiples of $LOCAL_PROCS_PER_NODE
if [ $LOCAL_PROCS_PER_NODE -gt 1 ] ; then
remainder=$(echo "$cpus%$LOCAL_PROCS_PER_NODE"|bc);
if [ $remainder -ne 0 ] ; then
cpus_old=$cpus;
# if there is a remainder we make it ....
cpus=$(echo "$LOCAL_PROCS_PER_NODE*($cpus_old/$LOCAL_PROCS_PER_NODE+1)"|bc);
echo "WARNING: setting cpus from $cpus_old to $cpus since we accept only multiples of $LOCAL_PROCS_PER_NODE " ;
fi
fi
# 2006/06/02 this is only for tacc for now
# IMPORTANT: this eval uses and modifies the variable "cpus" - SO IT HAS TO BE HERE ie LAST of cpus changing commands
[ -n "$local_fix_cpus_cmd" ] && eval "$local_fix_cpus_cmd"
# Done with modifying cpus - so now we can get the nnodes and also fix the gaussian directives
# 2006/04/19 here it is the right place to evaluate nnodes=number of nodes
nnodes=`echo "$cpus/$LOCAL_PROCS_PER_NODE" |bc ` # 2006/04/19 number of nodes - i am not sure if LOCAL_PROCS_PER_NODE is a go
od variable name
#echo "DEBUG111: nnodes=$nnodes - cpus=$cpus - LOCAL_PROCS_PER_NODE=$LOCAL_PROCS_PER_NODE ";
if [ $PROGRAM_TO_RUN = "g03l" -o $PROGRAM_TO_RUN = "g09" ] ; then
# NOW CHANGE THE input FILE to use the correct directive in each case
echo "DEBUG: *originally* the input file has these gaussian %nproc directives in it "
grep "%nproc" $inputfile
# i ll first change all %nprocs* to nprocshared, and %nprocl* to nproclinda
perl -i.orig -pe 's/^\s*\%nprocs\w*\s*=/\%nprocshared=/i ; s/^\s*\%nprocl\w*\s*=/\%nproclinda=/i ; ' $inputfile
if [ "$LindaIsAvailable" = "1" ] ; then
# if there are any %nproc make them nproclinda
perl -i -pe 's/^\s*\%nproc\s*=/\%nproclinda=/i ; '
if [ $LOCAL_PROCS_PER_NODE -gt 1 ] ; then
# 2006/05/24 skk - try to fix mike4 where if user asks for nproclinda=4, it should be nproclinda=2 and nprocshared=2
# we assume that the local_fix_cpus_cmd has already taken care of the cpus being a multiple of LOCAL_PROCS_PER_NODE
nprocshared=$LOCAL_PROCS_PER_NODE
nproclinda=$(echo "$cpus / $LOCAL_PROCS_PER_NODE" | bc )
# (2006/07/20 documenting: 1. replace nproclinda= with correct $nproclinda= - 2. same for nprocshared - 3. if "nproclind
a=1" make it nprocshared=1 (because?))
# 2006/07/26 I had to remove the "\s*$" from "nproclinda=)\s*\d+\s*$/" etc because it was removing the %n too - for the
same reason the last one needs the \n
perl -i -pe ' s/(?<=\%nproclinda=)\s*\d+/'"$nproclinda"'/i ; s/(?<=\%nprocshared=)\s*\d+/'"$nprocshared"'/i ; s/^\s*\%n
proclinda=\s*1\s*$/%nprocshared=1\n/i ' $inputfile
###2006/07/20 perl -i -pe ' s/(?<=\%nproclinda=)\w*\d+\w*$/'"$nproclinda"'/i ; s/(?<=\%nprocshared=)\w
*\d+\w*$/'"$nprocshared"'/i ; s/^\s*\%nproclinda=\w*1\w*$/%nprocshared=1/i ' $inputfile
# 2006/07/26 but if nproclinda is NOT there AT ALL we need to add it - otherwise BKMRK_LINE01 below will not work
if ! grep -q -i "%nproclinda" $inputfile ; then
echo "WARNING: there is no nproclinda in file - adding: %nproclinda=$nproclinda in first part and for each --Link1-- (
right before first \"#\" "
# note that the "\n#" indicates where to put the gaussian directives - per sudhakar - skk 2006/07/26
# 2006/08/31
perl -i -0777 -pe ' # -0777 to slurp whole file as one record
s/^/%nproclinda='"$nproclinda"'\n/; # put the directive in first line in file
s/(--link1--(\n|.)*?)#/$1%nproclinda='"$nproclinda"'\n#/img; # put the directive on top and after each --Link1--
right before the first #
' $inputfile
### 2006/08/31 perl -i -pe 's/^\s*#/%nproclinda='"$nproclinda"'\n#/i' $inputfile
else
echo "DEBUG: there is nproclinda in the file `cat $inputfile` "
fi
# if there is no nprocshared we need to add it
if ! grep -q -i "%nprocshared" $inputfile ; then
# perl -i -ne 'print "%nprocshared='"$nprocshared"'\n" if ( $. eq 1 ); print' $inputfile
# 2006/07/26 BKMRK_LINE01
perl -i -pe 's/%nproclinda=/%nprocshared='"$nprocshared"'\n%nproclinda=/i' $inputfile
fi
else
# bkmrk200608091745
perl -i -pe 's/^\s*\%nproc\w*\s*=/\%nproclinda=/i ;' $inputfile
fi
#### 2006/10/11 skk - we think the following line creates a problem for tungsten at least
#### 2006/10/11 replace the line %nproclinda=1 (if it exists) with NOTHING
#### 2006/10/11 perl -i -pe 's/^\s*\%nproclinda=\s*1\s*$//i ' $inputfile
else
perl -i -pe 's/^\s*\%nproc\w*\s*=/\%nprocshared=/i' $inputfile
fi
echo "DEBUG: *now* the input file has these gaussian %nproc directives in it"
grep "%nproc" $inputfile
fi
if [ $clustername = "gordon.sdsc.edu" ]; then
echo "DEBUG: processors request based on input is $cpus "
ppnm=`echo "(($cpus - 1) / 16) + 1" | bc`
LOCAL_PROCS_PER_NODE=`echo "16 * $ppnm" | bc`
fi
# PROCESS OUTPUT FILE
#
Diskoutputdir="$Diskinputdir"
Diskoutputfile_no_dir="$job.out"
# EMAIL OF USER
# if the is EmailOfUser is not specified in commandline, do not send emails out to UNKNOWN
if [ "$EmailOfUser" = "UNKNOWN" ] ; then
#email_addresses="ccg-admin@gridchem.org"
email_addresses="yefan@ncsa.uiuc.edu"
else
email_addresses="$EmailOfUser"
fi
# WALLTIME (walltime)
# for debug queue set the cpu time limit
if [ ${CPU_Time_arg:-null} = "null" ]
# NCSA_SPECIFIC_START 2005/05/17 skk
then
Junk_time=`echo $Queue | grep "debug" `
if [ ${Junk_time:-null} != "null" ]
then
CPU_Time_arg="0:30"
# don't set queuename for these special queues
# Queue="set" - TODO is this needed? 2005/06/14 skk
fi
# NCSA_SPECIFIC_END 2005/05/17 skk
elif [ ${CPU_Time_arg:-null} != "null" ] # 2005/12/20 should the "elif" be just "else" here?
then
# if the CPU_Time is input by user then it can
# be either in minutes or hh:mm format
# first change it to minutes format
# 2005/12/20 skk - This is Just in case for whatever reason we get hh:mm:ss instead of hh:mm - shouldnt really happen
if tmp=`echo $CPU_Time_arg |grep ':.*:'` ; then
echo "WARNING: CPU_Time_arg is $CPU_Time_arg in the form of hh:mm:ss - should be hh:mm - will drop the :ss"
CPU_Time_arg=`echo $tmp|awk -F : '{print $1":"$2}'`
fi
Junk_colon=`echo $CPU_Time_arg | /bin/grep ":" `
if [ ${Junk_colon:-null} != "null" ]
then
echo "DEBUG: CPU_Time_arg=$CPU_Time_arg before making it pure minutes"
CPU_Time_arg=`echo $CPU_Time_arg | sed 's/^/(/' | sed 's/:/\*60)+/' | bc ` #making it pure minutes
echo "DEBUG: CPU_Time_arg=$CPU_Time_arg after making it pure minutes "
# echo " Total CPU Time is (in Minutes) $CPU_Time_arg"
fi
if [ ${CPU_Time_arg:-null} != "null" ]
then
# by now CPU_Time_arg should have an value
# now convert CPU_Time back into hours
CPU_Time_hrs=`echo "$CPU_Time_arg / 60" | bc `
# divide hours by number of cpus for wall clock setting
# CPU_Time_hrs=`echo "$CPU_Time_hrs / $NProc_arg" | bc `
hour=$CPU_Time_hrs
CPU_Time_min=`echo "$CPU_Time_arg % 60" | bc `
minute2=$CPU_Time_min
# (2005/12/20 I think this justification is not right but does not hurt>>) if CPU_Time_hrs=0 then dont use, else bsub comp
lains in tungsten: "00:30:00: Bad RUNLIMIT specification. Job not submitted."
# 2005/12/21 skk if [ ${CPU_Time_hrs:-null} -eq 0 ]; then CPU_Time_arg="$CPU_Time_min" ; else CPU_Time_arg
="$CPU_Time_hrs:$CPU_Time_min"; fi
CPU_Time_arg="$CPU_Time_hrs:$CPU_Time_min"; # 2005/12/22
# 2006/04/14 added by sudhakar
SoftHardInterval=5; # Time in min between soft wall time and hard wall time to be used for cleanup in loadleveler pvs 13Ap
r06
if [ ${CPU_Time_min} -ge 5 ]; then
CPU_Time_minSoft=`echo "$CPU_Time_min - $SoftHardInterval" | bc`
else
if [ ${CPU_Time_hrs} -ge 1 ]; then
CPU_Time_hrs=`echo "$CPU_Time_hrs - 1" | bc `
CPU_Time_minSoft=`echo "$CPU_Time_min + 60 - $SoftHardInterval" | bc`
else
CPU_Time_min=6;
CPU_Time_minSoft=`echo "$CPU_Time_min - $SoftHardInterval" | bc`
fi
fi
CPU_Time_softarg="$CPU_Time_hrs:$CPU_Time_minSoft";
# 2005/11/02 skk - IN LSF the walltime needs HH:MM NOT HH:MM:SS like the rest - TODO: walltime= cputime/cpus ?? bkrmk20051
220
case $local_scheduler in # 2007/06/21 Note: local_scheduler may be a misnomer - local_batch_system might be a better wo
rd (eg moab is the scheduler)
LSF )
walltime="$CPU_Time_arg"
;;
*)
walltime="$CPU_Time_arg:0"
walltimeSoft="$CPU_Time_softarg:0" # 2006/04/14 this is used by loadleveler only
;;
esac
#### 2005/12/21 CPU_Time_LL=" -W $CPU_Time_arg "
fi
fi
# ACCOUNT
# if user wants to use different account/project
if [ ${Account:-null} != "$null" ]
then
# echo "PSN/Account has been set on commandline to $Account "
account=yes
accntin=$Account
else
exit_with_message "we need an Account"
fi
# 2006/02/06 moved the following 6-7 lines from gridchem_qg03 to gridchem_common since it is something that all applics could
benefit from and because gridchem_qg03 eventually might be absorbed into here
#This is for 'debug' queue
# 2008/09/24 skk - added the '&& [ -n "$QUEUE_DEBUG" ]' so that we dont need to define QUEUE_DEBUG
if [ $hour -eq 0 ] && [ $minute2 -le 30 ] && [ $cpus -le 4 ] && [ -n "$QUEUE_DEBUG" ];
then
Queue=$QUEUE_DEBUG
echo "DEBUG: since walltime less or equal to 30min and less or equal to 4 cpus: this job will be submitted to the debug que
ue: $Queue"; echo
fi
# for other queues the cpu time limit is not available
# TODO+ 2005/06/28 This should be last right before start writing the scheduler script
# CREATE THE SCHEDULER SPECIFIC SEQUENCE
# (for TACC)
# this will possibly change the local_preamble_part1 depending on MemSize NumProc
###echo "$LL_LOCAL_OPTIONS_TO_EVAL"
eval "$LL_LOCAL_OPTIONS_TO_EVAL"
###echo "$local_preamble_part1" ; exit
# this is needed to expand the variables in the string
eval "local_preamble_part1=\"$local_preamble_part1\""
echo "DEBUG: local_preamble_part1= $local_preamble_part1"
tmp_email_string=""; # init
case $local_scheduler in
SLURM )
cpuswr="$cpus"
Nodes="$cpus"
# Step 1 - Variables for running LAMMPS MD
PWD=`pwd`
export MPI1=`echo "$Nodes*16" | bc `
# Step 2 - Variables for running LAMMPS diffraction
export Diffraction_Base=DiffFromDumpLoop.in
export Find_Suffix1=dump
export Input=tmp_Input
export OpenMP=16
export MPI2=$Nodes
export MIC=240
# Step 3 - Variables for running VisIt
export MPI3=`echo "$Nodes*16" | bc `
export Find_Suffix2=vtk
# MVM: changed this
# MVM: render.py is a link to the most recent version
export Vis_Script="/home1/00421/ccguser/apps/ds_lammps/bin/render.py"
# MVM: changed this
# MVM: these are here because of CCG's file upload limits
export Vis_Session="/home1/00421/ccguser/apps/ds_lammps/data/*session"
# Step 1- LAMMPS MD simulation
# Step 2- Create unique LAMMPS jobs that run diffraction on each *.dump file
# Step 3- Multiple VisIt sessions creating images from *vtk files
SCHEDULER_PREAMBLE="\
#!/bin/sh
#SBATCH -J ${job}
#SBATCH -p $Queue
#SBATCH -o ${job}.o%j
##SBATCH -n \$cpuswr
#SBATCH -N $Nodes -n $MPI1
#SBATCH -t \$walltime
#SBATCH --mail-user=$email_addresses
#SBATCH --mail-type=ALL
#SBATCH -A TG-CHE080068N"
JOBID="\`echo \$SLURM_JOBID |sed 's/\..*//' \`"
;;
PBS )
# this is the common scheduler script instructions
tmp=""; [ -n "$PBS_V_FLAG" ] && tmp="
#PBS $PBS_V_FLAG"
[ -n "$email_addresses" ] && tmp_email_string="
#PBS -M $email_addresses"
# 2006/06/05 if g03l: we dont want a queue in osc itanium especially so that vmem will be used correctly (if over 4GB seri
al queue fails)
tmp3="
#PBS -q $Queue"
if [ "$PROGRAM_TO_RUN" = "g03l" -a "$clustername" = "tg-login1.osc.edu" ] ; then
tmp3=""
fi
# 2006/04/12 "-m" arguments: "a" = mail is sent when the job is aborted by the batch system. "b"=mail is sent when the jo
b begins execution. "e"=mail is sent when the job terminates.
### 2007/05/18 lines moved further down due to need for local_no_nnodes_allowed to be changed later
# Gordon requies resource vsmp added to the cpus 2012sep 16 pvs.
if [ "$clustername" = "gordon.sdsc.edu" ]; then
cpuswr="$LOCAL_PROCS_PER_NODE:vsmp"
else
cpuswr="$cpus"
fi
SCHEDULER_PREAMBLE="\
#!/bin/sh
#PBS -m bae
#PBS -N ${job} \
$tmp3 \
$tmp_email_string
$tmp
#PBS -j oe \
#PBS -l walltime=\$walltime,nodes=1:ppn=\$cpuswr"
#PBS -l mem=\${MemSize}mb"
if [ $accntin != "default" ] ; then
DIRECTIVE_ACCOUNT="#PBS -A \$accntin" #scheduler directive for account
fi
if [ "$clustername" = "gordon.sdsc.edu" -a "$Queue" = "vsmp" ]; then
DIRECTIVE_ACCOUNT="$DIRECTIVE_ACCOUNT
export LD_PRELOAD=/opt/ScaleMP/libvsmpclib/0.1/lib64/libvsmpclib.so
export LD_PRELOAD=/usr/lib/libhoard.so:$LD_PRELOAD
export PATH=/opt/ScaleMP/numabind/bin:$PATH
export OMP_NUM_THREADS=\$LOCAL_PROCS_PER_NODE
"
fi
NUMABIND="\`numabind --offset=$LOCAL_PROCS_PER_NODE\`"
NODELIST_SETUP="NODELIST=\`cat \$PBS_NODEFILE|perl -pe 's/\\\n/ /' \` "
JOBID="\`echo \$PBS_JOBID |sed 's/\..*//' \`" # i want just the number (eg 221112) not the full JOBID (eg 221112.agt-s001
.epn.osc.edu)
WORKDIR="\$PBS_O_WORKDIR" # 2006/03/31 skk this is where we executed the qsub command ie the TEMPDIR1 ??- is it needed?
why not use TEMPDIR1? todo--
;;
LSF )
# -B sends job at begin, -N= sends report by email etc
[ -n "$email_addresses" ] && tmp_email_string="
#BSUB -u $email_addresses"
# 2006/04/09 todo - why arent these lines below in the local part for tungsten??
# 2005/07/21 LSF complains (at least in tungtem if you ask for the debug queue and in addition you give a walltime - so re
move it
tmp_walltime_string="
#BSUB -W \$walltime"
# 2005/07/21 the "if hostname ..." line below may need to be broadened for other LSF clusters with the same behavior?
if hostname|egrep '^tun' >/dev/null ; then # TODO- '^tun; could characterize other hostnames in the future
if [ "$Queue" = "$QUEUE_DEBUG" ]; then
echo "INFO: Removing the walltime directive since we use the debug queue (tungsten)"; echo ""
tmp_walltime_string=""
fi
fi
SCHEDULER_PREAMBLE="\
#!/bin/sh
#BSUB -n $cpus \
$TUNGSTEN_SPAN_PTILE_1 \
$tmp_walltime_string
#BSUB -o ${job}.o
#BSUB -J ${job} \
$tmp_email_string
#BSUB -N
#BSUB -B
#BSUB -q $Queue"
DIRECTIVE_ACCOUNT="#BSUB -P \$accntin" #scheduler directive for account
NODELIST_SETUP="NODELIST=\$LSB_HOSTS"
### in LSF we have eg LSB_JOBID=454672
JOBID="\$LSB_JOBID"
WORKDIR="\$LS_SUBCWD" ### LS_SUBCWD is PBS_O_WORKDIR
# LOADLEVELER
;;
loadleveler )
# 2005/06/13 this is copied from champ gamess script
[ -n "$email_addresses" ] && tmp_email_string="
#@ notify_user = $email_addresses"
if [ $DOMAIN = "iu_bigred" ]; then
LL_LOCAL_OPTIONS=""
BigRed_LOCAL_OPTION='#@ node = $nnodes
#@ tasks_per_node = $LOCAL_PROCS_PER_NODE '
else
LL_LOCAL_OPTIONS="#@ resources = ConsumableCpus($LOCAL_LL_CONSUMABLE_CPUS) ConsumableMemory(\${MemSize}mb)"
fi
# 2005/12/15 seems the "#@ class = normal \" WAS NOT COORECT - should be "#@ class = $Queue \"
# 2007/03/01 #@ error = $Diskinputfile.err was made into #@ error = $Diskinputfile.log for better output
SCHEDULER_PREAMBLE="\
$LL_SHELL_TO_USE
#@ shell = /bin/sh
######################################################
#####2006/03/31 #@ initialdir = $Currdir
#@ initialdir = $TEMPDIR1
$LL_LOCAL_OPTIONS
$BigRed_LOCAL_OPTION
$ADF_LOCAL_CU_OPTION
#@ notification = always \
$tmp_email_string
#@ job_type = parallel \
$NWCHEM_LAPI_NETWORK
#@ output = $Diskinputfile.log
#@ error = $Diskinputfile.log
#@ environment = COPY_ALL; MP_SHARED_MEMORY=yes
#@ class = $Queue \
$local_preamble_part1
#@ account_no = $accntin
#@ queue "
#### 2005/12/22 DIRECTIVE_ACCOUNT=""
# 2005/12/22 TODO: this is still problematic because it should NOT come after #@ queue - but it is ok to not have the acco
unt (according to Sudhakar)
# 2005/12/22 DIRECTIVE_ACCOUNT="#@ account_no = \$accntin"
# 2005/12/22 FINALLY I DECIDED TO ADD THE #@ account_no = $accntin - directly into PREAMBLE since we ALWAYS HAVE Account n
ow available
NODELIST_SETUP="" # TODO
JOBID="\`echo \$LOADL_STEP_ID | awk -F. '{print \$(NF-1)}'\`" # $LOADL_STEP_ID is eg champ.tacc.utexas.edu.179693.0
#2005/10/21 JOBSTEP_NUM="\`echo \$LOADL_STEP_ID | awk -F. '{print \$(NF)}'\`" # 2005/10/14 not so important
WORKDIR="\$LOADL_STEP_INITDIR"
;;
*) exit_with_message "local_scheduler is $local_scheduler"
;;
esac
# create a unique qsub file
echo "INFO: cd'ing into $TEMPDIR1 "; echo
eval "cd $TEMPDIR1" # go into $TEMPDIR1
qsubin="$TEMPDIR1/qsubin$$"
test -f "$qsubin" && /bin/rm $qsubin
touch $qsubin
chmod 700 $qsubin
# 2006/04/07
qsubin_cleanup="$TEMPDIR1/qsubin$$_cleanup" # this will do the cleanup in case of expired walltime
test -f "$qsubin_cleanup" && /bin/rm $qsubin_cleanup
touch $qsubin_cleanup
chmod 700 $qsubin_cleanup
#uberftp qsub
# 2008/06/23
qsubin_uberftp1="$TEMPDIR1/qsubin$$_uberftp1" #
test -f "$qsubin_uberftp1" && /bin/rm $qsubin_uberftp1
touch $qsubin_uberftp1
chmod 700 $qsubin_uberftp1
qsubin_uberftp2="$TEMPDIR1/qsubin$$_uberftp2" #
test -f "$qsubin_uberftp2" && /bin/rm $qsubin_uberftp2
touch $qsubin_uberftp2
chmod 700 $qsubin_uberftp2
qsubin_uberftp_nodir=qsubin$$_uberftp
qsubin_uberftp="$TEMPDIR1/$qsubin_uberftp_nodir" #
test -f "$qsubin_uberftp" && /bin/rm $qsubin_uberftp
touch $qsubin_uberftp
chmod 700 $qsubin_uberftp
date_YYMMDD=$(date +%y%m%d)
# 2006/04/06 this is for gaussian for now
uberftp_command_formated_chk="" # default is empty
uberftp_command_chk="" # default is empty
architecture1=`$local_uname_arch_command`
if [ $? != 0 ] ; then
echo "WARNING: command failed: $local_uname_arch_command"
architecture1="unknown"
fi
architecture2=`uname` # will give linux or HP-UX
if [ $? != 0 ] ; then
echo "WARNING: command failed: uname"
architecture2="unknown"
fi
# 2006/07/19 unfortunately the same arch eg ia64 may still be binary incompatible if linux vs hpux so let us use the
architecture=${architecture1}_$architecture2
gridchem_g03l (){
# PROCESS CHECKPOINT FILE
# Check to see if the checkpoint file is given a name in input deck
# Input file to look into
dummy="$Diskinputdir/$Diskinputfile"
checkpt="no"
junk=`/usr/bin/head -5 $dummy | /bin/grep -i "%chk"`
if [ ${junk:-null} != $null ]; then
junk=`echo $junk | /bin/sed 's/=/@/'`
junk=`echo $junk | /bin/sed 's/ //'`
#
# Be careful: Don't lose the subdirectory information for CHKPT file
# Also, add .chk if there is no extension to the Checkpoint file
#
Chkfile=`expr $junk : '.*@\(.*\)'`
Chkdir="$Diskinputdir"
Chkfile=`/bin/basename $Chkfile`
dummy=`expr $Chkfile : '.*\(\..*\)'`
Chkfile=`/bin/basename $Chkfile $dummy`
ChkfileWNE="$Chkfile"
Chkfile="$Chkfile${dummy:-.chk}"
# 2005/12/08 create name for $formated_chkfile
formated_chkfile="$ChkfileWNE.fchk"
Chkfile_with_arch="${Chkfile}_$architecture"
echo "DEBUG: formated_chkfile = $formated_chkfile ";
checkpt="yes"
fi
# PROCESS READWRITE FILE
# Check to see if the read-write file is given a name in input deck
# Input file to look into
dummy="$Diskinputdir/$Diskinputfile"
junk=`/usr/bin/head -5 $dummy | /bin/grep -i "%rwf"`
if [ ${junk:-null} != $null ]; then
junk=`echo $junk | /bin/sed 's/=/@/'`
junk=`echo $junk | /bin/sed 's/ //'`
# Be careful: Don't lose the subdirectory information for RWF file
# Also, add .rwf if there is no extension to the rwf file
Rwffile=`expr $junk : '.*@\(.*\)'`
Chkdir="$Diskinputdir"
Rwffile=`/usr/bin/basename $Rwffile`
dummy=`expr $Rwffile : '.*\(\..*\)'`
Rwffile=`/usr/bin/basename $Rwffile $dummy`
RwffileWNE="$Rwffile"
Rwffile="$Rwffile${dummy:-.rwf}"
fi
# PROCESS SCRATCH FILE
# Check to see if the scratch file is given a name in input deck
# Input file to look into
dummy="$Diskinputdir/$Diskinputfile"
junk=`/usr/bin/head -5 $dummy | /bin/grep -i "%scr"`
if [ ${junk:-null} != $null ]; then
junk=`echo $junk | /bin/sed 's/=/@/'`
junk=`echo $junk | /bin/sed 's/ //'`
# Be careful: Don't lose the subdirectory information for SCR file
# Also, add .scr if there is no extension to the scr file
Scrfile=`expr $junk : '.*@\(.*\)'`
Chkdir="$Diskinputdir"
Scrfile=`/usr/bin/basename $Scrfile`
dummy=`expr $Scrfile : '.*\(\..*\)'`
Scrfile=`/usr/bin/basename $Scrfile $dummy`
ScrfileWNE="$Scrfile"
Scrfile="$Scrfile${dummy:-.scr}"
fi
# MEMORY SWAPMEM
# Calculate Peak Swap Memory from %mem data
# First one has to convert them to bytes from words
dummy="$Diskinputdir/$Diskinputfile"
junk_M=`/bin/grep -i "%mem" $dummy | /usr/bin/awk -F= '{print $2}' | /usr/bin/tr "[a-z]" "[A-Z]" | /usr/bin/tr -d '\r'`
# check if two setting are given and take the larger
junk_M=`echo $junk_M | /usr/bin/awk -F, '{if ($2 > $1 )print $2; else print $1}' | /usr/bin/tr "[a-z]" "[A-Z]" `
# convert memory into bytes - default is Words
junk=$null
for i_mem in $junk_M ; do
junk=`echo $i_mem | sed 's/[0-9]//g'`
if [ ${junk:-null} = "null" ]; then
junk="W"
fi
case $junk in
W) i_mem=`echo "$i_mem*8" | bc `;;
KW) i_mem=`echo $i_mem | sed 's/KW//'`
i_mem=`echo "$i_mem*8000" | bc `;;
MW) i_mem=`echo $i_mem | sed 's/MW//'`
i_mem=`echo "$i_mem*8000000" | bc `;;
GW) i_mem=`echo $i_mem | sed 's/GW//'`
i_mem=`echo "$i_mem*8000000000" | bc `;;
KB) i_mem=`echo $i_mem | sed 's/KB//'`
i_mem=`echo "$i_mem*1000" | bc `;;
MB) i_mem=`echo $i_mem | sed 's/MB//'`
i_mem=`echo "$i_mem*1000000" | bc `;;
GB) i_mem=`echo $i_mem | sed 's/GB//'`
i_mem=`echo "$i_mem*1000000000" | bc `;;
esac
JUNK_MEMORY="$JUNK_MEMORY $i_mem"
done
junk_M=`echo $JUNK_MEMORY | /usr/bin/awk 'BEGIN { max = -1 } { for (i = 1; i <=NF; i++) \
{if ( $i > max) max = $i}} END {print max} '`
if [ $junk_M != "-1" ]; then
G98_mem=$junk_M
echo "Swap Memory Requested is $G98_mem B from %MEM line in the input"
fi
G98_memM=`echo "($G98_mem/1000000)" | bc `
# 2006/02/24 (based on sudhakar's notes - this is for copper to add some administrative memory to the job)
if [ $G98_memM -gt 4000 ]; then
G98_memM=`echo "($G98_memM + 600)" | bc `
elif [ $G98_memM -gt 2000 ]; then
G98_memM=`echo "($G98_memM + 400)" | bc `
elif [ $G98_memM -gt 1000 ]; then
G98_memM=`echo "($G98_memM + 200)" | bc `
elif [ $G98_memM -gt 500 ]; then
G98_memM=`echo "($G98_memM + 500)" | bc `
elif [ $G98_memM -le 500 ]; then
G98_memM=`echo "($G98_memM + 550)" | bc `
fi
SwapMem_arg=$G98_memM
MemSize=$SwapMem_arg # 2005/06/23 this will be in MB from now on - skk
# 2007/05/18 skk This is a TEMP? solution for OSC where
# - altix does not have PVFS (but has very large TMPDIR so we should use it if altix)
# - a job goes to altix if you use the ncpus instead of nodes= or if the vmem>12000MB
# Note: this would belong to local but we dont know the MemSize during local call
# Note: the setup unfortunately has to come after MemSize and cpus is set and BEFORE SCHEDULER_PREAMBLE is eval below becaus
e SCHEDULER_PREAMBLE contains the LOCAL_APPEND_TO_PBS_RESOURCES_LINE variable.
LOCAL_APPEND_TO_PBS_RESOURCES_LINE=""
if [ "$clustername" = "tg-login1.osc.edu" ] ; then
local_osc_will_use_altix=0;
## 2008/10/31 skk - we dont have the 2-core 12GB nodes anymore so let us use altix if >4GB - if [ "$MemSize" -gt 12000 ] ;
then local_osc_will_use_altix=1; fi
if [ "$MemSize" -gt 4000 ] ; then local_osc_will_use_altix=1; fi
if [ $PROGRAM_TO_RUN = "g03l" -a "$cpus" -gt 2 ]; then local_osc_will_use_altix=1; fi
if [ $local_osc_will_use_altix = 1 ] ; then
echo "INFO: going to altix nodes - so let us not use pvfs - and use ncpus instead"
local_no_nnodes_allowed="YES"
if [ "$cpus" -gt $OSC_MAX_NUM_ALTIX_CPUS ]; then exit_with_message "AT OSC YOU CAN NOT HAVE NUMBER OF ALTIX CPUS = $cpu
s - max altix cpus should be $OSC_MAX_NUM_ALTIX_CPUS "; fi
else
echo "INFO: going to non-altix queue - so let us use pvfs"
LOCAL_APPEND_TO_PBS_RESOURCES_LINE=":pvfs"
fi
fi
if [ "$clustername" = "ember.ncsa.illinois.edu" ] ; then
# memory on ember could be set to max available on cpuset * 5000MB
# cpuset is MemSize/6000MB in integer units of nodeboard cpus which is 6 * 5000 MB
if [ $MemSize -gt 30000 ]; then
emb_cpuset=`echo "($MemSize/30000)" |bc`
# compare this with processor based cpuset and assign the max of the two.
emb_p_cpuset=`echo "($cpus-1)/6 + 1" | bc`
if [ $emb_p_cpuset -gt $emb_cpuset ]; then emb_cpuset=$emb_p_cpuset; fi
MemSize=`echo "$emb_cpuset * 30000" | bc`
fi
fi
###2007/05/18 preamble_nodes_and_ppn="nodes=$nnodes:ppn=$LOCAL_PROCS_PER_NODE"
preamble_nodes_and_ppn="nodes=$nnodes:ppn=$LOCAL_PROCS_PER_NODE$LOCAL_APPEND_TO_PBS_RESOURCES_LINE"
###2007/05/18 if [ -n "$local_no_nnodes_allowed" ] ; then preamble_nodes_and_ppn="ncpus=$cpus" ; fi
if [ -n "$local_no_nnodes_allowed" ] ; then preamble_nodes_and_ppn="ncpus=$cpus$LOCAL_APPEND_TO_PBS_RESOURCES_LINE" ; fi
# 2005/06/01 check that the tsnet.config agrees with REMOTE_EXEC otherwise change it and give warning
if [ ! -e ~/.tsnet.config ]; then
echo " Linda remote shell being set to $REMOTE_EXEC "
echo ""
echo "Tsnet.Node.lindarsharg: $REMOTE_EXEC" > ~/.tsnet.config
elif grep -q $REMOTE_EXEC ~/.tsnet.config ; then
echo "We will use already existing ~/.tsnet.config with contents:"
cat ~/.tsnet.config
echo ""
else
echo "WARNING: Linda remote shell being set to $REMOTE_EXEC "
echo "Before we changed it, it was:"
cat ~/.tsnet.config
echo ""
echo "Tsnet.Node.lindarsharg: $REMOTE_EXEC" > ~/.tsnet.config
fi
# STEP 4: WRITE QSUB FILE
eval "echo \"$SCHEDULER_PREAMBLE\" " >> $qsubin
if [ ${account:-null} = "yes" ]; then
eval "echo \"$DIRECTIVE_ACCOUNT\" " >> $qsubin
fi
} # END gridchem_g03l
# JOB SPECIFIC STAFF
if [ $PROGRAM_TO_RUN = "g03l" -o $PROGRAM_TO_RUN = "g09" ] ; then
echo "INFO: calling gridchem_g03l function to setup some g03l specific stuff" ; echo
gridchem_g03l
else
# MEMORY # TODO++ move into common -- as well as the one from g03
# Peak Swap Memory in Mbytes
MemSize=$SwapMem_arg # 2005/06/23 skk - added
# 2007/05/18 skk This is a TEMP? solution for OSC where
# - altix does not have PVFS (but has very large TMPDIR so we should use it if altix)
# - a job goes to altix if you use the ncpus instead of nodes= or if the vmem>12000MB
# Note: this would belong to local but we dont know the MemSize during local call
# Note: the setup unfortunately has to come after MemSize and cpus is set and BEFORE SCHEDULER_PREAMBLE is eval below becaus
e SCHEDULER_PREAMBLE contains the LOCAL_APPEND_TO_PBS_RESOURCES_LINE variable.
LOCAL_APPEND_TO_PBS_RESOURCES_LINE=""
if [ "$clustername" = "tg-login1.osc.edu" ] ; then
local_osc_will_use_altix=0;
if [ "$MemSize" -gt 12000 ] ; then local_osc_will_use_altix=1; fi
if [ $PROGRAM_TO_RUN = "g03l" -a "$cpus" -gt 2 ]; then local_osc_will_use_altix=1; fi
if [ $local_osc_will_use_altix = 1 ] ; then
echo "INFO: going to altix queue - so let us not use pvfs - and use ncpus instead"
local_no_nnodes_allowed="YES"
else
echo "INFO: going to non-altix queue - so let us use pvfs"
LOCAL_APPEND_TO_PBS_RESOURCES_LINE=":pvfs"
fi
fi
###2007/05/18 preamble_nodes_and_ppn="nodes=$nnodes:ppn=$LOCAL_PROCS_PER_NODE"
preamble_nodes_and_ppn="nodes=$nnodes:ppn=$LOCAL_PROCS_PER_NODE$LOCAL_APPEND_TO_PBS_RESOURCES_LINE"
###2007/05/18 if [ -n "$local_no_nnodes_allowed" ] ; then preamble_nodes_and_ppn="ncpus=$cpus" ; fi
if [ -n "$local_no_nnodes_allowed" ] ; then preamble_nodes_and_ppn="ncpus=$cpus$LOCAL_APPEND_TO_PBS_RESOURCES_LINE" ; fi
if [ $PROGRAM_TO_RUN = "nwchem" ] ; then
# TEMP 2005/06/28 since stupid MemSize is defined very late? (it belongs to local)
if [ $DOMAIN = "tacc" ]; then
if [ "$MemSize" -gt 91840 -a "$cpus" -gt 11320 ]; then
exit_with_message "Can not run job with n>32 and memory/task > 1840mb"
fi
fi
elif [ $PROGRAM_TO_RUN = "molpro" ] ; then
# TEMP 2005/06/28 since stupid MemSize is defined very late? (it belongs to local)
if [ $clustername = "ccg-login.ncsa.uiuc.edu" ]; then
#### 2005/12/28 if [ $DOMAIN = "ncsa" ]; then
if [ "$MemSize" -gt 1840 -a "$cpus" -gt 12 ]; then
exit_with_message "Can not run job with n>12 and memory/task > 1840mb"
fi
fi
elif [ $PROGRAM_TO_RUN = "qmcpack" ] ; then
# TEMP 2005/06/28 since stupid MemSize is defined very late? (it belongs to local)
if [ $clustername = "tun.ncsa.uiuc.edu" ]; then
#### 2005/12/28 if [ $DOMAIN = "ncsa" ]; then
if [ "$MemSize" -gt 1840 -a "$cpus" -gt 1256 ]; then
exit_with_message "Can not run job with n>1256 and memory/task > 1840mb"
fi
fi
elif [ $PROGRAM_TO_RUN = "abaqus" ] ; then
MemSize=30000
fi
DDI_RSH="$REMOTE_EXEC" # for GAMESSS -- TODO is this needed? 2005/05/31
# STEP 4: WRITE QSUB FILE
#
# First block of qsubin
eval "echo \"$SCHEDULER_PREAMBLE\" " >> $qsubin
if [ ${account:-null} = "yes" ]; then
eval "echo \"$DIRECTIVE_ACCOUNT\" " >> $qsubin
fi
if [ $PROGRAM_TO_RUN = "nwchem" ] ; then
echo "$NWCHEM_LOCAL_VARS" >> $qsubin
fi
fi # if NOT gaussian g03l
if [ "$clustername" = "gordon.sdsc.edu" -a "$Queue" = "vsmp" ]; then
/bin/cat >> $qsubin <<EOF
NUMABIND=$NUMABIND
export KMP_AFFINITY=compact,verbose,0,\$NUMABIND
EOF
fi
echo "set -x" >> $qsubin
# 2006/08/25 skk - TEMP - let us have a verbose scheduler script
###echo "set -x" >> $qsubin
### 2007/12/12 moving some osc stuff further down AFTER the load of the g03 module which overwrites and ....
# 2006/04/27 Moved the local_module_env_command HERE otherwise unfchk is not found
[ -n "$local_module_env_command" ] && echo "$local_module_env_command" >> $qsubin
# we need this before we do the OutDir which has embedded the jobdir
# 2005/10/25 moved here from below since they are needed by uberftp command in chkpoint
/bin/cat >> $qsubin << EOF1a
export X509_USER_PROXY=$X509_USER_PROXY
###2007/05/03 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH # NOT NEEDED
jobid=$JOBID
jobdir="$job.$clustername.\$jobid.$date_YYMMDD"
export OutDir=$OutDir
#if [ ! -d \$OutDir ] ; then
# echo "Making dir: \$OutDir";
# mkdir -p \$OutDir || exit 1; # let us stop if we can not make this dir - it is too important
#fi
# we need to export SCRATCH_BATCH
EOF1a
### 2007/12/12 moved the osc related SCRATCH_BATCH down here since ....
# 2007/05/18 I need to have these evaluated in the batch environment (the variable PFSDIR and TMPDIR exist only in batch env)
if [ "$clustername" = "tg-login1.osc.edu" ] ; then
/bin/cat >> $qsubin <<EOF
if [ -n "\$PFSDIR" ] ; then
export SCRATCH_BATCH="\$PFSDIR";
elif [ -n "\$TMPDIR" ] ; then
export SCRATCH_BATCH="\$TMPDIR";
else
echo "ERROR: no PFSDIR and no TMPDIR var"
exit 1
fi
EOF
else
/bin/cat >> $qsubin <<EOF1a
export HOME_BATCH=$OutDir
export SCRATCH_BATCH=$SCRATCH_BATCH
EOF1a
fi
/bin/cat >> $qsubin <<EOF1a
if [ ! -d \$SCRATCH_BATCH ] ; then
mkdir -p \$SCRATCH_BATCH
fi
EOF1a
# 2007/12/12 From here and further down i NEED TO BE USING \$SCRATCH_BATCH NOT $SCRATCH_BATCH due to OSC ....
/bin/cat >> $qsubin <<EOF1a
if [ ! -d \$HOME_BATCH ] ; then
echo "Linking HOME_BATCH to SCRATCH_BATH: \$HOME_BATCH -> \$SCRATCH_BATCH" ;
ln -s \$SCRATCH_BATCH \$HOME_BATCH || exit 1;
fi
# change directory to SCRATCH_BATCH
cd \$SCRATCH_BATCH
pwd
# 2006/03/31 make a link of the Outdir into OutDir for convenience
#/bin/ln -s $TEMPDIR1 \$OutDir/
###which uberftp
###echo "DEBUG: TEMP01AAAAAA 20070301 skk "; ls -l \$X509_USER_PROXY
# 20140115:MVM - not sure where the best place to do this is
# need to copy files from ds_lammps/data to working directory to avoid having
# to upload them through Gridchem
# also not sure if there's a variable containig this path
#if [ $PROGRAM_TO_RUN = 'lammps' -o $PROGRAM_TO_RUN = "lammps_ds" ] ; then
# cp $HOME/apps/ds_lammps/data/* .
#fi
EOF1a
# 2006/04/20 moved some stuff around to remove duplication of code
if [ $PROGRAM_TO_RUN = "g03l" -o $PROGRAM_TO_RUN = "g09" ] ; then
# set up some variables needed by gaussian
echo "g03lscr=\$SCRATCH_BATCH" >> $qsubin
# Point to the qsubin, the proper input file
echo "/bin/cp $Diskinputdir/$Diskinputfile input.com" >> $qsubin
if [ ${Rwffile:-null} != "$null" ]; then
echo "/bin/cp $Diskinputdir/$Rwffile $Rwffile" >> $qsubin
fi
if [ ${Scrfile:-null} != "$null" ]; then
echo "/bin/cp $Diskinputdir/$Scrfile $Scrfile" >> $qsubin
fi
# Point to the qsubin, the proper checkpoint file
if [ ${Chkfile:-null} != "$null" ]; then
echo "if [ -f \"$Diskinputdir/$Chkfile\" ]; then " >> $qsubin
echo " /bin/cp $Diskinputdir/$Chkfile $Chkfile" >> $qsubin
echo "else " >> $qsubin
if [ -n "$DO_UBERFTP_ON_LOGINNODE" ] ; then
/bin/cat >> $qsubin <<EOF
ChkTest_arch=\`$gsissh 'cd mss/$topdir_name/$GridchemUserid; ls $Chkfile_with_arch' \" | /bin/grep -i $Chkfile_with_arch
| awk ' { print \$NF } ' | sed 's/\\r//' \`
ChkTest=\`$gsissh 'cd mss/$topdir_name/$GridchemUserid; ls $formated_chkfile' \" | /bin/grep -i $formated_chkfile | awk '
{ print \$NF } ' |sed 's/\\r//' \`
EOF
else
/bin/cat >> $qsubin <<EOF
ChkTest_arch=\`$gsissh "cd mss/$topdir_name/$GridchemUserid; ls $Chkfile_with_arch" | /bin/grep -i $Chkfile_with_arch
| awk ' { print \$NF } ' | sed 's/\\r//' \`
ChkTest=\`$gsissh "cd mss/$topdir_name/$GridchemUserid; ls $formated_chkfile" | /bin/grep -i $formated_chkfile | awk '
{ print \$NF } ' | sed 's/\\r//' \`
EOF
fi
echo " echo \"DEBUG: ChkTest_arch= \$ChkTest_arch\" " >> $qsubin
# if we find the correct arch chk file ...
echo " checkpoint_file_not_found_or_invalid=1 " >> $qsubin
echo " if [ \${ChkTest_arch:-null} = $Chkfile_with_arch ]; then " >> $qsubin
# 2006/05/02 included a "quote site wait ;" to avoid issues with delayed retrieval. I should also let the user know about
the wait BUT HOW?
if [ -n "$DO_UBERFTP_ON_LOGINNODE" ] ; then
/bin/cat >> $qsubin <<EOF
$gsiscp $mss_machine:mss/$topdir_name/$GridchemUserid/$Chkfile_with_arch ./
EOF
else
/bin/cat >> $qsubin <<EOF
$gsiscp $mss_machine:mss/$topdir_name/$GridchemUserid/$Chkfile_with_arch ./
EOF
fi
# 2006/05/05 Save the file back to mss with a .old ending just in case a gaussian run corrupts the file later on (suggesti
on by Sudhakar)
if [ -n "$DO_UBERFTP_ON_LOGINNODE" ] ; then
/bin/cat >> $qsubin <<EOF
$uberftp 'cd $topdir_name/$GridchemUserid; lcd \$SCRATCH_BATCH ; put $Chkfile_with_arch $Chkfile_with_arch.old' \"
EOF
else
/bin/cat >> $qsubin <<EOF
$gsissh "cd mss/$topdir_name/$GridchemUserid; mv $Chkfile_with_arch $Chkfile_with_arch.old"
EOF
fi
# 2005/12/08 now transform formated checkpoint file fchk into unformated chk
echo " mv $Chkfile_with_arch $Chkfile " >> $qsubin
# Define the gaussian utility environments
# Jayeeta commented following
# if [ "$clustername" = "co.ncsa.uiuc.edu" ]; then
# if [ $PROGRAM_TO_RUN = "g03l" ] ; then
# echo "soft add +gaussian-03" >>$qsubin
# elif [ $PROGRAM_TO_RUN = "g09" ] ; then
# echo "soft add +gaussian-09" >> $qsubin
# fi
# fi
# Jayeeta added following
if [ "$clustername" = "ember.ncsa.uiuc.edu" ]; then
if [ $PROGRAM_TO_RUN = "g03l" ] ; then
echo "module load g09_b01" >>$qsubin
elif [ $PROGRAM_TO_RUN = "g09" ] ; then
echo "module load g09" >> $qsubin
fi
fi
echo " if chkchk $Chkfile 2>/dev/null 1>/dev/null ; then checkpoint_file_not_found_or_invalid=0 ; else echo \"WARNING:
chkchk command failed - we will use the formatted checkpoint if it exists \" ;fi " >> $qsubin
echo " fi " >> $qsubin
echo " if [ \$checkpoint_file_not_found_or_invalid -eq 1 ] ; then " >> $qsubin
# 2006/04/06 If we did not find the correct arch chk file OR it failed the chkchk command - let us try for the fchk
echo " echo \"DEBUG: ChkTest= \$ChkTest\" " >> $qsubin
echo " if [ \${ChkTest:-null} = $formated_chkfile ]; then " >> $qsubin
if [ -n "$DO_UBERFTP_ON_LOGINNODE" ] ; then
/bin/cat >> $qsubin <<EOF
$uberftp 'quote site wait; cd $topdir_name/$GridchemUserid; lcd \$SCRATCH_BATCH ; get $formated_chkfile' \"
EOF
else
/bin/cat >> $qsubin <<EOF
$gsiscp $mss_machine:mss/$topdir_name/$GridchemUserid/$formated_chkfile ./
EOF
fi
# 2005/12/08 now transform formated checkpoint file fchk into unformated chk
echo " unfchk $formated_chkfile $Chkfile " >> $qsubin
echo " else " >> $qsubin
echo " echo \"WARNING: no formated checkpoint file exists on MSS \" " >> $qsubin
echo " fi " >> $qsubin
echo " fi " >> $qsubin
echo "fi" >> $qsubin
fi
# 2006/04/27 Moved the local_module_env_command further up since otherwise unfchk is not found
/bin/cat >> $qsubin << EOF2
###echo "DEBUG: TEMP01BBBB 20070301 skk "; ls -l \$X509_USER_PROXY
# (for OSC) THIS HAS TO COME AFTER THE module command since module would overwrite with default GAUSS_SCRDIR
export GAUSS_SCRDIR="\$SCRATCH_BATCH"
EOF2
#echo "THIS IS THE ENVIRONMENT RIGHT AFTER THE: export GAUSS_SCRDIR ...:" >> $qsubin # 2006/08/16 13:34 TEMP
#echo "env" >> $qsubin
# 2006/06/09 todo: if no linda we dont need the two lines NODELIST_SETUP and the GAUSS_LFLAGS line
echo "$NODELIST_SETUP" >> $qsubin
/bin/cat >> $qsubin << EOF2
export GAUSS_LFLAGS="-vv -nodelist \\\"\$NODELIST\\\""
echo "THIS IS THE INPUT FILE:"
cat $Diskinputdir/$Diskinputfile
EOF2
elif [ $PROGRAM_TO_RUN = "amber_sander" ] ; then
# 2008/04/09 skk i moved the line here from below since these are our implicit $inputfile
echo "cp -p $Diskinputdir/mdin ./" >> $qsubin
echo "cp -p $Diskinputdir/*.prmtop ./prmtop" >> $qsubin
echo "cp -p $Diskinputdir/*.inpcrd ./inpcrd" >> $qsubin
# echo "cp -p $TEMPDIR1/mdin $TEMPDIR1/prmtop $TEMPDIR1/inpcrd ." >> $qsubin
elif [ $PROGRAM_TO_RUN = "dmol3" ] ; then
Basename=${Diskinputfile%.*} #Get basename (remove extension)
echo "cp -p $Diskinputdir/$Basename.* ." >> $qsubin
elif [ $PROGRAM_TO_RUN = "castep" ] ; then
Basename=${Diskinputfile%.*} #Get basename (remove extension)
echo "cp -p $Diskinputdir/$Basename.* ." >> $qsubin
elif [ $PROGRAM_TO_RUN = "ddscat-v7_1" -o $PROGRAM_TO_RUN = "ddscat-v7_2" ] ; then
echo "cp -p $Diskinputdir/*.par ./ddscat.par" >> $qsubin
echo "cp -p $Diskinputdir/*.tab ./" >> $qsubin
elif [ $PROGRAM_TO_RUN = "charmm" ] ; then
echo "cp -p $Diskinputdir/*.* ./" >> $qsubin
elif [ $PROGRAM_TO_RUN = "abaqus" ] ; then
echo "cp -p $Diskinputdir/*.* ./" >> $qsubin
elif [ $PROGRAM_TO_RUN = "lammps" ] ; then
echo "cp -p $Diskinputdir/*.* ./" >> $qsubin
elif [ $PROGRAM_TO_RUN = "lammps_ds" ] ; then
echo "cp -p $Diskinputdir/* ." >> $qsubin
echo "cp \$HOME/apps/ds_lammps/data/* ." >>$qsubin
elif [ $PROGRAM_TO_RUN = "tinker_monte" ] ; then
echo "cp -p $Diskinputdir/* ." >> $qsubin
elif [ $PROGRAM_TO_RUN = "dftb+" ] ; then
echo "cp -p $Diskinputdir/* ." >> $qsubin
#echo "cp -p $Diskinputdir/*_in.hsd dftb_in.hsd" >> $qsubin
else
echo "cp -p $inputfile ." >> $qsubin # gaussian does it a little diff: " echo "/bin/cp $Diskinputdir/$Diskinputfile input
.com" >> $qsubin"
fi # if [ $PROGRAM_TO_RUN = "g03l" ]
if [ -n "$OUTPUT_DIR_IS_ACCESSIBLE" ] ; then # 2007/11/30 skk this if statement is always true now that sdx is gone but keep
it just in case
if [ -n "$SCRATCH_BATCH_DIR_IS_ACCESSIBLE" ] ; then
echo "Diskoutputfile_with_dir=\$SCRATCH_BATCH/$Diskoutputfile_no_dir" >> $qsubin
#echo "/bin/ln -sf \$Diskoutputfile_with_dir \$OutDir/$Diskoutputfile_no_dir" >> $qsubin
else
echo "Diskoutputfile_with_dir=\$OutDir/$Diskoutputfile_no_dir" >> $qsubin
fi
else
# 2006/07/14 ADDED THIS CASE: ALLOW CLUSTERS WHERE THE SCRATCH AREA (OUTPUT DIR) IS LOCAL
# if NOT -n $OUTPUT_DIR_IS_ACCESSIBLE then
# create tmpdir1=gridchem_outputs.dir/$jobid on global ~/
# on sdx create the outputdir using mkdir -p
# create the filename for the output to be in this tmpdir1
# on sdx make soft link from the outputdir to this outputfilename with the correct name
# (later on) on sdx just before job is done , remove soft link and copy output file to the same position with the correct na
me (just in case)
/bin/cat >> $qsubin << EOF2aa
tmpdir1="$HOME/gridchem_outputs.dir/\$jobid"
mkdir \$tmpdir1
ssh sdx "mkdir -p \$OutDir "
Diskoutputfile_with_dir=\$tmpdir1/$Diskoutputfile_no_dir
ssh sdx "/bin/ln -sf \$Diskoutputfile_with_dir \$OutDir/$Diskoutputfile_no_dir "
EOF2aa
fi
# 2006/04/06 ADD TRAP FOR SIGNALS: 2=keyboard_interrupt 15=TERM XCPU=24=CPUTime Exceeded CHLD=child stopped or terminated
### 2006/04/19 echo "trap \". $qsubin_cleanup ; exit 99 \" 24 2 15 30 " >> $qsubin
signals_to_trap="XCPU INT TERM CHLD" # 2006/09/01
echo "trap \". $qsubin_cleanup ; exit 99 \" $signals_to_trap " >> $qsubin
# PUT IN THE COMMANDS TO RUN THE PROGRAM ASKED
###2007/11/30 uberftp_applic_specific_cmd="" # 2006/08/29 this command is executed while we are in the OutPut DIR # 2007/11/
30 skk OBSOLETE NOW
if [ $PROGRAM_TO_RUN = "g03l" ] ; then
# this is for the tungsten and mike4 that do not have modules to source g03.profile
# Note that the quotes '"' are essential
[ -n "$local_gauss_source_command" ] && echo "$local_gauss_source_command" >> $qsubin
if [ "$clustername" = "co.ncsa.uiuc.edu" ]; then
echo "#alias g03l=g09" >>$qsubin
fi
/bin/cat >> $qsubin << EOF
#
# RUN GAUSSIAN
#Jayeeta commented following line
#soft add +gaussian-03
echo "DEBUG: PATH is \$PATH "; echo
#Jayeeta changed following line
#g03l input.com \$Diskoutputfile_with_dir
g09 input.com \$Diskoutputfile_with_dir
EOF
elif [ $PROGRAM_TO_RUN = "g09" ] ; then
# this is for the tungsten and mike4 that do not have modules to source g03.profile
# Note that the quotes '"' are essential
[ -n "$local_gauss_source_command" ] && echo "$local_gauss_source_command" >> $qsubin
if [ "$clustername" = "co.ncsa.uiuc.edu" ]; then
echo "#alias g03l=g09" >>$qsubin
fi
/bin/cat >> $qsubin << EOF
#
# RUN GAUSSIAN
#Jayeeta commented following line
#soft add +gaussian-09
echo "DEBUG: PATH is \$PATH "; echo
g09 input.com \$Diskoutputfile_with_dir
EOF
elif [ $PROGRAM_TO_RUN = "tinker_monte" ] ; then
if [ "$DOMAIN" = "tacc_stampede" ] ; then
echo "/home1/00421/ccguser/apps/tinker/tinker/bin/monte.x $xyzf -k $keyf $stps $CTc $stpsz $temp $Rconv >| \$Diskoutput
file_with_dir" >> $qsubin
fi
elif [ $PROGRAM_TO_RUN = "dftb+" ] ; then
if [ "$DOMAIN" = "tacc_stampede" ] ; then
echo "/bin/cp $Diskinputdir/$Diskinputfile dftb_in.hsd" >> $qsubin
echo "/home1/00421/ccguser/apps/dftb+/bin/dftb+_1.2.2.x86_64-linux >| \$Diskoutputfile_with_dir" >> $qsubin
fi
elif [ $PROGRAM_TO_RUN = "nwchem" ] ; then
if [ "$DOMAIN" = "tacc_stampede" ] ; then
echo "export NWCHEM_BASIS_LIBRARY=/opt/apps/intel13/mvapich2_1_9/nwchem/6.3/data/libraries/" >> $qsubin
echo "/opt/apps/intel13/mvapich2_1_9/nwchem/6.3/bin/nwchem $Diskinputfile >| \$Diskoutputfile_with_dir" >> $qsubin
else
echo "export NWCHEM_BASIS_LIBRARY=/home/diag/opt/nwchem/data/libraries/" >> $qsubin
echo "/home/diag/opt/nwchem/bin/nwchem $Diskinputfile >| \$Diskoutputfile_with_dir" >> $qsubin
fi
elif [ $PROGRAM_TO_RUN = "adf" ] ; then
if [ "$DOMAIN" = "ncsa_copper" ] ; then #2006.1 version
echo "\$ADFBIN/adf -n$cpus < $Diskinputfile >| \$Diskoutputfile_with_dir " >> $qsubin
echo "\$ADFBIN/dmpkf TAPE21 > TAPE21.txt " >> $qsubin
# 2007/10/04 skk - Joohyun's changes
else
echo "adf -n$cpus < $Diskinputfile >| \$Diskoutputfile_with_dir " >> $qsubin
echo "dmpkf TAPE21 > TAPE21.txt " >> $qsubin
fi
# 2007/11/30 joohyun
elif [ $PROGRAM_TO_RUN = "adf_quild" ] ; then
echo "\$ADFBIN/quild -n$cpus < ${job}.inp >| \$Diskoutputfile_with_dir " >> $qsubin
elif [ $PROGRAM_TO_RUN = "abaqus" ]; then
input_file_name="$inputfile" # inputfile with extension
abq_job=${Diskinputfile%.*} #"" # input is $job.inp from $1 argument
user_sub_option="" # --user_sub file argument -a -n $lib for libraries
restart_option="" # --restart_file option requires a .res file defined as oldjob=restart_file_stem (a path?!)
# globalmodel_option="" #--globalmodal_file
datach="" # --datafile_file or --datafile2_file2.tar
doubl="double" # --double (precision)
# parallel_option="" # multi processor run explicit domain decompostion set parallel=domain
# domain_option="" # --mpi_domain (Explicit Only use domain decomposition mpi solver) domains=#np supplied with --np option
# abaqus run command here
/bin/cat >> $qsubin << HERE
# run the abaqus executable
echo "y" | \\
$abaqus_executable \\
job="$abq_job" scratch="\$PWD" cpus="$LOCAL_PROCS_PER_NODE" mem="80%" input="\$PWD/$Diskinputfile" mp_mode=threads interact
ive \\
$user_sub_option $restart_option $datach $doubl
HERE
elif [ $PROGRAM_TO_RUN = "ddscat-v7_1" ] ; then
echo "cp -p $TEMPDIR1/*.* $SCRATCH_BATCH" >> $qsubin
command_to_run="mpirun_rsh -np $cpus -hostfile \$PBS_NODEFILE /home/gridchem/apps/ddscat/ddscat7.1/bin/ddscat >& ${job}.ou
t"
echo "$command_to_run" >> $qsubin
elif [ $PROGRAM_TO_RUN = "ddscat-v7_2" ] ; then
echo "cp -p $TEMPDIR1/*.* $SCRATCH_BATCH" >> $qsubin
echo ". /etc/profile.d/modules.sh; module load mkl/10.3.3;ldd /home/gridchem/apps/ddscat/ddscat7.2.2/src/ddscat " >> $qsub
in
command_to_run="mpirun_rsh -np $cpus -hostfile \$PBS_NODEFILE /home/gridchem/apps/ddscat/ddscat7.2.2/src/ddscat >& ${job}.
out"
echo "$command_to_run" >> $qsubin
elif [ $PROGRAM_TO_RUN = "rungms" ] ; then
# 2005/06/30 this is just a warning to save me from stupidity of gamess
# 2007/05/09 skk moved into qsubin where it belongs
Basename=${Diskinputfile%.*}
echo 'tmp=`echo $PATH | wc -c ` ; if [ $tmp -gt 750 ] ; then echo "YOU HAVE A PATH LENGTH OF $tmp CHARACTERS - SO YOU ARE
APPROACHING OR YOU ARE OVER THE GAMESS_STUPIDITY PATH LENGTH OF 846 - PLEASE CLEAN PATH "; fi' >> $qsubin
# 2007/05/08 (JK) at the moment only for copper
echo 'echo "length of path : \$tmp"' >> $qsubin
# if [ $DOMAIN = "ncsa_copper" ]; then
# echo 'INFO: we are manipulating PATH in copper for gamess due to a bug in gamess'
# echo 'if [ $tmp -gt 750 ] ; then ' >> $qsubin
# echo ' export PATH=/usr/bin:/bin:/usr/local/bin' >> $qsubin
# echo ' soft add +gamess' >> $qsubin
# echo ' soft add +globus' >> $qsubin
# echo 'fi' >> $qsubin
# fi
# this is for mike4 that do not have modules to source g03.profile
# Note that the quotes '"' are essential
[ -n "$local_gms_source_command" ] && echo "$local_gms_source_command" >> $qsubin
if [ `hostname|grep '^lslogin'` ]; then
echo "echo Checking for input file..." >> $qsubin
echo "ls -l $Diskinputfile" >> $qsubin
echo "echo Checking current working directory" >> $qsubin
echo "pwd" >> $qsubin
#echo "ibrun gamess.run $Diskinputfile >| \$Diskoutputfile_with_dir" >> $qsubin
echo "/home/diag/opt/gamess/rungms $Basename ${GMS_VERNO} $cpus > \$Diskoutputfile_with_dir" >> $qsubin
else
echo "/home/diag/opt/gamess/rungms $Basename ${GMS_VERNO} $cpus > \$Diskoutputfile_with_dir" >> $qsubin
fi
# 2005/07/21 TODO+ check for return code and if nonzero exit 1 ?
elif [ $PROGRAM_TO_RUN = "molpro" ] ; then
echo "which molpro ">> $qsubin
if [ $DOMAIN = "ncsa_cobalt" ]; then
echo "soft add +sgi-mathlibs-pp4">> $qsubin
fi
# 2006/03/31 These 2-3 lines are not very clear - what is $SCRATCH?
echo "echo 'SCRATCH is $SCRATCH'" >> $qsubin
echo "#Resetting SCRATCH to current directory" >> $qsubin
echo "export SCRATCH=\$SCRATCH_BATCH " >> $qsubin
echo "molpro -n $cpus $Diskinputfile >| \$Diskoutputfile_with_dir" >> $qsubin
# 2005/07/21 TODO+ check for return code and if nonzero exit 1 ?
elif [ $PROGRAM_TO_RUN = "qmcpack" ] ; then
echo "which qmcpack">> $qsubin
# 2006/03/31 These 2-3 lines are not very clear - what is $SCRATCH?
echo "echo 'SCRATCH is $SCRATCH'" >> $qsubin
echo "#Resetting SCRATCH to current directory" >> $qsubin
echo "export SCRATCH=\$SCRATCH_BATCH " >> $qsubin
echo "cmpirun -np $cpus -lsf /usr/projects/mcc/bin/qmcapp_mpi ${job}.xml >| \$Diskoutputfile_with_dir" >> $qsubin
# 2005/07/21 TODO+ check for return code and if nonzero exit 1 ?
elif [ $PROGRAM_TO_RUN = "dmol3" ] ; then
echo "cp -p $TEMPDIR1/*.* $SCRATCH_BATCH" >> $qsubin
command_to_run="/usr/apps/chemistry/Accelrys/MaterialsStudio501/etc/DMol3/bin/RunDMol3.sh -np $cpus $Basename"
echo "$command_to_run" >> $qsubin
elif [ $PROGRAM_TO_RUN = "castep" ] ; then
echo "cp -p $TEMPDIR1/*.* $SCRATCH_BATCH" >> $qsubin
command_to_run="/usr/apps/chemistry/Accelrys/MaterialsStudio501/etc/CASTEP/bin/RunCASTEP.sh -np $cpus $Basename"
# command_to_run="$CASTEPHOME/RunCASTEP.sh"
echo "$command_to_run" >> $qsubin
elif [ $PROGRAM_TO_RUN = "charmm" ] ; then
echo "cp -p $TEMPDIR1/*.* $SCRATCH_BATCH" >> $qsubin
command_to_run="mpirun -np $cpus /usr/apps/chemistry/charmm/c36a3/exec/em64t/charmm < $Diskinputfile > ${Diskinputfile}.ou
t"
echo "$command_to_run" >> $qsubin
elif [ $PROGRAM_TO_RUN = "mdrun_mpi" ] ; then
Basename=${Diskinputfile%.*}
echo "cp -p $TEMPDIR1/*.* $SCRATCH_BATCH" >> $qsubin
command_to_run="mpirun_rsh -np $cpus -hostfile \$PBS_NODEFILE /home/diag/opt/gromacs/bin/mdrun -s $Basename -o $Basename -
c $Basename"
# command_to_run="$CASTEPHOME/RunCASTEP.sh"
echo "$command_to_run" >> $qsubin
elif [ $PROGRAM_TO_RUN = "mber_sander" ] ; then # 2006/08/25
amber_mdout=$job.out
#amber_mdinfo=mdinfo
#amber_mdcrd=mdcrd
#amber_mdvel=mdvel
#amber_mden=mden
#amber_restrt=restrt
# 2007/12/22 JK
echo "cp -p $TEMPDIR1/mdin $TEMPDIR1/prmtop $TEMPDIR1/inpcrd $SCRATCH_BATCH" >> $qsubin
if [ $cpus = 1 ]; then
# command_to_run="/usr/apps/chemistry/amber/amber11/bin/sander -O -i mdin -o $amber_mdout -inf $amber_mdinfo -p prmtop
-c inpcrd -ref refc -x $amber_mdcrd -v $amber_mdvel -e $amber_mden -r $amber_restrt"
command_to_run="/home/diag/opt/amber/AmberTools/bin/sander.MPI -O -i mdin -c inpcrd -p prmtop -o $amber_mdout"
else
# command_to_run="mpirun -np $cpus /usr/apps/chemistry/amber/amber11/bin/sander.MPI -O -i mdin -o $amber_mdout -inf $am
ber_mdinfo -p prmtop -c inpcrd -ref refc -x $amber_mdcrd -v $amber_mdvel -e $amber_mden -r $amber_restrt"
command_to_run="mpirun_rsh -np $cpus -hostfile \$PBS_NODEFILE /home/diag/opt/amber/AmberTools/bin/sander.MPI -O -i mdi
n -c inpcrd -p prmtop -o $amber_mdout"
fi
echo "$command_to_run" >> $qsubin
echo "if [ \$? != 0 ] ; then echo \"ERROR: command failed: $command_to_run\"; exit 1 ; fi" >> $qsubin
# 2006/08/25 per Scott: the output files that must be returned: $amber_mdout $amber_restrt $amber_mdinfo $amber_mdcrd $amb
er_mdvel $amber_mden in descending order of importance. some files especially the last three may not exist.
###2007/11/30 files_to_save="$amber_mdout $amber_restrt $amber_mdinfo $amber_mdcrd $amber_mdvel $amber_mden"
###2007/11/30 echo "files_to_save_TRUE=; for ifile in $files_to_save ; do if [ -f \"\$ifile\" ] ; then files_to_save_TRUE=
\"\$files_to_save_TRUE \$ifile \"; fi ; done" >> $qsubin
###2007/11/30 echo "if [ -n \"\$files_to_save_TRUE\" ]; then " >> $qsubin
###2007/11/30 command_to_run="cp -p \$files_to_save_TRUE \$OutDir"
###2007/11/30 echo " $command_to_run" >> $qsubin
###2007/11/30 echo " if [ \$? != 0 ] ; then echo \"ERROR: command failed: $command_to_run\"; exit 1 ; fi" >> $qsubin
###2007/11/30 echo " uberftp_applic_specific_cmd=\"\"; for ifile in \$files_to_save_TRUE; do uberftp_applic_specific_cmd=
\"\$uberftp_applic_specific_cmd put \$ifile; \" ; done " >> $qsubin
###2007/11/30 echo "else " >> $qsubin
###2007/11/30 echo " echo \"ERROR: None of the $files_to_save exists. \"; " >> $qsubin
###2007/11/30 echo " exit 1" >> $qsubin
###2007/11/30 echo "fi " >> $qsubin
# 2007/11/29 joohyun
elif [ $PROGRAM_TO_RUN = "aces3" ] ; then
# cobalt 2007/11/16
echo "$ACES3_LOCAL_OPTION" >> $qsubin
echo "cp -p $inputfile ZMAT" >> $qsubin
echo "mpirun -np $cpus ./xaces3 >| \$Diskoutputfile_with_dir " >> $qsubin
elif [ $PROGRAM_TO_RUN = "lammps" ] ; then
#echo "cp -p $inputfile lammps.input" >> $qsubin
if [ "$clustername" = "gordon.sdsc.edu" ] ; then
echo "mpirun -np $cpus /home/gridchem/apps/lammps-27Jan13/src/lmp_mvapich2 < $Diskinputfile" >> $qsubin
else
echo "ibrun /opt/apps/intel13/mvapich2_1_9/lammps/10Dec12/bin/lmp_tacc < $Diskinputfile > ${job}.out" >> $qsubin
fi
elif [ $PROGRAM_TO_RUN = "lammps_ds" ] ; then
#echo "cp -p $inputfile lammps.input" >> $qsubin
if [ "$clustername" = "gordon.sdsc.edu" ] ; then
echo "mpirun -np $cpus /home/gridchem/apps/lammps-27Jan13/src/lmp_mvapich2 < $Diskinputfile" >> $qsubin
else
# run the local compute job
# Step 1 lAMMPS MD
echo "ibrun ~/apps/ds_lammps/bin/lmp_stampede < $Diskinputfile > ${job}.out" >> $qsubin
# if no dump files were created by lmp_stampede,
# then skip trying to submit a vis job
# on Gordon (or Stampede)
/bin/cat >> $qsubin << EOF
Ndumps=\`find . -type f -name \*dump | wc -l\`
if [ \$Ndumps -eq "0" ]
then
echo "No dump files were generated, skipping vis job." ;
else
echo "dump files generated, creating vis job."
fi
### New loopdiffraction.sh with MIC outputs
# Step 2 LAMMPS Diffraction Copute
Ndumps=\`ls -l *$Find_Suffix1 |wc -l\`
if [ \$Ndumps -eq "0" ]; then
echo " No dump files found in $PWD "
exit
fi
n=0
for i in *.$Find_Suffix1
do
D=\`echo \${i}\`
P=\`echo \${i%%.$Find_Suffix1}\`
tmp=\`echo \${i%%.$Find_Suffix1} | sed 's#^.*/##' | rev\`
N=\`echo "\${tmp%%_*}" | rev\`
sed 's/XXXXX/'\$N'/g' $Diffraction_Base > $Input.\$n
sed -i 's/DUMPFILE/'\$D'/g' $Input.\$n
sed -i 's/PREFIX/'\$P'/g' $Input.\$n
n=\$((\$n+1))
done
n=\$((\$n-1))
# Setting up environment variables
export SLURM_NNODES=$MPI2
export SLURM_TACC_NODES=$MPI2
export SLURM_NTASKS=$MPI2
export SLURM_TASKS_PER_NODE='1(x$MPI2)'
export SLURM_NPROCS=$MPI2
export SLURM_JOB_CPUS_PER_NODE='16(x$MPI2)'
export SLURM_JOB_NUM_NODES=$MPI2
export SLURM_TACC_CORES=$MPI2
export SLURM_TACC_NNODES_SET=1
export OMP_NUM_THREADS=$OpenMP
export MIC_OMP_NUM_THREADS=$MIC
export LAMMPS=/home1/00421/ccguser/apps/ds_lammps/bin/lmp_stampede
# Run LAMMPS Jobs
for i in \$(seq 0 \$n)
do
#ibrun tacc_affinity \$LAMMPS < $PWD/$Input.\$i > $PWD/$Input.\$i.out
ibrun tacc_affinity \$LAMMPS < $Input.\$i > $Input.\$i.out
done
# Step 3 (On vis queue)
#exit
# Run multiple VisIt sessions for each *.vtk
#module load visit/2.7
# We will use visit from ccguser
export visit=/work/00421/ccguser/VisIt/visit2.7.2/src/bin/visit
Nvtk=\`ls -l *$Find_Suffix2 |wc -l\`
if [ \$Nvtk -eq "0" ]; then
echo " No vtk files found in $PWD "
exit
fi
Nmin=\`echo \$((\$Nvtk<$MPI3?\$Nvtk:$MPI3))\`
for i in \$(seq 1 \$Nmin )
do
mkdir tmp.\$i
cp $Vis_Script $Vis_Session tmp.\$i/
done
n=1
for i in *.vtk
do
cp \${i} tmp.\$n/
n=\$((\$n+1))
if [ \$n -gt \$Nmin ]; then
n=1
fi
done
for i in \$(seq 1 \$Nmin )
do
#cd $PWD/tmp.\$i
cd tmp.\$i
ibrun -o \$i -n 1 \$visit -cli -nowin -quiet -s $Vis_Script &
cd ..
done
#cd $PWD/
jobs -p > JOB
for i in \$(seq 1 \$Nmin )
do
fg \`sed "\$i!d" JOB \`
done
wait
#mv -f $PWD/tmp.*/*png $PWD/
mv -f tmp.*/*png .
#rm -rf $PWD/tmp.* $PWD/JOB $PWD/$Input.*
#rm -rf $PWD/tmp.* $PWD/JOB $PWD/$Input.*
rename .0.vtk. . *png
rename .session.0000. . *png
#### End of new loopDiffraction
EOF
# Create the Vis specific files here
qsubin_vis="$TEMPDIR1/qsubin$$_vis"
test -f "$qsubin_vis" && /bin/rm $qsubin_vis
touch $qsubin_vis
chmod 700 $qsubin_vis
qsubin_uberftp_vis1=qsubin$$_uberftp_vis1
qsubin_uberftp_vis1="$TEMPDIR1/$qsubin_uberftp_vis1"
test -f "$qsubin_uberftp_vis1" && /bin/rm $qsubin_uberftp_vis1
touch $qsubin_uberftp_vis1
chmod 700 $qsubin_uberftp_vis1
# MVM: 20140318 Commenting this section out as we're returning to
# doing both compute and vis on stampede.
# qsubin_uberftp_gordon_vis=qsubin$$_uberftp_gordon_vis
# qsubin_uberftp_gordon_vis="$TEMPDIR1/$qsubin_uberftp_gordon_vis"
# test -f "$qsubin_uberftp_gordon_vis" && /bin/rm $qsubin_uberftp_gordon_vis
# touch $qsubin_uberftp_gordon_vis
# chmod 700 $qsubin_uberftp_gordon_vis
# ssh_gordon_sh=ssh$$_gordon_sh
# ssh_gordon_sh="$TEMPDIR1/$ssh_gordon_sh"
# test -f "$ssh_gordon_sh" && /bin/rm $ssh_gordon_sh
# touch $ssh_gordon_sh
# chmod 700 $ssh_gordon_sh
#TODO - create gordon cleanup script here.
# START of script that ssh on login1.stampede will use to copy files to Gordon
# echo "#!/bin/bash" >> $ssh_gordon_sh
# echo "set +x" >> $ssh_gordon_sh
# echo "module load GLOBUS-5.0" >> $ssh_gordon_sh
# # mvm: note to self, this confuses me, $X509_USER_PROXY exists, but I still have to
# # export it?
# echo "echo X509_USER_PROXY: $X509_USER_PROXY" >> $ssh_gordon_sh
# echo "export X509_USER_PROXY=$X509_USER_PROXY" >> $ssh_gordon_sh
# echo "grid-proxy-info" >> $ssh_gordon_sh
# echo "gsissh gordon.sdsc.edu mkdir $GridchemUserid/$ResearchProject/\$jobdir" >> $ssh_gordon_sh
#
#
# echo "cd \$OutDir" >> $ssh_gordon_sh
# echo "for f in *" >> $ssh_gordon_sh
# echo "do" >> $ssh_gordon_sh
# echo " gsiscp \$f gordon.sdsc.edu:$GridchemUserid/$ResearchProject/\$jobdir" >> $ssh_gordon_sh
# echo "done" >> $ssh_gordon_sh
#
# # LoopDiffraction_normal.sh creates the PBS job script, SubmitScript, only; it still has to be submitted.
# echo "gsissh gordon.sdsc.edu <<EOF
# cp apps/ds_lammps_hybrid/bin/LoopDiffraction_normal.sh $GridchemUserid/$ResearchProject/\$jobdir
# cd $GridchemUserid/$ResearchProject/\$jobdir
##
# sed -i \"s%^EOF%gsiscp ./images.tar.gz $mss_machine:mss/$topdir_name/$GridchemUserid/$ResearchProject/\$jobdir\n
EOF%\" ./LoopDiffraction_normal.sh
# sed -i \"s%^EOF%gsiscp ./*.xrd $mss_machine:mss/$topdir_name/$GridchemUserid/$ResearchProject/\$jobdir\nEOF%\" .
/LoopDiffraction_normal.sh
# sed -i \"s%^EOF%gsiscp ./*.vtk $mss_machine:mss/$topdir_name/$GridchemUserid/$ResearchProject/\$jobdir\nEOF%\" .
/LoopDiffraction_normal.sh
#
# ./LoopDiffraction_normal.sh
# qsub SubmitScript
#EOF" >> $ssh_gordon_sh
# #echo "gsiscp $qsubin_uberftp_gordon_vis gordon.sdsc.edu:$GridchemUserid/$ResearchProject" >> $ssh_gordon_sh
# # END of scrpt
# This starts the process of connecting to Gordon
# Define $jobdir with current information in the script before running
# /bin/cat >> $qsubin << EOF
# export jobdir=\$jobdir
# export OutDir=\$OutDir
# sed -i "s/export/export jobdir=\$jobdir \nexport/" $ssh_gordon_sh
# sed -i "s%export%export OutDir=\$OutDir \nexport%" $ssh_gordon_sh
# ssh login1.stampede.tacc.utexas.edu $ssh_gordon_sh > ssh_gordon.out 2>&1
#fi #end of Ndump conditional
#EOF
fi # closes check for gordon clustername
fi # closes check for $PROGRAM_TO_RUN
# ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
################################################################
# 2006/04/10 QSUBIN_CLEANUP FILE WILL BE SEPARATE SO THAT IT CAN BE USED EITHER THROUGH THE TRAP OR THROUGH REGULAR RUN
################################################################
# 2006/06/13 the #!/bin/ksh was inserted to allow vim etc to do nice fontification
echo "#!/bin/ksh" > $qsubin_cleanup
# 2006/09/01 we dont need to trap anymore (we dont want to go into a loop like today :-( )
echo "trap $signals_to_trap " >> $qsubin_cleanup
/bin/cat >> $qsubin_cleanup << EOF
rm \$HOME_BATCH
mkdir -p \$HOME_BATCH
cp \$SCRATCH_BATCH/*.out \$HOME_BATCH/
rm Gau-*
cp \$SCRATCH_BATCH/* \$HOME_BATCH/
cd \$HOME_BATCH
/bin/ln -s $TEMPDIR1 \$HOME_BATCH/
EOF
# 2008/05/21 srb output filename is job.castep not job.out; rename it
if [ $PROGRAM_TO_RUN = "castep" ] ; then
castep_output_extension="castep"
echo "castep_output_extension=$castep_output_extension " >> $qsubin_cleanup
echo "mv $job.\$castep_output_extension $Diskoutputfile_no_dir " >> $qsubin_cleanup
fi
# 2008/05/06 skk Joohyun says that instead of job.out it creates job.outmol so we have to move it to the right name job.out
if [ $PROGRAM_TO_RUN = "dmol3" ] ; then
extra_output_sufffix="mol"
echo "extra_output_sufffix=$extra_output_sufffix " >> $qsubin_cleanup
echo "mv $Diskoutputfile_no_dir\$extra_output_sufffix $Diskoutputfile_no_dir " >> $qsubin_cleanup
fi
/bin/cat >> $qsubin_cleanup << EOF2a
RunMachine="\`/bin/uname -n\`"
echo "Job ran on Machine=\$RunMachine " >> \$Diskoutputfile_with_dir
EOF2a
###2007/11/30 # 2007/11/09 skk - prepare to send the gamess dat files to MSS
###2007/11/30 if [ $PROGRAM_TO_RUN = "rungms" ] ; then uberftp_applic_specific_cmd2="put $job.dat ;" fi # this command is
executed while we are in the SCRATCH_BATCH DIR
# 2008/05/21 made changes so as to mv instead of cp the chk file to Chkfile_with_arch - to avoid wasting space etc.
if [ $PROGRAM_TO_RUN = "g03l" -o $PROGRAM_TO_RUN = "g09" ] ; then
/bin/cat >> $qsubin_cleanup << EOF
if [ -f "$Chkfile" ] ; then
# 2005/12/08 convert checkpoint file back to formated
if formchk $Chkfile $formated_chkfile ; then
uberftp_command_formated_chk=" put $formated_chkfile ; "
else
echo "WARNING: formchk command failed - thus we will not send the formated file to MSS"
fi
mv $Chkfile $Chkfile_with_arch
uberftp_command_chk=" put $Chkfile_with_arch ; "
else
echo "INFO: no checkpoint file was found"
fi
EOF
# 2005/10/09 skk - rename and cp fort.7 which is the .pun file (according to Sudhakar )
/bin/cat >> $qsubin_cleanup << EOF2a
if [ -f fort.7 ]; then
/bin/mv fort.7 $job.pun
fi
EOF2a
fi
if [ $PROGRAM_TO_RUN = "abaqus" ] ; then
/bin/cat >> $qsubin_cleanup << EOF
/bin/mv ${abq_job}.odb ${job}.odb
EOF
fi
# 2008/06/05 about 20 lines moved here from below so that we do that rm before the mv of all files from SCRATCH_BATCH INTO THE
OutDir
# 2008/06/05 not sure if the mv $Chkfile_with_arch $OutDir etc are needed now since we move everything from SCRATCH_BATCH - to
do
# 2008/04/22 skk about 10 lines moved here from below the uberftp command
# 2006/04/27 Added moving the chk_with_arch and fchk files in Outdir
if [ $PROGRAM_TO_RUN = "g03l" -o $PROGRAM_TO_RUN = "g09" ] ; then
/bin/cat >> $qsubin_cleanup << EOF
if [ -f "$Chkfile_with_arch" ] ; then
mv $Chkfile_with_arch $OutDir/
fi
if [ -f "$formated_chkfile" ] ; then
mv $formated_chkfile $OutDir/
fi
EOF
echo "/bin/rm -f $SCRATCH_BATCH/Gau-*.??? " >> $qsubin_cleanup # 2008/05/22 Now remove ALL Gau-* files eg .rwf just in ca
se gaussian fails to remove them
fi
# 2007/11/30 MOVE ALL FILES FROM SCRATCH_BATCH INTO THE OUTPUT DIR - per 2007/11/28 discussions - skk
if [ ! -n "$OUTPUT_DIR_IS_ACCESSIBLE" ] ; then
exit_with_message "not fixed yet 2007/11/30 "; # 2007/11/30 skk if needed i can fix it too
# on sdx just after main program is done , remove soft link and move output file Diskoutputfile_with_dir to the same positio
n (just in case) -
/bin/cat >> $qsubin_cleanup << EOF2aa
#2006/08/30 ssh sdx " \rm \$OutDir/$Diskoutputfile_no_dir "
ssh sdx " \mv \$OutDir/$Diskoutputfile_no_dir \$OutDir/${Diskoutputfile_no_dir}.TO_REMOVE" # 2006/08/30 TEMPORARY
#2006/08/30 -- # ssh sdx " \mv \$Diskoutputfile_with_dir \$OutDir/$Diskoutputfile_no_dir "
\cp -p \$Diskoutputfile_with_dir \$OutDir/$Diskoutputfile_no_dir # 2006/08/30 (we want the file in the correct position in
scratch on compute node, so that it transfers to the login node through the after-job-mirroring that happens in sdx
ssh sdx " \cp -p \$Diskoutputfile_with_dir \$OutDir/$Diskoutputfile_no_dir " # TEMPORARY SOLUTION 2006/08/30
Diskoutputfile_with_dir=\$OutDir/$Diskoutputfile_no_dir # just in case
EOF2aa
else
# NOTE: unlink was not working on sdx so now i use rm -- 2005/06/17
# The /bin/rm IS NEEDED - otherwise cp will fail (even cp --force will fail)
# 2008/06/05 skk IMPORTANT - todo: i should use /bin/mv * $OutDir instead of /bin/cp -p * $OutDir since i dont need the f
iles anymore -- also maybe do /bin/mv $SCRATCH_BATCH/* ... for more safety?
/bin/cat >> $qsubin_cleanup << EOF2a
/bin/cp -p * $OutDir
EOF2a
fi
# SEND OUTPUT FILES TO MSS
# 2007/02/15 skk TEMP - to debug issues with cobalt and copper with grid-proxy-info "failing" but ok
/bin/cat >> $qsubin_cleanup << EOF1
#$grid_proxy_info
ls -l \$X509_USER_PROXY
EOF1
# 2008/04/30 moved 20-30 lines out of the if loop below
# 2008/04/09 moved here from below
# 2006/03/31 skk mv some stuff from the tempdir (this is the $WORKDIR) to the outdir (the true outdir which is OutDir?)
# clean some stuff
# make a link to the tempdir so that we can easily see the batch-stdout and batch-stderr
echo "echo \"Removing some scratch files from tempdir $TEMPDIR1\" " >> $qsubin_cleanup
echo "/bin/mv $qsubin $OutDir/ " >> $qsubin_cleanup
echo "/bin/mv $qsubin_cleanup $OutDir/ " >> $qsubin_cleanup
# 2008/04/09
if [ $PROGRAM_TO_RUN = "amber_sander" ] ; then
for tmp in mdin prmtop inpcrd; do
echo "[ -f $TEMPDIR1/$tmp ] && /bin/mv $TEMPDIR1/$tmp $OutDir/ " >> $qsubin_cleanup ;
done
else
echo "/bin/mv $TEMPDIR1/$Diskinputfile $OutDir/ " >> $qsubin_cleanup # 2008/04/09 the file should exist
fi
if [ $PROGRAM_TO_RUN = "aces3" ] ; then
echo "/bin/rm ${OutDir}/*sio" >> $qsubin_cleanup
echo "/bin/rm ${OutDir}/GENBAS" >> $qsubin_cleanup
echo "/bin/rm ${OutDir}/xaces3" >> $qsubin_cleanup
fi
echo "[ -f $TEMPDIR1/$Diskinputfile.orig ] && /bin/mv $TEMPDIR1/$Diskinputfile.orig $OutDir/ " >> $qsubin_cleanup # 2006/03/3
1 this assumes that the perl script is using the *.orig ending too
# 2007/11/09 SRB Added moving the gamess dat files to Outdir.
if [ $PROGRAM_TO_RUN = "rungms" ] ; then
/bin/cat >> $qsubin_cleanup << EOF
mv $job.dat $OutDir/
EOF
fi
# 2006/06/02 first check if proxy is valid - otherwise notify us (i include the \$OutDir so as to know hostname jobid etc
###2007/03/06 skk - TEMP COMMENTING DUE TO ISSUES WITH COPPER AND COBALT -- $grid_proxy_info -exists || echo ""|mail -s "PROX
Y IS EXPIRED (or missing??) - PLEASE send files to MSS by hand (\$OutDir) " kyriacou@osc.edu spamidig@NCSA.UIUC.EDU # 2007/1
2/17 note that i ll have to modify it to use sendmail from now on due to OSC and NCSA ...
/bin/cat >> $qsubin_cleanup << EOF1
qsubin_nodir=`echo $qsubin|sed 's/.*\///' ` # uberftp wants a filename with no dir names in it
qsubin_cleanup_nodir=`echo $qsubin_cleanup|sed 's/.*\///' ` # uberftp wants a filename with no dir names in it
if [[ -n "\$X509_USER_PROXY" ]] ; then
remotedir="$remotedir"
EOF1
# 2005/09/26 skk - had to do the mkdir a separate uberftp for all possibly existing dirs (eg kyriacou etc)
# due to the new uberftp in ccglogin giving an error if dir exists and not doing the rest of the commands in
the command-string
# 2005/09/26 TODO maybe unite the two "if" loops for simplicity
# 2005/10/10 adding an lcd because when we do uberftp from the login node we ll end up to be in the home dir
# 2006/06/13 NOTE: "lcd $OutDir; put $Diskoutputfile_no_dir; " should work even if not SCRATCH_BATCH_DIR_IS_ACCESSIBLE (--)
# 2007/01/09 DONE >> 2006/08/29 todo: make "put $job.pun" etc application specific
# 2008/04/09 skk now Diskinputfile may not exist eg if multi-input
# 2008/05/01 skk IMPORTANT - fixed a major bug introduced on version 1.100 on 2008/04/22 when i moved the cd commands around -
the fix was the addition of " cd;" (being cautious, I put it in 2 places )
# 2008/05/20 skk I changed the grep strings in : $uberftp 'dir \$remotedir' \" 2>&1 1>/dev/null |grep "Could not lis
t " && ....
# 2007/11/30 skk seems that "put *" gives an error and disallows executing subsequent commands , so i put it last
# 2007/12/07 ssk UNFORTUNATELY uberftp does not have a good exit code - a workaround is to check if the dir was created ... -
based on the error "Could not list internal/....: No such file or directory."
# 2008/05/22 skk use mput * instead of put * - to allow mike4 to work too
/bin/cat >> $qsubin_cleanup << EOF1
fi # if [ -n "$DO_UBERFTP_ON_LOGINNODE"
EOF1
# 2007/02/15 skk : -n checks if the variable exists and -f if the file exists - (why the "[["?)
/bin/cat >> $qsubin_cleanup <<EOF
if [[ -n "\$X509_USER_PROXY" ]] ; then
[ -f "\$X509_USER_PROXY" ] && echo "DEBUG: TEMP: 2007/02/15 NOT DOING THIS FOR NOW: /bin/rm -rf \$X509_USER_PROXY"
fi
EOF
# 2006/03/29 skk - CLEAN UP
### 2008/04/22 skk moved lines further up
echo "echo \"Removing directory \$SCRATCH_BATCH\" " >> $qsubin_cleanup
echo "cd $OutDir" >> $qsubin_cleanup # 2006/08/25 Let us get out of the directory before we remove it
# 2006/10/09 This seems to be the last command in the file so if it fails it give a bad exit code.
# So let us make sure we get 0 = true even if the rm fails
echo "/bin/rm -rf \$SCRATCH_BATCH 2>/dev/null 1>/dev/null || true " >> $qsubin_cleanup
# ))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))
################################################################
# 2006/04/10 EXECUTE THE CLEANUP SCRIPT NOW THAT THE SCRIPT IS CREATED
if [ $PROGRAM_TO_RUN = "lammps_ds" ]; then
echo ". $qsubin_cleanup" >> $qsubin
# this is (replicated) moved into $qsubin_vis
echo "# Qsubin CleanUp will be done in Qsubin_Vis " >> $qsubin
else
echo ". $qsubin_cleanup" >> $qsubin
fi
################################################################
#
# STEP 5: SUBMIT THE JOB
#
echo "DEBUG: pwd is `pwd`"
echo "$LOGNAME `date`" >> $SYSTEM_LOG
echo "SUBMITTING JOB"
# 2005/06/14 skk - eval is needed due to the $local_qsub_extra being a "<" sometimes and not evaluated correctly by shell
qsub_out=`eval "$qsub $local_qsub_extra $qsubin " | tee -a $SYSTEM_LOG `
echo "QSUB OUTPUT IS:
$qsub_out"
# Save job submission for the dependent job
echo "QSUB OUTPUT IS:
$qsub_out" >> $qsubin
jobid=`grep Submitted $qsubin | awk '{print $4}'`
#export $jobid
# mvm: testing launching a dependent vis job. For now, the vis batch script is hard coded.
if [ $PROGRAM_TO_RUN = "lammps_ds" ] ; then
# define variables
jobdir="$job.$clustername.$jobid.$date_YYMMDD"
OutDir="/home1/00421/$USER/scratch/$GridchemUserid/$ResearchProject/$jobdir"
# Create uberftp script
echo "#!/bin/ksh" > $qsubin_uberftp_vis1
echo "jobdir=$jobdir" >> $qsubin_uberftp_vis1
echo "remotedir=\"$topdir_name/$remotedir_common\" " >> $qsubin_uberftp_vis1
echo "remotedir=mss/$remotedir" >> $qsubin_uberftp_vis1
echo "OutDir=$OutDir" >> $qsubin_uberftp_vis1
# 2008/09/26 skk moved here out of the if statement and improved
uberftp_failure_email="To: $userMail\nCc: $userMail_cc
Subject: UBERFTP_FAILED for local job id $jobid
UberFTP failed to transfer files to mass storage.
Please use \"Monitor Output\" function on MyCCG Panel to retrieve the output and
if it fails create a ticket with consulting portal.
DETAILS (for sysadmins mostly):
To retry sending the data to MSS we should use this command (after setting up appropriate credentials) - would need to remove
the last few lines in that script though starting with line '^||' :
$OutDir/$qsubin_uberftp_nodir
"
/bin/cat >> $qsubin_uberftp_vis1 << EOF1
$gsissh "mkdir mss/$topdir_name" 1>/dev/null 2>&1
$gsissh "mkdir mss/$topdir_name/$GridchemUserid" 1>/dev/null 2>&1
$gsissh "mkdir mss/$topdir_name/$GridchemUserid/$ResearchProject" 1>/dev/null 2>&1
$gsiscp -r $OutDir $mss_machine:mss/$topdir_name/$GridchemUserid/$ResearchProject
EOF1
# MVM - this should not be echoed to $qsubin, which has already been
# submitted at this point.
#echo "MVM: vis batch is being created based on $jobid" >> $qsubin
# MVM - create the qsubin_vis script
# MVM this is the Stampede vis job
VIS_BATCH="#!/bin/bash
#SBATCH -J ${job}
#SBATCH -p vis
#SBATCH -o ${job}.o%j
#SBATCH -N 1
#SBATCH -n 1
#SBATCH -t 00:05:00
#SBATCH --mail-user=$email_addresses
#SBATCH --mail-type=ALL
#SBATCH -A TG-CHE080068N"
eval "echo \"$VIS_BATCH\" " >> $qsubin_vis
echo "set -x" >> $qsubin_vis
echo "echo \"copying render.py to pwd\" " >> $qsubin_vis
echo "jobdir=\"$job.$clustername.$jobid.$date_YYMMDD\" " >> $qsubin_vis
echo "export OutDir=/home1/00421/$USER/scratch/$GridchemUserid/$ResearchProject/\$jobdir" >> $qsubin_vis
echo "# SCRATCH_BATCH=\"/home/00421/$USER/$OutDir\" " >> $qsubin_vis
echo "#cd \$SCRATCH_BATCH" >> $qsubin_vis
echo "cd \$OutDir" >> $qsubin_vis
echo "cp /home1/00421/ccguser/apps/ds_lammps/bin/render.py ." >> $qsubin_vis
echo "echo \"loading visit module\" " >> $qsubin_vis
echo "module load visit" >> $qsubin_vis
echo "echo \"timing visit run\" " >> $qsubin_vis
echo "time visit -cli -nowin -quiet -s render.py >> ${job}.out 2>&1" >> $qsubin_vis
echo "cp * \$OutDir " >> $qsubin_vis
echo ". $qsubin_uberftp_vis1" >> $qsubin_vis
echo "exit" >> $qsubin_vis
# MVM - submit the qsubin_vis script
# eval "$qsub --dependency=afterok:$jobid $qsubin_vis"
fi
# 2008/07/18 skk - this is to tell the user what could be wrong if they got the qsub: illegal -N value
echo "$qsub_out" |grep -q "qsub: illegal -N value" && echo "IMPORTANT NOTE: \"qsub: illegal -N value\" means that you used a
job name that maybe longer than 15 chars or it may be starting with a numeric character instead of an alphabetic character"
# STEP 6: GET THE JOBID FROM THE OUTPUT OF QSUB ABOVE
# 2005/11/14 19:53
# I added code to find jobid based on this output of the "qstat"
# LOADLEVELER::_
# PARADIGM= MPI
# ADVISORY User should set network.MPI job values.
# # @ network.MPI = csss,shared,US
# or # @ network.MPI = csss,not_shared,US
# {US is faster than IP}
# ADVISORY: node_usage reset from shared to not_shared.
# to conform to TACC policy.
# LoadLeveler submit filter ver_1.0.05 complete.
# llsubmit: Processed command file through Submit Filter: "/home/loadl/llsubmitfilter".
# llsubmit: The job "champ.tacc.utexas.edu.201485" has been submitted.
# LSF::_
# Welcome to Lonestar
# --> Submitting 2 procs/node...
# --> Submitting exclusive job...
# --> Requesting running sshd...
# --> Requesting valid file system(s)...
# --> Validating project accounting...
# --> Adding default memory limit (900000 KB)
# Job <718983> is submitted to queue <systest>.
# PBS::_
# $ qsub < trash
# 47388.nfs2.osc.edu
case $local_scheduler in
SLURM )
jobid_line=$qsub_out
jobid=`echo $jobid_line | grep Submitted | awk '{print $4}'`
;;
PBS )
jobid_line=$qsub_out
jobid=`echo $jobid_line | perl -ne '@jobid = $_ =~ /(\d+)\..+$/; print "$jobid[0]" if ($jobid[0]);'`
;;
LSF|loadleveler )
jobid_line=`echo "$qsub_out" | grep submitted`
jobid=`echo $jobid_line | perl -ne '@jobid = $_ =~ /[\.\<](\d+)/; print "$jobid[0]" if ($jobid[0]);'`
;;
esac
if [ ! -n "$jobid" ] ; then
exit_with_message "jobid seems to NOT EXIST -- please check for errors above"
fi
# 2005/11/14 Lets keep the wording JOBID_FROM_CLUSTER_SCRIPT unique to facilitate parsing if need be
# 2006/03/30 DONT CHANGE THE FORMAT OF THIS because cgi script uses awk to find the second field
echo "JOBID_FROM_CLUSTER_SCRIPT= $jobid"
jobdir="$job.$clustername.\$SLURM_JOBID.$date_YYMMDD" # 2006/03/30 skk - there are two instance of this line unfortunately so
be careful if you change - todo
# 2006/03/30 skk - add the scratch dir name with the SCRATCHDIR_FROM_CLUSTER_SCRIPT= for cgi to parse easily
# 2006/03/30 DONT CHANGE THE FORMAT OF THIS because cgi script uses awk to find the second field
# 2006/03/30 now lets do the eval of $OutDir so that jobdir will be picked up
eval "echo \"SCRATCHDIR_FROM_CLUSTER_SCRIPT= $OutDir\" "
# 2006/10/06 chona: SEND THE DATE TO SERVER (indirectly)
### print the date included as part of $jobdir. The GMS_WS.Jobs.created
### field will be set equal to this. ***Do not change***
echo "JOB_SUBMITTED_DATE=$date_YYMMDD"
sleep 2
# 2005/11/14 sudhakar wants info only on the current job
# 2005/11/14 $qstat -u $LOGIN
echo "INFO: executing $qstat $jobid"; echo
$qstat $jobid
##mjs 2008/06/10
echo "#!/bin/ksh" > $qsubin_uberftp1
echo "jobdir=$jobdir" > $qsubin_uberftp1
echo "remotedir=mss/$remotedir" >> $qsubin_uberftp1
echo "OutDir=$OutDir" >> $qsubin_uberftp1
# 2008/09/26 skk moved here out of the if statement and improved
uberftp_failure_email="To: $userMail\nCc: $userMail_cc
Subject: UBERFTP_FAILED for local job id $jobid
UberFTP failed to transfer files to mass storage.
Please use \"Monitor Output\" function on MyCCG Panel to retrieve the output and
if it fails create a ticket with consulting portal.
DETAILS (for sysadmins mostly):
To retry sending the data to MSS we should use this command (after setting up appropriate credentials) - would need to remove
the last few lines in that script though starting with line '^||' :
$OutDir/$qsubin_uberftp_nodir
"
# 2008/09/26 i now use user ccguser@osc.edu instead of ccguser in sendmail -f, because otherwise mail was not accepted by ncsa
(for Sudhakar) due to eg "<ccguser@ipf258.ten.osc.edu> rejected address saying: <amantadine.ncsa.uiuc.edu[141.142.2.201]>:
Client host rejected: Access denied (in reply to MAIL FROM command)"
if [ -n "$DO_UBERFTP_ON_LOGINNODE" ] ; then
/bin/cat >> $qsubin_uberftp2 << EOF1
$uberftp 'mkdir $topdir_name' \" 1>/dev/null 2>&1
$uberftp 'mkdir $topdir_name/$GridchemUserid' \" 1>/dev/null 2>&1
$uberftp 'mkdir $topdir_name/$GridchemUserid/$ResearchProject' \" 1>/dev/null 2>&1
$uberftp 'mkdir \$remotedir' \" 1>/dev/null 2>&1
$uberftp 'lcd $OutDir; cd;cd $topdir_name/$GridchemUserid; \$uberftp_command_formated_chk \$uberftp_command_chk
cd;cd \$remotedir; mput * ; quit ' \"
$uberftp 'dir \$remotedir/\$qsubin_nodir' \" 2>/dev/null |grep -q "\$qsubin_nodir" \
|| echo -e "$uberftp_failure_email" | sendmail -f ccguser@osc.edu
EOF1
else
# 2005/10/11 skk - lcd $SCRATCH_BATCH may not be needed but for symmetry keep it there.
/bin/cat >> $qsubin_uberftp2 << EOF1
$gsissh "mkdir mss/$topdir_name" 1>/dev/null 2>&1
$gsissh "mkdir mss/$topdir_name/$GridchemUserid" 1>/dev/null 2>&1
$gsissh "mkdir mss/$topdir_name/$GridchemUserid/$ResearchProject" 1>/dev/null 2>&1
$gsiscp -r $OutDir $mss_machine:mss/$topdir_name/$GridchemUserid/$ResearchProject
EOF1
# Check if the chk files are ok to save before updating files in mss_machine
/bin/cat >> $qsubin_uberftp2 << EOF2
if chkchk $OutDir/$Chkfile_with_arch 2>/dev/null 1>/dev/null ; then
checkpoint_file_valid=0;
echo "WARNING: chkchk command succeeded and we will save it back to mss machine ";
$gsiscp $OutDir/$Chkfile_with_arch $mss_machine:mss/$topdir_name/$GridchemUserid;
$gsiscp $OutDir/$formated_chkfile $mss_machine:mss/$topdir_name/$GridchemUserid;
else
echo "WARNING: chkchk command failed - we will not save the check point file to mass storage machine ";
fi
EOF2
# $gsiscp $OutDir/$formated_chkfile $mss_machine:mss/$topdir_name/$GridchemUserid
# $gsiscp $OutDir/$Chkfile_with_arch $mss_machine:mss/$topdir_name/$GridchemUserid
/bin/cat >> $qsubin_uberftp2 << EOF1
$gsissh "ls \$remotedir/\$qsubin_nodir" 2>/dev/null |grep -q "\$qsubin_nodir" \
|| echo -e "$uberftp_failure_email" | sendmail -f ccguser@osc.edu
EOF1
fi
echo "jobdir=$jobdir" >> $qsubin_cleanup
/bin/cat >> $qsubin_cleanup << EOF
/bin/cat >> $qsubin_uberftp1 << EO1
uberftp_command_formated_chk="\$uberftp_command_formated_chk"
uberftp_command_chk="\$uberftp_command_chk"
qsubin_nodir=\$qsubin_nodir
EO1
EOF
echo "/bin/cat $qsubin_uberftp1 >> $qsubin_uberftp" >> $qsubin_cleanup
echo "/bin/cat $qsubin_uberftp2 >> $qsubin_uberftp" >> $qsubin_cleanup
echo "/bin/rm -f $qsubin_uberftp1" >> $qsubin_cleanup
echo "/bin/rm -f $qsubin_uberftp2" >> $qsubin_cleanup
echo ". $qsubin_uberftp" >> $qsubin_cleanup
echo "/bin/mv $qsubin_uberftp $OutDir/" >> $qsubin_cleanup
Overview
Content Tools
Apps
For an overview of Apache Airavata visit the Website - https://airavata.apache.org/