#!/bin/csh -f
##============================================================================
## This is a CCSM batch job script for phoenix
##============================================================================
## BATCH QUEUE INFO
## Job requires 56 tasks; 56 requested (must be a factor of 8)
#PBS -A CLI017bgc
#PBS -N i01.10cn
##PBS -q debug
#PBS -l walltime=12:00:00,mppe=56
#PBS -W group_list=cli017,umask=007
#PBS -j oe
#PBS -m abe
#PBS -M forrest@climatemodeling.org,8656843670@messaging.sprintpcs.com
##PBS -M forrest@climatemodeling.org,fmhoffman@ipipi.com

#-----------------------------------------------------------------------
# Resolved task/thread counts
# This is provider as user information only
# These are csh comments, DO NOT UNCOMMENT
#-----------------------------------------------------------------------
### COMPONENTS = ( cpl xice clm xocn datm )
### NTASKS_CPL=32     NTHRDS_CPL=1
### NTASKS_ICE=1     NTHRDS_ICE=1
### NTASKS_LND=21     NTHRDS_LND=1
### NTASKS_OCN=1     NTHRDS_OCN=1
### NTASKS_ATM=1     NTHRDS_ATM=1
 

#-----------------------------------------------------------------------
# Determine necessary environment variables
#-----------------------------------------------------------------------

cd /autofs/spin/home/hof/CCSM/ccsm3_1_beta40.bgcmip/scripts/i01.10cn
setenv MACH phoenix

source env_conf         || "problem sourcing env_conf"    && exit -1  
source env_run          || "problem sourcing env_run"     && exit -1  
source env_mach.$MACH   || "problem sourcing env_mach.$MACH" && exit -1
source env_pes.$MACH    || "problem sourcing env_pes.$MACH" && exit -1
source /autofs/spin/home/hof/CCSM/ccsm3_1_beta40.bgcmip/scripts/i01.10cn/Tools/env_setcomp  || "problem sourcing env_setcomp"   && exit -1

#-----------------------------------------------------------------------
# Determine time-stamp/file-ID string
# Clean up previous run timing files
#-----------------------------------------------------------------------
       
setenv LID "`date +%y%m%d-%H%M%S`"
rm -f /tmp/work/hof/i01.10cn/*/timing.[0-9]*    >& /dev/null
rm -f /tmp/work/hof/i01.10cn/*/spmdstats.[0-9]* >& /dev/null

# -------------------------------------------------------------------------
# Build the models
# -------------------------------------------------------------------------

module list
./${CASE}.${MACH}.build || exit 1

# -------------------------------------------------------------------------
# Set up the model.  For the single executable, this means determining the
# total number of tasks.  For the multiple executable, this means linking
# each executable to the execution directory.
# -------------------------------------------------------------------------

if ($SINGLE_EXEC == 'TRUE') then
   @ tasks = 0
   foreach n (1 2 3 4 5)
      @ tasks = $tasks + $NTASKS[$n]
   end
endif

if ($SINGLE_EXEC == 'FALSE') then
   foreach n (1 2 3 4 5)
      ln -s $EXEROOT/$MODELS[$n]/$COMPONENTS[$n] $EXEROOT/all/.
   end
endif

env | egrep '(MP_|LOADL|XLS|FPE|DSM|OMP|MPC)' # document env vars

# -------------------------------------------------------------------------
# Run the model
# -------------------------------------------------------------------------

cd $EXEROOT/all
echo "`date` -- CSM EXECUTION BEGINS HERE" 
if ($SINGLE_EXEC == 'TRUE') then
   mpirun -c core=unlimited -np ${tasks} ./ccsm_se.exe
else
   mpirun -c core=unlimited -np $NTASKS[1] ./$COMPONENTS[1] : -np $NTASKS[2] ./$COMPONENTS[2] : -np $NTASKS[3] ./$COMPONENTS[3] : -np $NTASKS[4] ./$COMPONENTS[4] : -np $NTASKS[5] ./$COMPONENTS[5]
endif

wait
echo "`date` -- CSM EXECUTION HAS FINISHED" 

# -------------------------------------------------------------------------
# Save model output stdout and stderr 
# -------------------------------------------------------------------------

cd $EXEROOT/cpl 
set CplLogFile = `ls -1t cpl.log* | head -1` 
grep 'end of main program' $CplLogFile   || echo "Model did not complete - see $CplLogFile" && exit -1

cd $EXEROOT
gzip */*.$LID

if ($LOGDIR != "") then
  if (! -d $LOGDIR/bld) mkdir -p $LOGDIR/bld || echo " problem in creating $LOGDIR/bld" && exit -1
  cp -p */*buildexe*$LID.* $LOGDIR/bld   || echo "Error in copy of logs " && exit -1
  cp -p */*log*$LID.*      $LOGDIR       || echo "Error in copy of logs " && exit -1
endif

# -------------------------------------------------------------------------
# Perform short term archiving of output
# -------------------------------------------------------------------------

if ($DOUT_S == 'TRUE') then
  echo "Archiving ccsm output to $DOUT_S_ROOT"
  echo "In $CASEROOT directory using the short term archiving script ccsm_s_archive.csh"
  cd $CASEROOT; $UTILROOT/Tools/ccsm_s_archive.csh
endif

# -------------------------------------------------------------------------
# Submit longer term archiver if appropriate
# -------------------------------------------------------------------------

if ($DOUT_L_MS == 'TRUE' && $DOUT_S == 'TRUE') then
  cd $CASEROOT
  if (-f $CASE.$MACH.l_archive) then 	  
     #echo "Long term archiving ccsm output using the script $CASE.$MACH.l_archive"
     #FILL THIS IN#
  endif 
endif

# -------------------------------------------------------------------------
# Resubmit another run script
# -------------------------------------------------------------------------

set echo
cd $CASEROOT
source env_run
if ($RESUBMIT > 0) then
    echo RESUBMIT is $RESUBMIT
    @ RESUBMIT = $RESUBMIT - 1
    echo RESUBMIT is $RESUBMIT

    sed 's/^ *setenv *CONTINUE_RUN .*/setenv CONTINUE_RUN TRUE/;' \
         env_run > env_run.tmp; mv env_run.tmp env_run
    sed "s/^ *setenv *RESUBMIT .*/setenv RESUBMIT $RESUBMIT/;" \
         env_run > env_run.tmp; mv env_run.tmp env_run

    qsub ${CASE}.${MACH}.run
  endif
endif

cd $CASEROOT
source env_run
if ($CHECK_TIMING == 'TRUE') then
  $UTILROOT/Tools/timing/getTiming.csh -lid $LID -mach $MACH
  if !(-d timing) mkdir timing
  mv timing*.$LID* timing/
endif