#!/bin/bash

DAX_FILE=""
CACHE_FILE=""
LOCAL_PEGASUS_DIR=""
HOSTS=""
DEVELOPMENT=""

GETOPT_CMD=`getopt -o d:n:Dc:a:h:l --long dax:,number-of-hosts:,development,cache-file:,accounting-group:,local-dir:,help -n 'pycbc_submit_dax' -- "$@"`
eval set -- "$GETOPT_CMD"

while true ; do
  case "$1" in
    -d|--dax)
      case "$2" in
        "") shift 2 ;;
        *) DAX_FILE=$2 ; shift 2 ;;
      esac ;;
    -n|--number-of-hosts)
      case "$2" in
        "") shift 2 ;;
        *) HOSTS=$2 ; shift 2 ;;
      esac ;;
    -D|--development) DEVELOPMENT="-devel" ; shift ;;
    -c|--cache-file)
      case "$2" in
        "") shift 2 ;;
        *) CACHE_FILE=$2 ; shift 2 ;;
      esac ;;
    -a|--accounting-group)
      case "$2" in
        "") shift 2 ;;
        *) ACCOUNTING_GROUP=$2 ; shift 2 ;;
      esac ;;
    -l|--local-dir)
      case "$2" in
        "") shift 2 ;;
        *) LOCAL_PEGASUS_DIR=$2 ; shift 2 ;;
      esac ;;
    -h|--help)
      echo "usage: pycbc_submit_dax [-h]"
      echo "                        --dax DAX"
      echo "                        [--cache-file FILE]"
      echo
      echo "required arguments:"
      echo "  -d, --dax DAX           name of the dax file to plan"
      echo "  -n, --number-of-hosts   set the number of hosts to use on stampede"
      echo
      echo "optional arguments:"
      echo "  -h, --help              show this help message and exit"
      echo "  -c, --cache-file FILE   replica cache file for data reuse"
      echo "  -D, --development       use the stampede development queue rather"
      echo "  -a, --accounting-group GROUP tagged string used for site "
      echo "                               resource accounting."
      echo " "
      echo "If the environment variable TMPDIR is set then this is prepended to the "
      echo "path to the temporary workflow execte directory passed to pegasus-plan."
      echo
      exit 0 ;;
    --) shift ; break ;;
    *) echo "Internal error!" ; exit 1 ;;
  esac
done

if [ "x$DAX_FILE" == "x" ]; then
  echo "Error: --dax must be specified. Use --help for options."
   exit 1
fi

if [ "x$HOSTS" == "x" ]; then
  echo "Error: --number-of-hosts must be specified. Use --help for options."
   exit 1
fi

set -e

# Redirect stdout ( > ) into a named pipe ( >() ) running "tee"
exec > >(tee submit.log)
exec 2>&1

# Check I have two arguments supplied
if [ "x$2" == "x" ]; then
  echo "I take two arguments, the name of the dax file and the number of hosts to request"
  exit 1
fi

echo 'making a directory for the submit files'
export SUBMIT_DIR=`mktemp --tmpdir=${LOCAL_PEGASUS_DIR} -d pycbc-tmp.XXXXXXXXXX`
echo $SUBMIT_DIR

#Make sure the directory is world readable
chmod 755 $SUBMIT_DIR

echo 'finding remote scratch directory'
export SCRATCH=`gsissh -p 2222 login5.stampede.tacc.utexas.edu 'echo $SCRATCH'`/pycbc_workflow_$RANDOM
echo $SCRATCH

echo 'finding the remote pegasus home'
#PEGASUS_HOME=`gsissh -p 2222 login5.stampede.tacc.utexas.edu 'which pegasus_submit_dax'`
export PEGASUS_HOME=/home1/02969/ahnitz/pegasus-4.5.0
echo $PEGASUS_HOME

echo 'finding the site-local template directory'
PEG_DIR=`python -c 'from pycbc.workflow import PEGASUS_FILE_DIRECTORY;print PEGASUS_FILE_DIRECTORY'`
SITE_TEMPLATE=$PEG_DIR/xsede_site_template.xml
echo $SITE_TEMPLATE

export ACCOUNTING_GROUP
export HOSTS

# Set up the site-local with the correct paths
echo 'cat <<END_OF_TEXT' >  temp.sh
cat $SITE_TEMPLATE       >> temp.sh
echo 'END_OF_TEXT'       >> temp.sh
bash temp.sh > site-local-parsed.xml

# Plan the workflow
echo "Generating concrete workflow"
touch _reuse.cache

if [ "x$3" == "x--cache" ]; then
  cp $3 _reuse.cache
  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $1 --sites stampede${DEVELOPMENT},local -o local \
               --dir $SUBMIT_DIR --cleanup inplace --cluster label \
               --cache _reuse.cache --relative-submit-dir work --submit
else
  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $1 --sites stampede${DEVELOPMENT},local -o local \
               --dir $SUBMIT_DIR --cleanup inplace --cluster label --relative-submit-dir work --submit
fi

ln -s $SUBMIT_DIR submitdir

echo pegasus-status $SUBMIT_DIR/work > status
chmod 755 status

echo pegasus-analyzer $SUBMIT_DIR/work > debug
chmod 755 debug

echo pegasus-remove $SUBMIT_DIR/work > stop
chmod 755 stop

echo pegasus-run $SUBMIT_DIR/work > start
chmod 755 start

echo 'gsissh -p 2222 login5.stampede.tacc.utexas.edu -t "cd $SCRATCH; bash --login"' > remote_work
chmod 755 remote_work

echo Submitting from: $HOSTNAME
