#!/bin/bash
# Redirect stdout ( > ) into a named pipe ( >() ) running "tee"
set -e

exec > >(tee submit.log)

# Without this, only stdout would be captured - i.e. your
# log file would not contain any error messages.
# SEE answer by Adam Spiers, which keeps STDERR a seperate stream -
# I did not want to steal from him by simply adding his answer to mine.
exec 2>&1

DAX_FILE=""
CACHE_FILE=""
LOCAL_PEGASUS_DIR=""

GETOPT_CMD=`getopt -o d:c:a:h:l --long dax:,cache-file:,accounting-group:,local-dir:,help -n 'pycbc_submit_dax' -- "$@"`
eval set -- "$GETOPT_CMD"

while true ; do
  case "$1" in
    -d|--dax)
      case "$2" in
        "") shift 2 ;;
        *) DAX_FILE=$2 ; shift 2 ;;
      esac ;;
    -c|--cache-file)
      case "$2" in
        "") shift 2 ;;
        *) CACHE_FILE=$2 ; shift 2 ;;
      esac ;;
    -a|--accounting-group)
      case "$2" in
        "") shift 2 ;;
        *) ACCOUNTING_GROUP=$2 ; shift 2 ;;
      esac ;;
    -l|--local-dir)
      case "$2" in
        "") shift 2 ;;
        *) LOCAL_PEGASUS_DIR=$2 ; shift 2 ;;
      esac ;;
    -h|--help)
      echo "usage: pycbc_submit_dax [-h]"
      echo "                        --dax DAX"
      echo "                        [--cache-file FILE]"
      echo
      echo "required arguments:"
      echo "  -d, --dax DAX           name of the dax file to plan"
      echo
      echo "optional arguments:"
      echo "  -h, --help              show this help message and exit"
      echo "  -c, --cache-file FILE   replica cache file for data reuse"
      echo "  -a, --accounting-group GROUP tagged string used for site "
      echo "                               resource accounting."
      echo "  -l, --local-dir         Directory to put condor files under. "
      echo
      echo "If the environment variable TMPDIR is set then this is prepended to the "
      echo "path to the temporary workflow execte directory passed to pegasus-plan."
      echo "If the --local-dir option is not given."
      echo
      exit 0 ;;
    --) shift ; break ;;
    *) echo "Internal error!" ; exit 1 ;;
  esac
done

if [ "x$DAX_FILE" == "x" ]; then
  echo "Error: --dax must be specified. Use --help for options."
   exit 1
fi

#Make a directory for the submit files
SUBMIT_DIR=`mktemp --tmpdir=${LOCAL_PEGASUS_DIR} -d pycbc-tmp.XXXXXXXXXX`

#Make sure the directory is world readable
chmod 755 $SUBMIT_DIR

# find the site-local template directory
PEG_DIR=`python -c 'from pycbc.workflow import PEGASUS_FILE_DIRECTORY;print PEGASUS_FILE_DIRECTORY'`
SITE_TEMPLATE=$PEG_DIR/basic_site_template.xml

export ACCOUNTING_GROUP

# Set up the site-local with the correct paths
echo 'cat <<END_OF_TEXT' >  temp.sh
cat $SITE_TEMPLATE       >> temp.sh
echo 'END_OF_TEXT'       >> temp.sh
bash temp.sh > site-local-parsed.xml

# Plan the workflow
echo "Generating concrete workflow"
touch _reuse.cache

if [ "x$CACHE_FILE" == "x" ]; then
  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $DAX_FILE --sites local -o local --dir $SUBMIT_DIR --cleanup inplace --relative-submit-dir work --cluster label --submit 
else
  cp $CACHE_FILE _reuse.cache
  pegasus-plan --conf $PEG_DIR/basic_pegasus.conf -d $DAX_FILE --sites local -o local --dir $SUBMIT_DIR --cleanup inplace --cache _reuse.cache --relative-submit-dir work --cluster label --submit
fi

ln -s $SUBMIT_DIR submitdir

echo pegasus-status $SUBMIT_DIR/work > status
chmod 755 status

echo pegasus-analyzer $SUBMIT_DIR/work > debug
chmod 755 debug

echo pegasus-remove $SUBMIT_DIR/work > stop
chmod 755 stop

echo pegasus-run $SUBMIT_DIR/work > start
chmod 755 start

echo Submitting from: $HOSTNAME
