grid_submit: pre-stage extra input files and fix --wait

GRID/utils/grid_submit.sh

@@ -219,6 +219,7 @@ while [ $# -gt 0 ] ; do
         --wait-any) WAITFORALIENANY=ON; WAITFORALIEN=ON; shift 1 ;; #wait for any good==done alien jobs to return
         --outputspec) OUTPUTSPEC=$2; shift 2 ;; #provide comma separate list of JDL file specs to be put as part of JDL Output field (example '"*.log@disk=1","*.root@disk=2"')
         --split-on-collection) DATACOLLECTION=$2; shift 2 ;; # this will split the jobs on InputDataCollection and "file" mode
+        --inputfile) INPUTFILES+=("$2"); shift 2 ;; # extra file to pre-stage via JDL InputFile (repeatable). Accepts a bare LFN or "LF:<lfn>". Staged by AliEn before the job starts, so it is NOT charged to job CPU/walltime (unlike an in-job 'alien.py cp').
         -h) Usage ; exit ;;
         --help) Usage ; exit ;;
         --fetch-output) FETCHOUTPUT=ON; shift 1 ;; # if to fetch all JOB output locally (to make this job as if it ran locally); only works when we block until all JOBS EXIT
@@ -366,12 +367,20 @@ if [[ "${IS_ALIEN_JOB_SUBMITTER}" ]]; then
   if [ "${DATACOLLECTION}" ]; then
     SPLITMODE="file"
   fi
+
+  # Assemble the JDL InputFile list: always the job script, plus any extra files
+  # requested via --inputfile (each pre-staged by AliEn before the job starts).
+  INPUTFILE_ENTRIES="${QUOT}LF:${MY_JOBWORKDIR}/alien_jobscript.sh${QUOT}"
+  for _extra in "${INPUTFILES[@]}"; do
+    [[ "${_extra}" == LF:* ]] || _extra="LF:${_extra}"
+    INPUTFILE_ENTRIES="${INPUTFILE_ENTRIES}, ${QUOT}${_extra}${QUOT}"
+  done
   # ---- Generate JDL ----------------
   # TODO: Make this configurable or read from a preamble section in the jobfile
   cat > "${MY_JOBNAMEDATE}.jdl" <<EOF
 Executable = "${MY_BINDIR}/${MY_JOBNAMEDATE}.sh";
 Arguments = "${CONTINUE_WORKDIR:+"-c ${CONTINUE_WORKDIR}"} --local ${O2TAG:+--o2tag ${O2TAG}} --ttl ${JOBTTL} --label ${JOBLABEL:-label} --prodsplit ${PRODSPLIT} ${MATTERMOSTHOOK:+--mattermost ${MATTERMOSTHOOK}} ${CONTROLSERVER:+--controlserver ${CONTROLSERVER}}";
-InputFile = "LF:${MY_JOBWORKDIR}/alien_jobscript.sh";
+InputFile = { ${INPUTFILE_ENTRIES} };
 ${DATACOLLECTION:+InputDataList = ${QUOT}input.list${QUOT};}
 ${DATACOLLECTION:+InputDataListFormat = ${QUOT}txt-list${QUOT};}
 ${DATACOLLECTION:+InputDataCollection = ${QUOT}LF:${MY_JOBWORKDIR}/collection.xml,nodownload${QUOT};}
@@ -445,6 +454,13 @@ EOF
   fi
 
 
+  # true while a subjob is not final; ERROR_A (resubmitted) and a failed query count as active
+  subjobs_active() {
+    jq -e '(.results | length) == 0 or (.results | any(.status | IN("INSERTING", "SPLITTING", "WAITING", "ASSIGNED", "STARTED", "RUNNING", "SAVING", "SAVED", "ERROR_A")))' \
+      <<<"$1" >/dev/null
+    [ $? -ne 1 ]
+  }
+
   # wait here until all ALIEN jobs have returned
   spin[3]="-"
   spin[2]="/"
@@ -479,12 +495,16 @@ EOF
       else
         # check if there are still jobs running/waiting; if not also finish
         # this could happen when all jobs are zombies (in which case we also finish)
-        if ! jq -e '.results | any(.status == "WAITING" or .status == "RUNNING" or .status == "SAVING" or .status == "INSERTING")' \
-            <<<"${DETAILED_STATUS_JSON}" >/dev/null; then
+        if ! subjobs_active "${DETAILED_STATUS_JSON}"; then
               JOBSTATUS="D"  # some job finished successfully
               echo "No remaining good job"
         fi
       fi
+    elif [ "${JOBSTATUS}" == "D" ]; then
+      # the masterjob can briefly show DONE while an ERROR_A subjob is resubmitted
+      if subjobs_active "$(ALIENPY_JSON=true alien.py ps -a -m "${MY_JOBID}")"; then
+        JOBSTATUS="S"
+      fi
     fi
 
     if [ "${JOBSTATUS}" == "D" ]; then