From 16bb4298358790dc0bc2bf7a0f9b9c0bd0160c68 Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Tue, 16 Jun 2026 10:22:34 +0200 Subject: [PATCH 1/2] Add --inputfile to pre-stage extra files via JDL InputFile --- GRID/utils/grid_submit.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/GRID/utils/grid_submit.sh b/GRID/utils/grid_submit.sh index dcf6f78da..5220ff5c3 100755 --- a/GRID/utils/grid_submit.sh +++ b/GRID/utils/grid_submit.sh @@ -219,6 +219,7 @@ while [ $# -gt 0 ] ; do --wait-any) WAITFORALIENANY=ON; WAITFORALIEN=ON; shift 1 ;; #wait for any good==done alien jobs to return --outputspec) OUTPUTSPEC=$2; shift 2 ;; #provide comma separate list of JDL file specs to be put as part of JDL Output field (example '"*.log@disk=1","*.root@disk=2"') --split-on-collection) DATACOLLECTION=$2; shift 2 ;; # this will split the jobs on InputDataCollection and "file" mode + --inputfile) INPUTFILES+=("$2"); shift 2 ;; # extra file to pre-stage via JDL InputFile (repeatable). Accepts a bare LFN or "LF:". Staged by AliEn before the job starts, so it is NOT charged to job CPU/walltime (unlike an in-job 'alien.py cp'). -h) Usage ; exit ;; --help) Usage ; exit ;; --fetch-output) FETCHOUTPUT=ON; shift 1 ;; # if to fetch all JOB output locally (to make this job as if it ran locally); only works when we block until all JOBS EXIT @@ -366,12 +367,20 @@ if [[ "${IS_ALIEN_JOB_SUBMITTER}" ]]; then if [ "${DATACOLLECTION}" ]; then SPLITMODE="file" fi + + # Assemble the JDL InputFile list: always the job script, plus any extra files + # requested via --inputfile (each pre-staged by AliEn before the job starts). + INPUTFILE_ENTRIES="${QUOT}LF:${MY_JOBWORKDIR}/alien_jobscript.sh${QUOT}" + for _extra in "${INPUTFILES[@]}"; do + [[ "${_extra}" == LF:* ]] || _extra="LF:${_extra}" + INPUTFILE_ENTRIES="${INPUTFILE_ENTRIES}, ${QUOT}${_extra}${QUOT}" + done # ---- Generate JDL ---------------- # TODO: Make this configurable or read from a preamble section in the jobfile cat > "${MY_JOBNAMEDATE}.jdl" < Date: Mon, 19 Oct 2026 18:33:53 +0200 Subject: [PATCH 2/2] grid_submit: wait until no subjob is active This fixes a problem in the --wait and --wait-any modes of grid_submit.sh. - AliEn can briefly report a masterjob as DONE while one of its subjobs in ERROR_A is put back to WAITING. - --wait read only the masterjob status and returned at that moment, although the job was then retried. - Both modes now finish only when no subjob is still active, with ERROR_A counted as active. - A failed or empty status query also counts as active. Co-Authored-By: Claude Opus 5.5 --- GRID/utils/grid_submit.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/GRID/utils/grid_submit.sh b/GRID/utils/grid_submit.sh index 5220ff5c3..5d11e1a2b 100755 --- a/GRID/utils/grid_submit.sh +++ b/GRID/utils/grid_submit.sh @@ -454,6 +454,13 @@ EOF fi + # true while a subjob is not final; ERROR_A (resubmitted) and a failed query count as active + subjobs_active() { + jq -e '(.results | length) == 0 or (.results | any(.status | IN("INSERTING", "SPLITTING", "WAITING", "ASSIGNED", "STARTED", "RUNNING", "SAVING", "SAVED", "ERROR_A")))' \ + <<<"$1" >/dev/null + [ $? -ne 1 ] + } + # wait here until all ALIEN jobs have returned spin[3]="-" spin[2]="/" @@ -488,12 +495,16 @@ EOF else # check if there are still jobs running/waiting; if not also finish # this could happen when all jobs are zombies (in which case we also finish) - if ! jq -e '.results | any(.status == "WAITING" or .status == "RUNNING" or .status == "SAVING" or .status == "INSERTING")' \ - <<<"${DETAILED_STATUS_JSON}" >/dev/null; then + if ! subjobs_active "${DETAILED_STATUS_JSON}"; then JOBSTATUS="D" # some job finished successfully echo "No remaining good job" fi fi + elif [ "${JOBSTATUS}" == "D" ]; then + # the masterjob can briefly show DONE while an ERROR_A subjob is resubmitted + if subjobs_active "$(ALIENPY_JSON=true alien.py ps -a -m "${MY_JOBID}")"; then + JOBSTATUS="S" + fi fi if [ "${JOBSTATUS}" == "D" ]; then