From 62220e6833f2f619f8a713cfc7a42fe1a1a55c6e Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Sun, 4 Oct 2026 08:06:42 +0200 Subject: [PATCH 1/2] Use requested cuts and epochs and enforce lookup-test outcomes --- .../lookuptable-testing/test_tables.sh | 57 +++++++++---------- .../montecarlo/irf_plotting/irf_plotting.sh | 44 ++++++++------ tests/test_review_release_coverage.py | 32 +++++++++++ 3 files changed, 86 insertions(+), 47 deletions(-) create mode 100644 tests/test_review_release_coverage.py diff --git a/release_tests/lookuptable-testing/test_tables.sh b/release_tests/lookuptable-testing/test_tables.sh index 4ee42d8..3f81733 100755 --- a/release_tests/lookuptable-testing/test_tables.sh +++ b/release_tests/lookuptable-testing/test_tables.sh @@ -1,32 +1,29 @@ #!/bin/bash -# testing lookup tables by analysis -# a run and see if the table tests are -# survived at the start of the run -# (look at the log files for the lines after -# " lookup table file sanity check" -# - -VERSION=$(cat $VERITAS_EVNDISP_AUX_DIR/IRFVERSION) - -TFV6=$(find $VERITAS_EVNDISP_AUX_DIR/Tables/ -name "*.root") - -TESTFILE="$VERITAS_USER_DATA_DIR/analysis/Results/${VERSION}/Crab/evndisp/64080.root" - -for T in ${TFV6} -do - echo "Running mscw analysis for $T" - TB=$(basename $T .root) - - ODIR="$VERITAS_USER_DATA_DIR/analysis/Results/${VERSION}/table-tests/" - mkdir -p ${ODIR} - OFIL="test.$TB.mscw" - - echo $TB $OFIL $ODIR - - rm -f ${ODIR}/${OFIL}.log - ${EVNDISPSYS}/bin/mscw_energy -tablesfile ${T} -noshorttree -maxnevents=10 -arrayrecid=0 \ - -inputfile ${TESTFILE} \ - -writeReconstructedEventsOnly=1 -outputfile ${ODIR}/${OFIL}.root > ${ODIR}/${OFIL}.log - # (not interested in the root file with events) - rm -f ${ODIR}/${OFIL}.root +# Supply a reference observation matched to the tables under test. +VERSION=$(cat "$VERITAS_EVNDISP_AUX_DIR/IRFVERSION") || exit 1 +TESTFILE=${1:-${TESTFILE:-}} +[[ -s $TESTFILE ]] || { echo "Usage: $0 (or set TESTFILE)" >&2; exit 1; } +ODIR="$VERITAS_USER_DATA_DIR/analysis/Results/$VERSION/table-tests" +mkdir -p "$ODIR" || exit 1 +mapfile -t TABLES < <(find "$VERITAS_EVNDISP_AUX_DIR/Tables" -type f -name "${TABLE_PATTERN:-*.root}") +((${#TABLES[@]})) || { echo "No lookup tables found" >&2; exit 1; } +failed=0 +for T in "${TABLES[@]}"; do + TB=$(basename "$T" .root) + OFIL="$ODIR/test.$TB.mscw" + echo "TEST reference=$TESTFILE table=$T" + rm -f "$OFIL.root" + if ! "$EVNDISPSYS/bin/mscw_energy" -tablesfile "$T" -noshorttree -maxnevents=10 -arrayrecid=0 \ + -inputfile "$TESTFILE" -writeReconstructedEventsOnly=1 -outputfile "$OFIL.root" > "$OFIL.log" 2>&1; then + echo "FAILED executable: $T" >&2 + failed=$((failed+1)) + elif [[ ! -s $OFIL.root ]] || ! grep -q 'survived test of table file!' "$OFIL.log"; then + echo "FAILED missing output/sanity marker: $T" >&2 + failed=$((failed+1)) + else + echo "PASSED table sanity: $T" + rm -f "$OFIL.root" + fi done +echo "Lookup coverage: ${#TABLES[@]} tested; $failed failed" +((failed == 0)) diff --git a/release_tests/montecarlo/irf_plotting/irf_plotting.sh b/release_tests/montecarlo/irf_plotting/irf_plotting.sh index 7a41c35..4c096e9 100755 --- a/release_tests/montecarlo/irf_plotting/irf_plotting.sh +++ b/release_tests/montecarlo/irf_plotting/irf_plotting.sh @@ -13,18 +13,14 @@ if [ ! -n "$1" ]; then exit fi -#### TEMP FIXED VALUES -CUT="NTel3-PointSource-Hard-TMVA-BDT" -CUT="NTel2-PointSource-Soft-TMVA-BDT" -CUT="NTel2-PointSource-Moderate-TMVA-BDT" -CUT="NTel2-PointSource-Moderate" -echo "WARNING: CUT is hardwired to $CUT (for now)" -# Comparison plots - version and simtype hardwired -COMPAREVERSION="v492" -COMPARESIMTYPE="CARE_202404" -COMPARECUT="NTel2-PointSource-ModerateXGB" -#### (END TEMP FIXED VALUES) - +# Optional comparison is explicit in the invocation environment. +COMPAREVERSION=${COMPAREVERSION:-} +COMPARESIMTYPE=${COMPARESIMTYPE:-} +COMPARECUT=${COMPARECUT:-} +mapfile -t CUTLIST < <(awk '$1 == "*" && $2 == "CUT" {print $3}' "$1") +((${#CUTLIST[@]})) || { echo "No CUT entries in $1" >&2; exit 1; } +TESTED=0 +SKIPPED=0 # Analysis type ANATYPE="${VERITAS_ANALYSIS_TYPE:0:2}" # Eventdisplay version @@ -55,11 +51,10 @@ if [[ ! -z $VERITAS_ANALYSIS_TYPE ]]; then fi fi -ODIR="../../../../EventDisplay_Release_${VERSION}/irf_plotting/${ANALYSISTYPE}${DIRRECOTYPE}/${SIMTYPE}/${CUT}_ATM${ATMO}" -mkdir -p ${ODIR} - -DDIR="$VERITAS_IRFPRODUCTION_DIR/${VERSION}/${ANATYPE}/${SIMTYPE}" +DDIR="$VERITAS_IRFPRODUCTION_DIR/${VERSION}/${ANALYSISTYPE}/${SIMTYPE}" +for CUT in "${CUTLIST[@]}"; do +PLOT_BASE_DIR="${RELEASE_OUTPUT_DIR:-../../../../EventDisplay_Release_${VERSION}/irf_plotting}/${ANALYSISTYPE}${DIRRECOTYPE}/${SIMTYPE}/${CUT}" for Z in "${MCZE[@]}" do for W in "${MCWOFF[@]}" @@ -73,12 +68,23 @@ do fi for A in "${ATMO[@]}" do - if [[ ${E} == "V6_2016_2017" ]] && [[ ${A} == "61" ]]; then + ODIR="$PLOT_BASE_DIR/${E}_ATM${A}" + mkdir -p "$ODIR" + if true; then IRFDIR="${DDIR}/${E}_ATM${A}_gamma/EffectiveAreas_Cut-${CUT}_DISP" IRFFILE="EffArea-${SIMTYPE}-${E}-ID0-Ze${Z}deg-${W}wob-${N}-Cut-${CUT}" echo "IRFDIR $IRFDIR" echo "IRFFILE $IRFFILE" + if [[ ! -s "$IRFDIR/$IRFFILE.root" ]]; then + echo "SKIPPED missing $E ATM$A $CUT Ze$Z Woff$W NSB$N: $IRFDIR/$IRFFILE.root" + SKIPPED=$((SKIPPED+1)) + continue + fi + echo "TESTED $E ATM$A $CUT Ze$Z Woff$W NSB$N" + TESTED=$((TESTED+1)) if [[ -n $COMPAREVERSION ]]; then + COMPARESIMTYPE=${COMPARESIMTYPE:-$SIMTYPE} + COMPARECUT=${COMPARECUT:-$CUT} COMP_IRFDIR=${IRFDIR//"$VERSION"/"$COMPAREVERSION"} COMP_IRFDIR=${COMP_IRFDIR//"$SIMTYPE"/"$COMPARESIMTYPE"} COMP_IRFFILE=${IRFFILE//"$SIMTYPE"/"$COMPARESIMTYPE"} @@ -97,3 +103,7 @@ do done done done + +done +echo "IRF coverage: $TESTED tested; $SKIPPED missing combinations skipped" +((TESTED > 0)) || exit 1 diff --git a/tests/test_review_release_coverage.py b/tests/test_review_release_coverage.py new file mode 100644 index 0000000..f4e1da1 --- /dev/null +++ b/tests/test_review_release_coverage.py @@ -0,0 +1,32 @@ +from pathlib import Path +import os,subprocess,tempfile +ROOT=Path(__file__).resolve().parents[2] +def exe(p,s): + p.write_text("#!/bin/bash\n"+s);p.chmod(0o755) +with tempfile.TemporaryDirectory() as td: + d=Path(td);(d/"bin").mkdir();(d/"aux").mkdir() + env=os.environ.copy();env.update(PATH=str(d/"bin")+":"+env["PATH"],VERITAS_ANALYSIS_TYPE="AP_DISP",EVNDISPSYS=td,VERITAS_EVNDISP_AUX_DIR=str(d/"aux"),VERITAS_USER_DATA_DIR=td) + # Plotting coverage: every supplied cut x epoch x atmosphere must invoke ROOT. + (d/'params').write_text('* VERSION v492\n* SIMTYPE SIM\n* EPOCH V6_2024_2025w\n* EPOCH V6_2025_2025s\n* ATMOSPHERE 61\n* ATMOSPHERE 62\n* CUT Soft soft\n* CUT Hard hard\n* MC_ZE 20\n* MC_WOFF 0.5\n* MC_NSB 100\n* MC_AZ 0\n') + for cut in ['Soft','Hard']: + for epoch in ['V6_2024_2025w','V6_2025_2025s']: + for at in [61,62]: + p=d/f'irfs/v492/AP/SIM/{epoch}_ATM{at}_gamma/EffectiveAreas_Cut-{cut}_DISP';p.mkdir(parents=True);(p/f'EffArea-SIM-{epoch}-ID0-Ze20deg-0.5wob-100-Cut-{cut}.root').write_text('root') + exe(d/'bin/root', 'echo "$*" >> "$ROOT_TRACE"\npython - "$@" <<\'PY\'\nimport re, sys\nfrom pathlib import Path\narguments = re.findall(r\'"([^"]*)"\', sys.argv[-1])\noutput = Path(arguments[9]) / (arguments[1] + ".png")\noutput.write_text(arguments[3])\nPY\n');env.update(VERITAS_IRFPRODUCTION_DIR=str(d/'irfs'),RELEASE_OUTPUT_DIR=str(d/'plots'),ROOT_TRACE=str(d/'root.trace')) + p=ROOT/'EventDisplay_ReleaseTests_code/release_tests/montecarlo/irf_plotting/irf_plotting.sh' + res=subprocess.run(['bash',str(p),str(d/'params')],env=env,cwd=d,stdout=subprocess.PIPE,text=True,stderr=subprocess.STDOUT);assert res.returncode==0,res.stdout + assert len((d/'root.trace').read_text().splitlines())==8,res.stdout + assert '8 tested; 0' in res.stdout,res.stdout + plots = list((d/'plots').rglob('*.png')) + assert len(plots) == 8, 'Atmosphere-specific plots were overwritten' + assert {plot.read_text() for plot in plots} == {'61', '62'} + for plot in plots: + assert ('ATM'+plot.read_text()) in str(plot.parent) + print('PASS release plotting covers 2 cuts x 2 epochs x 2 atmospheres') + # Lookup sanity checker requires successful executable AND marker AND output. + aux=d/'aux';(aux/'IRFVERSION').write_text('v492\n');(aux/'Tables').mkdir();(aux/'Tables/table.root').write_text('table');(d/'reference.root').write_text('events') + exe(d/'bin/mscw_energy','[[ $MODE == producer-fail ]] && exit 9\nfor arg in "$@";do case $arg in -outputfile) next=1;;*) if [[ $next == 1 ]]; then [[ $MODE != missing ]] && printf new > "$arg";next=0;fi;;esac;done\n[[ $MODE != log-fail ]] && echo "...survived test of table file!"\nexit 0\n') + p=ROOT/'EventDisplay_ReleaseTests_code/release_tests/lookuptable-testing/test_tables.sh' + for mode in ['producer-fail','missing','log-fail','success']: + env['MODE']=mode;res=subprocess.run(['bash',str(p),str(d/'reference.root')],env=env,cwd=d,stdout=subprocess.PIPE,text=True,stderr=subprocess.STDOUT);assert (res.returncode==0)==(mode=='success'),(mode,res.stdout) + print('PASS lookup executable/missing-output/missing-sanity-marker failures') From b6a0c44da13840c05b2b4034ef62392d41a7ee90 Mon Sep 17 00:00:00 2001 From: GernotMaier Date: Sun, 4 Oct 2026 19:49:46 +0200 Subject: [PATCH 2/2] not needed --- tests/test_review_release_coverage.py | 32 --------------------------- 1 file changed, 32 deletions(-) delete mode 100644 tests/test_review_release_coverage.py diff --git a/tests/test_review_release_coverage.py b/tests/test_review_release_coverage.py deleted file mode 100644 index f4e1da1..0000000 --- a/tests/test_review_release_coverage.py +++ /dev/null @@ -1,32 +0,0 @@ -from pathlib import Path -import os,subprocess,tempfile -ROOT=Path(__file__).resolve().parents[2] -def exe(p,s): - p.write_text("#!/bin/bash\n"+s);p.chmod(0o755) -with tempfile.TemporaryDirectory() as td: - d=Path(td);(d/"bin").mkdir();(d/"aux").mkdir() - env=os.environ.copy();env.update(PATH=str(d/"bin")+":"+env["PATH"],VERITAS_ANALYSIS_TYPE="AP_DISP",EVNDISPSYS=td,VERITAS_EVNDISP_AUX_DIR=str(d/"aux"),VERITAS_USER_DATA_DIR=td) - # Plotting coverage: every supplied cut x epoch x atmosphere must invoke ROOT. - (d/'params').write_text('* VERSION v492\n* SIMTYPE SIM\n* EPOCH V6_2024_2025w\n* EPOCH V6_2025_2025s\n* ATMOSPHERE 61\n* ATMOSPHERE 62\n* CUT Soft soft\n* CUT Hard hard\n* MC_ZE 20\n* MC_WOFF 0.5\n* MC_NSB 100\n* MC_AZ 0\n') - for cut in ['Soft','Hard']: - for epoch in ['V6_2024_2025w','V6_2025_2025s']: - for at in [61,62]: - p=d/f'irfs/v492/AP/SIM/{epoch}_ATM{at}_gamma/EffectiveAreas_Cut-{cut}_DISP';p.mkdir(parents=True);(p/f'EffArea-SIM-{epoch}-ID0-Ze20deg-0.5wob-100-Cut-{cut}.root').write_text('root') - exe(d/'bin/root', 'echo "$*" >> "$ROOT_TRACE"\npython - "$@" <<\'PY\'\nimport re, sys\nfrom pathlib import Path\narguments = re.findall(r\'"([^"]*)"\', sys.argv[-1])\noutput = Path(arguments[9]) / (arguments[1] + ".png")\noutput.write_text(arguments[3])\nPY\n');env.update(VERITAS_IRFPRODUCTION_DIR=str(d/'irfs'),RELEASE_OUTPUT_DIR=str(d/'plots'),ROOT_TRACE=str(d/'root.trace')) - p=ROOT/'EventDisplay_ReleaseTests_code/release_tests/montecarlo/irf_plotting/irf_plotting.sh' - res=subprocess.run(['bash',str(p),str(d/'params')],env=env,cwd=d,stdout=subprocess.PIPE,text=True,stderr=subprocess.STDOUT);assert res.returncode==0,res.stdout - assert len((d/'root.trace').read_text().splitlines())==8,res.stdout - assert '8 tested; 0' in res.stdout,res.stdout - plots = list((d/'plots').rglob('*.png')) - assert len(plots) == 8, 'Atmosphere-specific plots were overwritten' - assert {plot.read_text() for plot in plots} == {'61', '62'} - for plot in plots: - assert ('ATM'+plot.read_text()) in str(plot.parent) - print('PASS release plotting covers 2 cuts x 2 epochs x 2 atmospheres') - # Lookup sanity checker requires successful executable AND marker AND output. - aux=d/'aux';(aux/'IRFVERSION').write_text('v492\n');(aux/'Tables').mkdir();(aux/'Tables/table.root').write_text('table');(d/'reference.root').write_text('events') - exe(d/'bin/mscw_energy','[[ $MODE == producer-fail ]] && exit 9\nfor arg in "$@";do case $arg in -outputfile) next=1;;*) if [[ $next == 1 ]]; then [[ $MODE != missing ]] && printf new > "$arg";next=0;fi;;esac;done\n[[ $MODE != log-fail ]] && echo "...survived test of table file!"\nexit 0\n') - p=ROOT/'EventDisplay_ReleaseTests_code/release_tests/lookuptable-testing/test_tables.sh' - for mode in ['producer-fail','missing','log-fail','success']: - env['MODE']=mode;res=subprocess.run(['bash',str(p),str(d/'reference.root')],env=env,cwd=d,stdout=subprocess.PIPE,text=True,stderr=subprocess.STDOUT);assert (res.returncode==0)==(mode=='success'),(mode,res.stdout) - print('PASS lookup executable/missing-output/missing-sanity-marker failures')