diff --git a/tests/test_script_source.py b/tests/test_script_source.py index be20afc..59f3013 100644 --- a/tests/test_script_source.py +++ b/tests/test_script_source.py @@ -219,3 +219,85 @@ def test_script_rejects_prefix_collision(tmp_path: Path): {"command": "echo ok", "output": str(evil_file)}, base_dir=candidate, ) + + +# --------------------------------------------------------------------------- +# Osservabilità: lo stderr del comando deve arrivare nel log mentre gira +# --------------------------------------------------------------------------- + + +def test_script_stderr_forwarded_to_log(script_dir: Path, capfd): + """Regressione: ``capture_output`` teneva lo stderr nascosto fino alla fine. + + Un harvest di 78 minuti senza una riga di log è indistinguibile da un + hang. Con il tee su stderr le righe del comando compaiono nel job log + mentre il comando gira. + """ + output_file = script_dir / "out.csv" + script = _make_script( + script_dir, + output_file, + f'echo "progress 1" >&2\necho "progress 2" >&2\necho "a,b" > {output_file}', + ) + + payload, _ = _fetch_payload( + "script", + {}, + {"command": f"bash {script}", "output": str(output_file)}, + base_dir=script_dir, + ) + + assert payload == b"a,b\n" + err = capfd.readouterr().err + assert "progress 1" in err + assert "progress 2" in err + assert "[script]" in err + + +def test_script_failure_keeps_tail_in_error(script_dir: Path): + """Sul fallimento il messaggio porta le ultime righe, anche col tee.""" + output_file = script_dir / "missing.csv" + script = _make_script( + script_dir, + output_file, + 'echo "errore specifico della fonte" >&2\nexit 3', + ) + + with pytest.raises(DownloadError, match="errore specifico della fonte"): + _fetch_payload( + "script", + {}, + {"command": f"bash {script}", "output": str(output_file)}, + base_dir=script_dir, + ) + + +def test_script_non_utf8_stderr_non_blocca(script_dir: Path, capfd): + """Byte non-UTF-8 sullo stderr non devono uccidere il reader. + + Con ``errors='strict'`` (default) la thread muore alla riga non + decodificabile, la pipe smette di essere drenata e il figlio può bloccarsi + fino a ``client.timeout``. Con ``errors='replace'`` le righe successive + continuano ad arrivare nel log. + """ + output_file = script_dir / "out.csv" + script = _make_script( + script_dir, + output_file, + 'echo "prima" >&2\n' + "printf '\\377\\376\\n' >&2\n" + 'echo "dopo" >&2\n' + f'echo "a,b" > {output_file}', + ) + + payload, _ = _fetch_payload( + "script", + {}, + {"command": f"bash {script}", "output": str(output_file)}, + base_dir=script_dir, + ) + + assert payload == b"a,b\n" + err = capfd.readouterr().err + assert "prima" in err + assert "dopo" in err diff --git a/toolkit/core/source_types.py b/toolkit/core/source_types.py index d2224aa..1afecf4 100644 --- a/toolkit/core/source_types.py +++ b/toolkit/core/source_types.py @@ -17,6 +17,9 @@ import os import subprocess +import sys +import threading +from collections import deque from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path @@ -250,17 +253,45 @@ def _fetch_script(stype: str, client: dict, formatted_args: dict) -> tuple[bytes f"Output path {output_abs} is outside candidate base directory {candidate_root}" ) - result = subprocess.run( + # Lo stderr del subprocess va nel log del job mentre il comando gira: + # ``capture_output`` lo teneva nascosto fino alla fine, quindi un harvest + # lungo (es. senato-votazioni, 78 min) era un buco nero nel log. Le ultime + # righe restano in un buffer per l'errore, se il comando fallisce. + timeout = (client or {}).get("timeout", 600) + proc = subprocess.Popen( command, shell=True, - capture_output=True, - text=True, - timeout=(client or {}).get("timeout", 600), cwd=candidate_root, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + bufsize=1, ) - if result.returncode != 0: - stderr_preview = result.stderr[:500] if result.stderr else "(no stderr)" - raise DownloadError(f"Script failed (exit {result.returncode}): {stderr_preview}") + tail: deque[str] = deque(maxlen=50) + + def _tee_output() -> None: + assert proc.stdout is not None + for line in proc.stdout: + line = line.rstrip("\n") + if line: + tail.append(line) + print(f"[script] {line}", file=sys.stderr, flush=True) + + reader = threading.Thread(target=_tee_output, daemon=True) + reader.start() + try: + returncode = proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + raise + finally: + reader.join(timeout=5) + + if returncode != 0: + preview = "\n".join(tail)[-500:] or "(no output)" + raise DownloadError(f"Script failed (exit {returncode}): {preview}") if not output_abs.exists(): raise DownloadError(f"Script did not produce expected output file: {output_abs}")