From 02db837a1eae88400a4d41fe76f6ef37521d56ad Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:11:41 +0000 Subject: [PATCH 1/2] fix(script): stderr del comando visibile nel log mentre gira MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _fetch_script usava subprocess.run(capture_output=True): lo stderr del preprocess restava nascosto fino alla fine e veniva mostrato solo (primi 500 caratteri) se il comando falliva. Un harvest lungo era un buco nero: senato-votazioni ha girato 78 minuti senza una riga di log nel job. Ora il subprocess è Popen con stderr=STDOUT letto da un thread: ogni riga viene inoltrata a sys.stderr con prefisso [script] mentre il comando gira, e le ultime 50 restano in un buffer per il messaggio d'errore. Il timeout resta quello di client.timeout e continua a sollevare TimeoutExpired. Test: stderr inoltrato (capfd) e coda dell'errore nel messaggio di DownloadError. --- tests/test_script_source.py | 51 ++++++++++++++++++++++++++++++++++++ toolkit/core/source_types.py | 44 ++++++++++++++++++++++++++----- 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/tests/test_script_source.py b/tests/test_script_source.py index be20afc5..87796ac0 100644 --- a/tests/test_script_source.py +++ b/tests/test_script_source.py @@ -219,3 +219,54 @@ def test_script_rejects_prefix_collision(tmp_path: Path): {"command": "echo ok", "output": str(evil_file)}, base_dir=candidate, ) + + +# --------------------------------------------------------------------------- +# Osservabilità: lo stderr del comando deve arrivare nel log mentre gira +# --------------------------------------------------------------------------- + + +def test_script_stderr_forwarded_to_log(script_dir: Path, capfd): + """Regressione: ``capture_output`` teneva lo stderr nascosto fino alla fine. + + Un harvest di 78 minuti senza una riga di log è indistinguibile da un + hang. Con il tee su stderr le righe del comando compaiono nel job log + mentre il comando gira. + """ + output_file = script_dir / "out.csv" + script = _make_script( + script_dir, + output_file, + f'echo "progress 1" >&2\necho "progress 2" >&2\necho "a,b" > {output_file}', + ) + + payload, _ = _fetch_payload( + "script", + {}, + {"command": f"bash {script}", "output": str(output_file)}, + base_dir=script_dir, + ) + + assert payload == b"a,b\n" + err = capfd.readouterr().err + assert "progress 1" in err + assert "progress 2" in err + assert "[script]" in err + + +def test_script_failure_keeps_tail_in_error(script_dir: Path): + """Sul fallimento il messaggio porta le ultime righe, anche col tee.""" + output_file = script_dir / "missing.csv" + script = _make_script( + script_dir, + output_file, + 'echo "errore specifico della fonte" >&2\nexit 3', + ) + + with pytest.raises(DownloadError, match="errore specifico della fonte"): + _fetch_payload( + "script", + {}, + {"command": f"bash {script}", "output": str(output_file)}, + base_dir=script_dir, + ) diff --git a/toolkit/core/source_types.py b/toolkit/core/source_types.py index d2224aa5..096c5bcf 100644 --- a/toolkit/core/source_types.py +++ b/toolkit/core/source_types.py @@ -17,6 +17,9 @@ import os import subprocess +import sys +import threading +from collections import deque from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path @@ -250,17 +253,44 @@ def _fetch_script(stype: str, client: dict, formatted_args: dict) -> tuple[bytes f"Output path {output_abs} is outside candidate base directory {candidate_root}" ) - result = subprocess.run( + # Lo stderr del subprocess va nel log del job mentre il comando gira: + # ``capture_output`` lo teneva nascosto fino alla fine, quindi un harvest + # lungo (es. senato-votazioni, 78 min) era un buco nero nel log. Le ultime + # righe restano in un buffer per l'errore, se il comando fallisce. + timeout = (client or {}).get("timeout", 600) + proc = subprocess.Popen( command, shell=True, - capture_output=True, - text=True, - timeout=(client or {}).get("timeout", 600), cwd=candidate_root, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, ) - if result.returncode != 0: - stderr_preview = result.stderr[:500] if result.stderr else "(no stderr)" - raise DownloadError(f"Script failed (exit {result.returncode}): {stderr_preview}") + tail: deque[str] = deque(maxlen=50) + + def _tee_output() -> None: + assert proc.stdout is not None + for line in proc.stdout: + line = line.rstrip("\n") + if line: + tail.append(line) + print(f"[script] {line}", file=sys.stderr, flush=True) + + reader = threading.Thread(target=_tee_output, daemon=True) + reader.start() + try: + returncode = proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + raise + finally: + reader.join(timeout=5) + + if returncode != 0: + preview = "\n".join(tail)[-500:] or "(no output)" + raise DownloadError(f"Script failed (exit {returncode}): {preview}") if not output_abs.exists(): raise DownloadError(f"Script did not produce expected output file: {output_abs}") From 44733f273b72b58a7d0bcd721415f9702929b68e Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:58:21 +0000 Subject: [PATCH 2/2] fix(script): errors="replace" sul reader dello stderr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review #552 (Low): con errors="strict" un byte non-UTF-8 nello stderr uccideva la thread che drena la pipe. La pipe smetteva di essere letta e il figlio poteva bloccarsi quando il buffer si riempiva, fino a client.timeout (7200s per senato-votazioni): da errore immediato a possibile stallo lungo. Ora i byte non decodificabili diventano U+FFFD e le righe successive continuano ad arrivare nel log. Test: stderr con 0xFF 0xFE in mezzo a due righe valide → entrambe nel log (senza il fix la seconda si perde). --- tests/test_script_source.py | 31 +++++++++++++++++++++++++++++++ toolkit/core/source_types.py | 1 + 2 files changed, 32 insertions(+) diff --git a/tests/test_script_source.py b/tests/test_script_source.py index 87796ac0..59f3013f 100644 --- a/tests/test_script_source.py +++ b/tests/test_script_source.py @@ -270,3 +270,34 @@ def test_script_failure_keeps_tail_in_error(script_dir: Path): {"command": f"bash {script}", "output": str(output_file)}, base_dir=script_dir, ) + + +def test_script_non_utf8_stderr_non_blocca(script_dir: Path, capfd): + """Byte non-UTF-8 sullo stderr non devono uccidere il reader. + + Con ``errors='strict'`` (default) la thread muore alla riga non + decodificabile, la pipe smette di essere drenata e il figlio può bloccarsi + fino a ``client.timeout``. Con ``errors='replace'`` le righe successive + continuano ad arrivare nel log. + """ + output_file = script_dir / "out.csv" + script = _make_script( + script_dir, + output_file, + 'echo "prima" >&2\n' + "printf '\\377\\376\\n' >&2\n" + 'echo "dopo" >&2\n' + f'echo "a,b" > {output_file}', + ) + + payload, _ = _fetch_payload( + "script", + {}, + {"command": f"bash {script}", "output": str(output_file)}, + base_dir=script_dir, + ) + + assert payload == b"a,b\n" + err = capfd.readouterr().err + assert "prima" in err + assert "dopo" in err diff --git a/toolkit/core/source_types.py b/toolkit/core/source_types.py index 096c5bcf..1afecf41 100644 --- a/toolkit/core/source_types.py +++ b/toolkit/core/source_types.py @@ -265,6 +265,7 @@ def _fetch_script(stype: str, client: dict, formatted_args: dict) -> tuple[bytes stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + errors="replace", bufsize=1, ) tail: deque[str] = deque(maxlen=50)