Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions tests/test_script_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,3 +219,85 @@ def test_script_rejects_prefix_collision(tmp_path: Path):
{"command": "echo ok", "output": str(evil_file)},
base_dir=candidate,
)


# ---------------------------------------------------------------------------
# Osservabilità: lo stderr del comando deve arrivare nel log mentre gira
# ---------------------------------------------------------------------------


def test_script_stderr_forwarded_to_log(script_dir: Path, capfd):
"""Regressione: ``capture_output`` teneva lo stderr nascosto fino alla fine.

Un harvest di 78 minuti senza una riga di log è indistinguibile da un
hang. Con il tee su stderr le righe del comando compaiono nel job log
mentre il comando gira.
"""
output_file = script_dir / "out.csv"
script = _make_script(
script_dir,
output_file,
f'echo "progress 1" >&2\necho "progress 2" >&2\necho "a,b" > {output_file}',
)

payload, _ = _fetch_payload(
"script",
{},
{"command": f"bash {script}", "output": str(output_file)},
base_dir=script_dir,
)

assert payload == b"a,b\n"
err = capfd.readouterr().err
assert "progress 1" in err
assert "progress 2" in err
assert "[script]" in err


def test_script_failure_keeps_tail_in_error(script_dir: Path):
"""Sul fallimento il messaggio porta le ultime righe, anche col tee."""
output_file = script_dir / "missing.csv"
script = _make_script(
script_dir,
output_file,
'echo "errore specifico della fonte" >&2\nexit 3',
)

with pytest.raises(DownloadError, match="errore specifico della fonte"):
_fetch_payload(
"script",
{},
{"command": f"bash {script}", "output": str(output_file)},
base_dir=script_dir,
)


def test_script_non_utf8_stderr_non_blocca(script_dir: Path, capfd):
"""Byte non-UTF-8 sullo stderr non devono uccidere il reader.

Con ``errors='strict'`` (default) la thread muore alla riga non
decodificabile, la pipe smette di essere drenata e il figlio può bloccarsi
fino a ``client.timeout``. Con ``errors='replace'`` le righe successive
continuano ad arrivare nel log.
"""
output_file = script_dir / "out.csv"
script = _make_script(
script_dir,
output_file,
'echo "prima" >&2\n'
"printf '\\377\\376\\n' >&2\n"
'echo "dopo" >&2\n'
f'echo "a,b" > {output_file}',
)

payload, _ = _fetch_payload(
"script",
{},
{"command": f"bash {script}", "output": str(output_file)},
base_dir=script_dir,
)

assert payload == b"a,b\n"
err = capfd.readouterr().err
assert "prima" in err
assert "dopo" in err
45 changes: 38 additions & 7 deletions toolkit/core/source_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@

import os
import subprocess
import sys
import threading
from collections import deque
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
Expand Down Expand Up @@ -250,17 +253,45 @@ def _fetch_script(stype: str, client: dict, formatted_args: dict) -> tuple[bytes
f"Output path {output_abs} is outside candidate base directory {candidate_root}"
)

result = subprocess.run(
# Lo stderr del subprocess va nel log del job mentre il comando gira:
# ``capture_output`` lo teneva nascosto fino alla fine, quindi un harvest
# lungo (es. senato-votazioni, 78 min) era un buco nero nel log. Le ultime
# righe restano in un buffer per l'errore, se il comando fallisce.
timeout = (client or {}).get("timeout", 600)
proc = subprocess.Popen(
command,
shell=True,
capture_output=True,
text=True,
timeout=(client or {}).get("timeout", 600),
cwd=candidate_root,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
errors="replace",
bufsize=1,
)
if result.returncode != 0:
stderr_preview = result.stderr[:500] if result.stderr else "(no stderr)"
raise DownloadError(f"Script failed (exit {result.returncode}): {stderr_preview}")
tail: deque[str] = deque(maxlen=50)

def _tee_output() -> None:
assert proc.stdout is not None
for line in proc.stdout:
line = line.rstrip("\n")
if line:
tail.append(line)
print(f"[script] {line}", file=sys.stderr, flush=True)

reader = threading.Thread(target=_tee_output, daemon=True)
reader.start()
try:
returncode = proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()
raise
finally:
reader.join(timeout=5)

if returncode != 0:
preview = "\n".join(tail)[-500:] or "(no output)"
raise DownloadError(f"Script failed (exit {returncode}): {preview}")

if not output_abs.exists():
raise DownloadError(f"Script did not produce expected output file: {output_abs}")
Expand Down