From 443327c145c819ca499ef62cd67b889e5d947ebd Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 21:34:13 -0500 Subject: [PATCH 01/10] camera_streamer: add deploy_sender.py to build and try argus_sender on a Jetson without activating it check and discover only read. deploy copies the committed argus_sender source into a new release directory under ~/camera-streamer/releases/, configures and builds it on the target and runs the camera-free unit tests; it never stops, replaces or starts a service and refuses an existing release. run starts a built release for a bounded time with a config from this machine, stop signals the senders that run started, and linktest measures TCP throughput from the target to this machine, connecting the way the camera stream does. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../argus_sender/deploy/deploy_sender.py | 312 ++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100755 examples/camera_streamer/argus_sender/deploy/deploy_sender.py diff --git a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py new file mode 100755 index 0000000000..69752c156c --- /dev/null +++ b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Build and try argus_sender on a Jetson from this checkout, without activating it. + +Run on the desktop: + + deploy_sender.py user@orin check read-only: release, camera processes, disk + deploy_sender.py user@orin discover read-only: platform files, tuning files, camera units + deploy_sender.py user@orin deploy [--with-sipl] [--name NAME] + deploy_sender.py user@orin run --name NAME --config FILE [--seconds N] + deploy_sender.py user@orin stop stop the senders that `run` started + deploy_sender.py user@orin linktest [--mb N] TCP throughput from the target to this machine + +`deploy` copies the committed source of argus_sender (git archive of HEAD) into a new release directory under +~/camera-streamer/releases/ on the target, configures and builds it there and runs the unit tests that need no camera. +It never stops, replaces or starts a service and never opens a camera; a release directory that already exists is +refused rather than overwritten. `run` starts a built release for a bounded time with a config from this machine and +opens the cameras; it prints a note, and does not stop anything, if another camera process is running. +""" + +from __future__ import annotations + +import argparse +import re +import shlex +import subprocess +import sys +from pathlib import Path + +RELEASES = "~/camera-streamer/releases" +SIPL_ROOT = "/usr/src/jetson_sipl_api/sipl" +SOURCE_ROOT = "examples/camera_streamer/argus_sender" +NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") + + +class DeployError(Exception): + """A failure with a message worth showing as is.""" + + +def ssh_command(target: str, remote: str) -> list[str]: + return ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", target, remote] + + +def remote(target: str, script: str, dry_run: bool) -> None: + """Run a bash script on the target; its output goes straight to this terminal.""" + command = ssh_command(target, "bash -lc " + shlex.quote(script)) + if dry_run: + print("DRY-RUN", shlex.join(command)) + return + result = subprocess.run(command, check=False) + if result.returncode != 0: + raise DeployError(f"remote command failed with exit code {result.returncode}") + + +def repo_root() -> Path: + out = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=False, + ) + if out.returncode != 0: + raise DeployError("run this inside the IsaacCapture checkout") + return Path(out.stdout.strip()) + + +def git_output(root: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(root), *args], capture_output=True, text=True, check=True + ).stdout.strip() + + +def cmd_check(target: str, dry_run: bool) -> None: + remote( + target, + r""" +set -u +echo "host: $(hostname) kernel: $(uname -r)" +head -1 /etc/nv_tegra_release 2>/dev/null || echo "no /etc/nv_tegra_release: not a Jetson" +echo "--- processes that may own the cameras" +ps -eo pid,etime,cmd | grep -Ei 'nvargus|argus_sender|sipl|camera' | grep -v grep | cut -c1-160 || true +echo "--- SIPL and multimedia packages" +for path in /usr/src/jetson_sipl_api /usr/src/jetson_multimedia_api; do [ -d "$path" ] && echo "present: $path" || echo "missing: $path"; done +echo "--- disk" +df -h ~ | tail -1 +""", + dry_run, + ) + + +def cmd_discover(target: str, dry_run: bool) -> None: + remote( + target, + r""" +set -u +echo "--- SIPL platform configuration candidates" +find /opt /usr/share /var/nvidia ~ -maxdepth 6 \( -iname 'shw5g*.json' -o -iname 'camera-platform*.json' \) 2>/dev/null | head -20 +echo "--- NITO tuning files" +ls -la /var/nvidia/nvcam/settings/sipl 2>/dev/null | head -20 || echo "none" +echo "--- camera units (read-only)" +systemctl list-unit-files 2>/dev/null | grep -Ei 'camera|argus|sipl' || echo "none" +systemctl cat camera-argus-sender.service 2>/dev/null | head -30 +echo "--- video devices and driver" +ls /dev/video* 2>/dev/null | head; lsmod 2>/dev/null | grep -Ei 'tegra_camera|nvhost_vi|max9296|max96712|sg8a' | head +""", + dry_run, + ) + + +def cmd_linktest(target: str, megabytes: int, dry_run: bool) -> None: + """Measure TCP throughput from the target to this machine, connecting the way the camera stream does. + + The target listens and sends; this machine connects and counts, like camera_receiver and argus_sender. + """ + import socket + import time + + port = 5611 + host = target.rsplit("@", 1)[-1] + sender = ( + "import socket;l=socket.socket();l.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1);" + "l.bind(('0.0.0.0',%d));l.listen(1);l.settimeout(20);c,_=l.accept();b=bytes(1<<20)\n" + "for _ in range(%d): c.sendall(b)\n" + "c.close()" % (port, megabytes) + ) + command = ssh_command(target, f"python3 -c {shlex.quote(sender)}") + if dry_run: + print("DRY-RUN", shlex.join(command)) + return + process = subprocess.Popen(command) + try: + connection = None + for _ in range(40): + try: + connection = socket.create_connection((host, port), timeout=2) + break + except OSError: + time.sleep(0.5) + if connection is None: + raise DeployError(f"could not connect to {host}:{port}") + connection.settimeout(60) + total, start = 0, time.monotonic() + while True: + chunk = connection.recv(1 << 20) + if not chunk: + break + total += len(chunk) + seconds = time.monotonic() - start + finally: + process.wait(timeout=30) + print( + f"{host} -> this machine: {total / 1e6:.1f} MB in {seconds:.2f} s = {total * 8 / seconds / 1e6:.1f} Mbit/s" + ) + + +def cmd_stop(target: str, dry_run: bool) -> None: + """Ask the senders that `run` started (./build/argus_sender --config run-config.yaml) to stop (SIGINT, so they drain and free the cameras).""" + remote( + target, + r""" +pids=$(pgrep -f "^\./build/argus_sender --config run-config.yaml" || true) +if [ -z "$pids" ]; then echo "no sender from ~/camera-streamer/releases is running"; exit 0; fi +echo "signalling: $pids" +kill -INT $pids +for _ in $(seq 1 20); do + pgrep -f "^\./build/argus_sender --config run-config.yaml" >/dev/null || { echo "stopped"; exit 0; } + sleep 1 +done +echo "still running after 20 s; leaving it alone" >&2 +exit 1 +""", + dry_run, + ) + + +def cmd_run(target: str, name: str, config: Path, seconds: int, dry_run: bool) -> None: + """Run a deployed release for a bounded time, in the foreground, with a config from this machine.""" + if not NAME.fullmatch(name): + raise DeployError(f"release name {name!r} is not a plain file name") + if not config.is_file(): + raise DeployError(f"{config} is not a file") + directory = f"{RELEASES}/{name}" + guard = f""" +set -e +d={directory} +[ -x $d/build/argus_sender ] || {{ echo "no built sender in $d" >&2; exit 1; }} +# Taking the cameras is intended; say who else may hold them, but do not stop anything. +if systemctl is-active --quiet camera-argus-sender.service 2>/dev/null; then + echo "note: camera-argus-sender.service is active and may hold the cameras" >&2 +fi +if pgrep -f '/argus_sender' >/dev/null; then + echo "note: another camera process is running" >&2 +fi +""" + upload = ssh_command(target, f"cat > {directory}/run-config.yaml") + if dry_run: + print("DRY-RUN", shlex.join(upload), "<", config) + else: + with config.open("rb") as handle: + if subprocess.run(upload, stdin=handle, check=False).returncode != 0: + raise DeployError("uploading the config failed") + remote( + target, + guard + + f""" +cd $d +# SIGINT first: the sender drains its encoder and returns every camera buffer on SIGINT/SIGTERM. +timeout --signal=INT --kill-after=15 {seconds} ./build/argus_sender --config run-config.yaml || echo "sender exited with status $?" +""", + dry_run, + ) + + +def cmd_deploy(target: str, with_sipl: bool, name: str | None, dry_run: bool) -> None: + root = repo_root() + if git_output(root, "status", "--porcelain", "--", SOURCE_ROOT): + raise DeployError( + f"{SOURCE_ROOT} has uncommitted changes; commit them (the release is a git archive of HEAD)" + ) + commit = git_output(root, "rev-parse", "--short=9", "HEAD") + release = name or commit + if not NAME.fullmatch(release): + raise DeployError(f"release name {release!r} is not a plain file name") + directory = f"{RELEASES}/{release}" + + print(f"==> [1/3] preparing {directory} on {target} (commit {commit})", flush=True) + remote( + target, + f'set -e; d={directory}; [ ! -e $d ] || {{ echo "release $d already exists" >&2; exit 1; }}; mkdir -p $d/src', + dry_run, + ) + + print("==> [2/3] copying the committed source", flush=True) + archive = ["git", "-C", str(root), "archive", "--format=tar", f"HEAD:{SOURCE_ROOT}"] + extract = ssh_command(target, f"tar -x -C {directory}/src") + if dry_run: + print("DRY-RUN", shlex.join(archive), "|", shlex.join(extract)) + else: + producer = subprocess.Popen(archive, stdout=subprocess.PIPE) + consumer = subprocess.run(extract, stdin=producer.stdout, check=False) + producer.stdout.close() + if producer.wait() != 0 or consumer.returncode != 0: + raise DeployError("copying the source failed") + + print("==> [3/3] configuring, building and testing on the target", flush=True) + options = "-DCMAKE_BUILD_TYPE=Release -DDISABLE_MULTIPROCESS=ON" + if with_sipl: + options += f" -DWITH_SIPL=ON -DSIPL_ROOT={SIPL_ROOT}" + remote( + target, + f""" +set -e +cd {directory} +cmake -S src -B build {options} -DBUILD_TESTING=ON +cmake --build build -j "$(nproc)" +# Only the tests that need no camera; ctest does not open any device. +(cd build && ctest --output-on-failure) || echo "some tests failed (see above); the build itself succeeded" +./build/argus_sender --help >/dev/null 2>&1 || true +echo "built: {directory}/build/argus_sender (not activated, no camera opened)" +""", + dry_run, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("target", help="user@host of the Jetson") + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("check", help="read-only look at the target") + sub.add_parser("discover", help="read-only: SIPL platform files and existing units") + sub.add_parser( + "stop", help="stop the senders started from ~/camera-streamer/releases" + ) + link = sub.add_parser("linktest", help="TCP throughput from the target to this PC") + link.add_argument("--mb", type=int, default=32, help="megabytes to send") + run = sub.add_parser("run", help="run a deployed release for a bounded time") + run.add_argument("--name", required=True, help="release directory name") + run.add_argument( + "--config", required=True, type=Path, help="sender config to upload" + ) + run.add_argument("--seconds", type=int, default=60, help="stop after this long") + deploy = sub.add_parser("deploy", help="copy, build and test; never activates") + deploy.add_argument( + "--with-sipl", action="store_true", help="also build the SIPL backend" + ) + deploy.add_argument("--name", help="release directory name (default: commit)") + parser.add_argument( + "--dry-run", action="store_true", help="print the commands instead of running" + ) + args = parser.parse_args(argv) + try: + if args.command == "check": + cmd_check(args.target, args.dry_run) + elif args.command == "stop": + cmd_stop(args.target, args.dry_run) + elif args.command == "linktest": + cmd_linktest(args.target, args.mb, args.dry_run) + elif args.command == "run": + cmd_run(args.target, args.name, args.config, args.seconds, args.dry_run) + elif args.command == "discover": + cmd_discover(args.target, args.dry_run) + else: + cmd_deploy(args.target, args.with_sipl, args.name, args.dry_run) + except DeployError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 17256b6275be315abd74f7a9ceb065d89c79bd88 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 23:17:00 -0500 Subject: [PATCH 02/10] camera_streamer: make deploy_sender.py report real failures A failing unit test now fails the deploy instead of being echoed and ignored. A run that reaches its time limit is a success, and any other sender exit status is propagated instead of being turned into success. The scripts that run on the target use [[ ]] for tests, as the repository's shell rule requires, and run takes repeatable --env NAME=VALUE options for the sender. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../argus_sender/deploy/deploy_sender.py | 50 ++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py index 69752c156c..189efa9bba 100755 --- a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py +++ b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py @@ -8,7 +8,7 @@ deploy_sender.py user@orin check read-only: release, camera processes, disk deploy_sender.py user@orin discover read-only: platform files, tuning files, camera units deploy_sender.py user@orin deploy [--with-sipl] [--name NAME] - deploy_sender.py user@orin run --name NAME --config FILE [--seconds N] + deploy_sender.py user@orin run --name NAME --config FILE [--seconds N] [--env NAME=VALUE] deploy_sender.py user@orin stop stop the senders that `run` started deploy_sender.py user@orin linktest [--mb N] TCP throughput from the target to this machine @@ -16,7 +16,8 @@ ~/camera-streamer/releases/ on the target, configures and builds it there and runs the unit tests that need no camera. It never stops, replaces or starts a service and never opens a camera; a release directory that already exists is refused rather than overwritten. `run` starts a built release for a bounded time with a config from this machine and -opens the cameras; it prints a note, and does not stop anything, if another camera process is running. +opens the cameras; it prints a note, and does not stop anything, if another camera process is running. `run` succeeds when the +sender ends at its time limit and fails with the sender's own exit status otherwise; `deploy` fails if any unit test fails. """ from __future__ import annotations @@ -81,7 +82,7 @@ def cmd_check(target: str, dry_run: bool) -> None: echo "--- processes that may own the cameras" ps -eo pid,etime,cmd | grep -Ei 'nvargus|argus_sender|sipl|camera' | grep -v grep | cut -c1-160 || true echo "--- SIPL and multimedia packages" -for path in /usr/src/jetson_sipl_api /usr/src/jetson_multimedia_api; do [ -d "$path" ] && echo "present: $path" || echo "missing: $path"; done +for path in /usr/src/jetson_sipl_api /usr/src/jetson_multimedia_api; do [[ -d "$path" ]] && echo "present: $path" || echo "missing: $path"; done echo "--- disk" df -h ~ | tail -1 """, @@ -160,7 +161,7 @@ def cmd_stop(target: str, dry_run: bool) -> None: target, r""" pids=$(pgrep -f "^\./build/argus_sender --config run-config.yaml" || true) -if [ -z "$pids" ]; then echo "no sender from ~/camera-streamer/releases is running"; exit 0; fi +if [[ -z "$pids" ]]; then echo "no sender from ~/camera-streamer/releases is running"; exit 0; fi echo "signalling: $pids" kill -INT $pids for _ in $(seq 1 20); do @@ -174,17 +175,23 @@ def cmd_stop(target: str, dry_run: bool) -> None: ) -def cmd_run(target: str, name: str, config: Path, seconds: int, dry_run: bool) -> None: +def cmd_run( + target: str, name: str, config: Path, seconds: int, env: list[str], dry_run: bool +) -> None: """Run a deployed release for a bounded time, in the foreground, with a config from this machine.""" if not NAME.fullmatch(name): raise DeployError(f"release name {name!r} is not a plain file name") if not config.is_file(): raise DeployError(f"{config} is not a file") directory = f"{RELEASES}/{name}" + for item in env: + if not re.fullmatch(r"[A-Z_][A-Z0-9_]*=[A-Za-z0-9_.:/-]*", item): + raise DeployError(f"--env {item!r} is not NAME=VALUE with a plain value") + env_prefix = "env " + " ".join(env) + " " if env else "" guard = f""" set -e d={directory} -[ -x $d/build/argus_sender ] || {{ echo "no built sender in $d" >&2; exit 1; }} +[[ -x $d/build/argus_sender ]] || {{ echo "no built sender in $d" >&2; exit 1; }} # Taking the cameras is intended; say who else may hold them, but do not stop anything. if systemctl is-active --quiet camera-argus-sender.service 2>/dev/null; then echo "note: camera-argus-sender.service is active and may hold the cameras" >&2 @@ -206,7 +213,11 @@ def cmd_run(target: str, name: str, config: Path, seconds: int, dry_run: bool) - + f""" cd $d # SIGINT first: the sender drains its encoder and returns every camera buffer on SIGINT/SIGTERM. -timeout --signal=INT --kill-after=15 {seconds} ./build/argus_sender --config run-config.yaml || echo "sender exited with status $?" +status=0 +{env_prefix}timeout --signal=INT --kill-after=15 {seconds} ./build/argus_sender --config run-config.yaml || status=$? +# 124 is the time limit this command set; any other nonzero status is the sender's own and fails the run. +if [[ $status -eq 124 ]]; then echo "sender stopped at its {seconds} s time limit"; exit 0; fi +if [[ $status -ne 0 ]]; then echo "sender exited with status $status" >&2; exit $status; fi """, dry_run, ) @@ -227,7 +238,7 @@ def cmd_deploy(target: str, with_sipl: bool, name: str | None, dry_run: bool) -> print(f"==> [1/3] preparing {directory} on {target} (commit {commit})", flush=True) remote( target, - f'set -e; d={directory}; [ ! -e $d ] || {{ echo "release $d already exists" >&2; exit 1; }}; mkdir -p $d/src', + f'set -e; d={directory}; [[ ! -e $d ]] || {{ echo "release $d already exists" >&2; exit 1; }}; mkdir -p $d/src', dry_run, ) @@ -254,10 +265,9 @@ def cmd_deploy(target: str, with_sipl: bool, name: str | None, dry_run: bool) -> cd {directory} cmake -S src -B build {options} -DBUILD_TESTING=ON cmake --build build -j "$(nproc)" -# Only the tests that need no camera; ctest does not open any device. -(cd build && ctest --output-on-failure) || echo "some tests failed (see above); the build itself succeeded" -./build/argus_sender --help >/dev/null 2>&1 || true -echo "built: {directory}/build/argus_sender (not activated, no camera opened)" +# Only the tests that need no camera; ctest does not open any device. A failing test fails the deploy. +(cd build && ctest --output-on-failure) +echo "built and tested: {directory}/build/argus_sender (not activated, no camera opened)" """, dry_run, ) @@ -280,6 +290,13 @@ def main(argv: list[str] | None = None) -> int: "--config", required=True, type=Path, help="sender config to upload" ) run.add_argument("--seconds", type=int, default=60, help="stop after this long") + run.add_argument( + "--env", + action="append", + default=[], + metavar="NAME=VALUE", + help="environment variable for the sender (repeatable)", + ) deploy = sub.add_parser("deploy", help="copy, build and test; never activates") deploy.add_argument( "--with-sipl", action="store_true", help="also build the SIPL backend" @@ -297,7 +314,14 @@ def main(argv: list[str] | None = None) -> int: elif args.command == "linktest": cmd_linktest(args.target, args.mb, args.dry_run) elif args.command == "run": - cmd_run(args.target, args.name, args.config, args.seconds, args.dry_run) + cmd_run( + args.target, + args.name, + args.config, + args.seconds, + args.env, + args.dry_run, + ) elif args.command == "discover": cmd_discover(args.target, args.dry_run) else: From 42d1bd8ec99b187978fa5a12ea0596d9a4846039 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Tue, 10 Nov 2026 21:33:40 -0500 Subject: [PATCH 03/10] camera_streamer: add cross_build.sh to type-check and cross-compile argus_sender on a PC syntax runs clang over every sender source against an Orin sysroot's headers, and build produces an aarch64 argus_sender (linked, not run) with an Arm GNU toolchain. The sysroot and toolchain live under ORIN_CROSS_ROOT; the script header lists what they must contain. TESTING.md notes that tests which include Argus headers compile this way but run only on the Orin. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../camera_streamer/argus_sender/TESTING.md | 7 ++ .../argus_sender/cross_build.sh | 68 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100755 examples/camera_streamer/argus_sender/cross_build.sh diff --git a/examples/camera_streamer/argus_sender/TESTING.md b/examples/camera_streamer/argus_sender/TESTING.md index 3a8e0a20c9..ef27073b65 100644 --- a/examples/camera_streamer/argus_sender/TESTING.md +++ b/examples/camera_streamer/argus_sender/TESTING.md @@ -34,6 +34,13 @@ The deployment snapshot/restore checks run from `argus_sender/`: python3 -m pytest tests/test_deployment.py -q ``` +## Cross-compiling on a PC + +`cross_build.sh syntax` type-checks every sender source against the Orin's headers, and `cross_build.sh build` +produces an aarch64 `argus_sender` that is linked but not run. Both use the sysroot and Arm toolchain from +`ORIN_CROSS_ROOT`; the script header says what the sysroot must contain. Tests that include Argus headers compile +this way but run only on the Orin. + ## 120-frame hardware smoke test The opt-in test opens the devices in diff --git a/examples/camera_streamer/argus_sender/cross_build.sh b/examples/camera_streamer/argus_sender/cross_build.sh new file mode 100755 index 0000000000..bd1fdc20ab --- /dev/null +++ b/examples/camera_streamer/argus_sender/cross_build.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Cross-compile argus_sender for a Jetson on an x86 PC, using an Orin sysroot and an Arm GNU toolchain under +# ORIN_CROSS_ROOT (default ~/github/orin-cross-39.2.1): +# /sysroot the Jetson Linux root: the L4T packages (including nvidia-l4t-multimedia, which +# holds libnvv4l2), the Multimedia API and, for the SIPL backend, the SIPL API +# /arm-gnu-toolchain-*/ Arm's aarch64-none-linux-gnu toolchain +# /orin-aarch64.cmake a CMake toolchain file naming both +# `syntax` also needs clang, and the yaml-cpp sources that a first `build` fetches into /build-streamer. +# +# cross_build.sh syntax type-check every source against the Orin headers (clang, no linking) +# cross_build.sh build configure and build the aarch64 argus_sender (extra arguments go to cmake) + +set -euo pipefail + +mode="${1:-build}" +shift || true +project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +root="${ORIN_CROSS_ROOT:-${HOME}/github/orin-cross-39.2.1}" +sysroot="${root}/sysroot" +mmapi="${sysroot}/usr/src/jetson_multimedia_api" +toolchain="${root}/orin-aarch64.cmake" + +for path in "${sysroot}" "${mmapi}/argus/include" "${toolchain}" "${sysroot}/usr/lib/aarch64-linux-gnu/nvidia/libnvv4l2.so"; do + if [[ ! -e "${path}" ]]; then + echo "error: ${path} is missing; see the header of this script" >&2 + exit 1 + fi +done + +case "${mode}" in +syntax) + cxx_include="$(ls -d "${root}"/arm-gnu-toolchain*/aarch64-none-linux-gnu/include/c++/* | head -n1)" + yaml_include="${root}/build-streamer/_deps/yaml_cpp-src/include" + if [[ ! -d "${yaml_include}" ]]; then + echo "error: ${yaml_include} is missing; run 'cross_build.sh build' once to fetch yaml-cpp" >&2 + exit 1 + fi + status=0 + for source in "${project_dir}"/src/*.cpp; do + # The SIPL headers need C++17; the rest of the sender is C++14. + standard=c++14 + [[ "${source}" == */SiplCapture.cpp ]] && standard=c++17 + clang++ --target=aarch64-linux-gnu --sysroot="${sysroot}" -std="${standard}" -fsyntax-only -Wno-everything \ + -I"${sysroot}/usr/src/jetson_sipl_api/sipl/include" -I"${sysroot}/usr/src/jetson_sipl_api/sipl/include/nvsci" -I"${sysroot}/usr/src/jetson_sipl_api/sipl/include/query/include" \ + -DNVMMAPI_SUPPORTED -I"${project_dir}/src" -I"${mmapi}/argus/include" -I"${mmapi}/include" \ + -I"${mmapi}/samples/common/classes" -I/usr/local/cuda/targets/sbsa-linux/include \ + -I"${sysroot}/usr/include/drm" -I"${yaml_include}" -isystem "${sysroot}/usr/include/aarch64-linux-gnu" \ + -isystem "${cxx_include}" -isystem "${cxx_include}/aarch64-none-linux-gnu" \ + "${source}" || status=1 + done + exit "${status}" + ;; +build) + build_dir="${root}/build-streamer" + cmake -S "${project_dir}" -B "${build_dir}" -DCMAKE_TOOLCHAIN_FILE="${toolchain}" -DCMAKE_BUILD_TYPE=Release \ + -DDISABLE_MULTIPROCESS=ON -DMMAPI_ROOT="${mmapi}" -DDRM_INCLUDE_DIR="${sysroot}/usr/include/drm" \ + -DBUILD_TESTING=OFF -DSIPL_ROOT="${sysroot}/usr/src/jetson_sipl_api/sipl" "$@" + cmake --build "${build_dir}" --target argus_sender -j "$(nproc)" + echo "Built ${build_dir}/argus_sender (aarch64; not run)" + ;; +*) + echo "usage: cross_build.sh syntax|build [cmake args]" >&2 + exit 2 + ;; +esac From 4eaefc7fd8c5ce880f54bb8eac0990dcbeb46bcc Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 12 Nov 2026 14:47:45 -0500 Subject: [PATCH 04/10] camera_streamer: add reset and remove-producer to deploy_sender.py reset restarts nvargus-daemon, or reboots the target and waits for ssh, so a camera that will not initialise can be freed without logging in. remove-producer uninstalls the sensing camera producer service and its release tree, leaving the shared SIPL platform file, so the cameras belong to the sender alone. Both need passwordless sudo and fail at once without it; reset refuses while a sender started by run is alive. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../argus_sender/deploy/deploy_sender.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py index 189efa9bba..972aa8ef86 100755 --- a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py +++ b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py @@ -10,6 +10,8 @@ deploy_sender.py user@orin deploy [--with-sipl] [--name NAME] deploy_sender.py user@orin run --name NAME --config FILE [--seconds N] [--env NAME=VALUE] deploy_sender.py user@orin stop stop the senders that `run` started + deploy_sender.py user@orin reset [--reboot] restart nvargus-daemon (or reboot) to free a stuck camera + deploy_sender.py user@orin remove-producer uninstall the sensing camera producer service and its releases deploy_sender.py user@orin linktest [--mb N] TCP throughput from the target to this machine `deploy` copies the committed source of argus_sender (git archive of HEAD) into a new release directory under @@ -18,6 +20,10 @@ refused rather than overwritten. `run` starts a built release for a bounded time with a config from this machine and opens the cameras; it prints a note, and does not stop anything, if another camera process is running. `run` succeeds when the sender ends at its time limit and fails with the sender's own exit status otherwise; `deploy` fails if any unit test fails. +`reset` is the one command that changes the target's state: it needs passwordless sudo there, refuses while a sender started +by `run` is alive, and otherwise restarts nvargus-daemon (or, with --reboot, reboots and waits for ssh to return). +`remove-producer` also needs passwordless sudo: it stops and disables isaaccapture-producer.service, deletes its unit file +and /opt/isaaccapture/{current,releases}, and leaves /opt/isaaccapture/camera-platform.json and everything else alone. """ from __future__ import annotations @@ -27,6 +33,7 @@ import shlex import subprocess import sys +import time from pathlib import Path RELEASES = "~/camera-streamer/releases" @@ -175,6 +182,88 @@ def cmd_stop(target: str, dry_run: bool) -> None: ) +def cmd_reset(target: str, reboot: bool, dry_run: bool) -> None: + """Free a camera that will not initialise: restart nvargus-daemon, or reboot the target. + + Needs passwordless sudo on the target (`sudo -n`), so a missing password fails at once instead of hanging. Refuses + while a sender started by `run` is alive, because restarting the daemon under it would kill its capture. + """ + guard = r""" +if pgrep -f "^\./build/argus_sender --config run-config.yaml" >/dev/null; then + echo "a sender from ~/camera-streamer/releases is running; run 'stop' first" >&2 + exit 1 +fi +sudo -n true || { echo "passwordless sudo is required on the target" >&2; exit 1; } +""" + if not reboot: + remote( + target, + guard + + r""" +sudo -n systemctl restart nvargus-daemon.service +for _ in $(seq 1 20); do + systemctl is-active --quiet nvargus-daemon.service && { echo "nvargus-daemon is active"; exit 0; } + sleep 1 +done +echo "nvargus-daemon did not become active within 20 s" >&2 +exit 1 +""", + dry_run, + ) + return + # The connection drops as the target goes down, so the reboot command's own exit status says nothing. + remote_ignoring_exit = ssh_command( + target, "bash -lc " + shlex.quote(guard + "sudo -n systemctl reboot") + ) + if dry_run: + print("DRY-RUN", shlex.join(remote_ignoring_exit)) + return + result = subprocess.run(remote_ignoring_exit, check=False) + if result.returncode not in (0, 255): + raise DeployError(f"could not start the reboot (exit code {result.returncode})") + # Wait for the target to go away, then for ssh to answer again, so the next command does not race the boot. + time.sleep(20) + deadline = time.monotonic() + 300 + while time.monotonic() < deadline: + if ( + subprocess.run( + ssh_command(target, "true"), check=False, capture_output=True + ).returncode + == 0 + ): + print("target is back") + return + time.sleep(5) + raise DeployError("the target did not answer ssh within 5 minutes of the reboot") + + +def cmd_remove_producer(target: str, dry_run: bool) -> None: + """Uninstall the sensing camera producer so the cameras belong to this sender alone. + + Removes only what the producer's own deploy creates: the service, its unit file and the release tree. The SIPL platform + file next to them is shared with this sender and stays. + """ + remote( + target, + r""" +sudo -n true || { echo "passwordless sudo is required on the target" >&2; exit 1; } +unit=isaaccapture-producer.service +echo "before:"; ls -la /opt/isaaccapture /etc/systemd/system/$unit 2>&1 | sed 's/^/ /' +sudo -n systemctl stop $unit || true +sudo -n systemctl disable $unit || true +sudo -n rm -f /etc/systemd/system/$unit /opt/isaaccapture/current /opt/isaaccapture/current.new +sudo -n rm -rf /opt/isaaccapture/releases +sudo -n systemctl daemon-reload +sudo -n systemctl reset-failed $unit 2>/dev/null || true +echo "after:"; ls -la /opt/isaaccapture 2>&1 | sed 's/^/ /' +systemctl list-unit-files "$unit" --no-legend | grep -q . && { echo "$unit is still installed" >&2; exit 1; } +pgrep -x camera_plugin_s >/dev/null && { echo "camera_plugin_sensing is still running" >&2; exit 1; } +echo "producer removed" +""", + dry_run, + ) + + def cmd_run( target: str, name: str, config: Path, seconds: int, env: list[str], dry_run: bool ) -> None: @@ -282,6 +371,15 @@ def main(argv: list[str] | None = None) -> int: sub.add_parser( "stop", help="stop the senders started from ~/camera-streamer/releases" ) + reset = sub.add_parser( + "reset", help="restart nvargus-daemon (or reboot) to free a stuck camera" + ) + reset.add_argument( + "--reboot", action="store_true", help="reboot the target instead" + ) + sub.add_parser( + "remove-producer", help="uninstall the sensing camera producer service" + ) link = sub.add_parser("linktest", help="TCP throughput from the target to this PC") link.add_argument("--mb", type=int, default=32, help="megabytes to send") run = sub.add_parser("run", help="run a deployed release for a bounded time") @@ -311,6 +409,10 @@ def main(argv: list[str] | None = None) -> int: cmd_check(args.target, args.dry_run) elif args.command == "stop": cmd_stop(args.target, args.dry_run) + elif args.command == "reset": + cmd_reset(args.target, args.reboot, args.dry_run) + elif args.command == "remove-producer": + cmd_remove_producer(args.target, args.dry_run) elif args.command == "linktest": cmd_linktest(args.target, args.mb, args.dry_run) elif args.command == "run": From c309038dac55e8fb7c52e65a9c4e701b8ca347bf Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 14:58:54 -0500 Subject: [PATCH 05/10] camera_streamer: make the deploy tool safer to remove, run and deploy remove-producer stops an installed producer and confirms it is stopped before it deletes anything. run uploads the config beside the shipped configs so config-relative remap paths resolve inside the release. deploy refuses remap files that are Git LFS pointers instead of data. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../argus_sender/deploy/deploy_sender.py | 42 ++++-- .../argus_sender/tests/test_deploy_sender.py | 134 ++++++++++++++++++ 2 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 examples/camera_streamer/argus_sender/tests/test_deploy_sender.py diff --git a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py index 972aa8ef86..6737fcb89c 100755 --- a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py +++ b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py @@ -37,6 +37,9 @@ from pathlib import Path RELEASES = "~/camera-streamer/releases" +# The uploaded config sits beside the shipped ones, so a path in it that is relative to the config file (the remaps) +# resolves inside the release. +RUN_CONFIG = "src/configs/run-config.yaml" SIPL_ROOT = "/usr/src/jetson_sipl_api/sipl" SOURCE_ROOT = "examples/camera_streamer/argus_sender" NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") @@ -163,16 +166,16 @@ def cmd_linktest(target: str, megabytes: int, dry_run: bool) -> None: def cmd_stop(target: str, dry_run: bool) -> None: - """Ask the senders that `run` started (./build/argus_sender --config run-config.yaml) to stop (SIGINT, so they drain and free the cameras).""" + """Ask the senders that `run` started (./build/argus_sender --config src/configs/run-config.yaml) to stop (SIGINT, so they drain and free the cameras).""" remote( target, r""" -pids=$(pgrep -f "^\./build/argus_sender --config run-config.yaml" || true) +pids=$(pgrep -f "^\./build/argus_sender --config src/configs/run-config.yaml" || true) if [[ -z "$pids" ]]; then echo "no sender from ~/camera-streamer/releases is running"; exit 0; fi echo "signalling: $pids" kill -INT $pids for _ in $(seq 1 20); do - pgrep -f "^\./build/argus_sender --config run-config.yaml" >/dev/null || { echo "stopped"; exit 0; } + pgrep -f "^\./build/argus_sender --config src/configs/run-config.yaml" >/dev/null || { echo "stopped"; exit 0; } sleep 1 done echo "still running after 20 s; leaving it alone" >&2 @@ -189,7 +192,7 @@ def cmd_reset(target: str, reboot: bool, dry_run: bool) -> None: while a sender started by `run` is alive, because restarting the daemon under it would kill its capture. """ guard = r""" -if pgrep -f "^\./build/argus_sender --config run-config.yaml" >/dev/null; then +if pgrep -f "^\./build/argus_sender --config src/configs/run-config.yaml" >/dev/null; then echo "a sender from ~/camera-streamer/releases is running; run 'stop' first" >&2 exit 1 fi @@ -249,8 +252,16 @@ def cmd_remove_producer(target: str, dry_run: bool) -> None: sudo -n true || { echo "passwordless sudo is required on the target" >&2; exit 1; } unit=isaaccapture-producer.service echo "before:"; ls -la /opt/isaaccapture /etc/systemd/system/$unit 2>&1 | sed 's/^/ /' -sudo -n systemctl stop $unit || true -sudo -n systemctl disable $unit || true +# An installed unit must be confirmed stopped before its files go: a failed stop would otherwise leave a running +# producer with its unit and release tree deleted under it. An absent unit has nothing to stop. +if systemctl list-unit-files "$unit" --no-legend | grep -q .; then + sudo -n systemctl stop $unit || { echo "could not stop $unit; nothing was removed" >&2; exit 1; } + if systemctl is-active --quiet $unit || pgrep -x camera_plugin_s >/dev/null; then + echo "the producer is still running after the stop; nothing was removed" >&2 + exit 1 + fi + sudo -n systemctl disable $unit || { echo "could not disable $unit; nothing was removed" >&2; exit 1; } +fi sudo -n rm -f /etc/systemd/system/$unit /opt/isaaccapture/current /opt/isaaccapture/current.new sudo -n rm -rf /opt/isaaccapture/releases sudo -n systemctl daemon-reload @@ -289,7 +300,7 @@ def cmd_run( echo "note: another camera process is running" >&2 fi """ - upload = ssh_command(target, f"cat > {directory}/run-config.yaml") + upload = ssh_command(target, f"cat > {directory}/{RUN_CONFIG}") if dry_run: print("DRY-RUN", shlex.join(upload), "<", config) else: @@ -303,7 +314,7 @@ def cmd_run( cd $d # SIGINT first: the sender drains its encoder and returns every camera buffer on SIGINT/SIGTERM. status=0 -{env_prefix}timeout --signal=INT --kill-after=15 {seconds} ./build/argus_sender --config run-config.yaml || status=$? +{env_prefix}timeout --signal=INT --kill-after=15 {seconds} ./build/argus_sender --config {RUN_CONFIG} || status=$? # 124 is the time limit this command set; any other nonzero status is the sender's own and fails the run. if [[ $status -eq 124 ]]; then echo "sender stopped at its {seconds} s time limit"; exit 0; fi if [[ $status -ne 0 ]]; then echo "sender exited with status $status" >&2; exit $status; fi @@ -343,6 +354,21 @@ def cmd_deploy(target: str, with_sipl: bool, name: str | None, dry_run: bool) -> if producer.wait() != 0 or consumer.returncode != 0: raise DeployError("copying the source failed") + # Without Git LFS set up here, `git archive` writes pointer text in place of the remap files; deploying that would + # build a sender whose maps cannot load. + remote( + target, + f""" +bad=$(find {directory}/src -name '*.remap' -print0 | xargs -0 -r grep -l -a '^version https://git-lfs.github.com/spec/v1' || true) +if [[ -n "$bad" ]]; then + echo "these remap files are Git LFS pointers, not data; run 'git lfs install && git lfs pull' here and deploy again:" >&2 + echo "$bad" >&2 + exit 1 +fi +""", + dry_run, + ) + print("==> [3/3] configuring, building and testing on the target", flush=True) options = "-DCMAKE_BUILD_TYPE=Release -DDISABLE_MULTIPROCESS=ON" if with_sipl: diff --git a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py new file mode 100644 index 0000000000..ed82c391dd --- /dev/null +++ b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib.util +from pathlib import Path +import shlex +import subprocess + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "deploy" / "deploy_sender.py" +spec = importlib.util.spec_from_file_location("deploy_sender", SCRIPT) +deploy_sender = importlib.util.module_from_spec(spec) +spec.loader.exec_module(deploy_sender) + +RUN_CONFIG = deploy_sender.RUN_CONFIG +RELEASES_DIR = deploy_sender.RELEASES +RELEASE = "~/camera-streamer/releases/rel" +LFS_POINTER = ( + "version https://git-lfs.github.com/spec/v1\noid sha256:" + "0" * 64 + "\nsize 12\n" +) + + +def dry_run_lines(capsys, *argv): + assert deploy_sender.main(["user@host", "--dry-run", *argv]) == 0 + # A quoted multi-line script spans lines, so split on the marker rather than on newlines. + return [part.strip() for part in capsys.readouterr().out.split("DRY-RUN ")[1:]] + + +def remote_scripts(lines): + """The bash scripts passed to `bash -lc` in the dry-run ssh commands.""" + scripts = [] + for line in lines: + for word in shlex.split(line): + if word.startswith("bash -lc "): + scripts.append(shlex.split(word)[2]) + return scripts + + +def test_run_config_lives_inside_the_release_source_tree(): + assert RUN_CONFIG == "src/configs/run-config.yaml" + + +def test_run_uploads_to_and_starts_with_the_run_config(capsys, tmp_path): + config = tmp_path / "mine.yaml" + config.write_text("cameras: []\n") + lines = dry_run_lines(capsys, "run", "--name", "rel", "--config", str(config)) + upload = shlex.split(lines[0]) + assert f"cat > {RELEASES_DIR}/rel/{RUN_CONFIG}" in upload + (script,) = remote_scripts(lines[1:]) + assert f"./build/argus_sender --config {RUN_CONFIG}" in script + + +def test_stop_and_reset_patterns_match_the_started_command(capsys): + started = f"^\\./build/argus_sender --config {RUN_CONFIG}" + (stop,) = remote_scripts(dry_run_lines(capsys, "stop")) + assert stop.count(f'pgrep -f "{started}"') == 2 + (reset,) = remote_scripts(dry_run_lines(capsys, "reset")) + assert f'pgrep -f "{started}"' in reset + (reboot,) = remote_scripts(dry_run_lines(capsys, "reset", "--reboot")) + assert f'pgrep -f "{started}"' in reboot + + +def test_remove_producer_stops_before_removing(capsys): + (script,) = remote_scripts(dry_run_lines(capsys, "remove-producer")) + removal = script.index("rm -rf /opt/isaaccapture/releases") + guards = [ + "list-unit-files", + "could not stop", + "still running after the stop", + "could not disable", + ] + positions = [script.index(guard) for guard in guards] + assert positions == sorted(positions) + assert positions[-1] < removal + assert script.index("rm -f /etc/systemd/system/") < removal + + +def deploy_script(monkeypatch, capsys): + monkeypatch.setattr(deploy_sender, "repo_root", lambda: Path("/repo")) + monkeypatch.setattr( + deploy_sender, + "git_output", + lambda root, *args: "" if args[0] == "status" else "abc123def", + ) + lines = dry_run_lines(capsys, "deploy", "--name", "rel") + return remote_scripts(lines) + + +def test_deploy_checks_for_lfs_remap_pointers_after_the_copy(monkeypatch, capsys): + scripts = deploy_script(monkeypatch, capsys) + checks = [s for s in scripts if "git-lfs.github.com/spec/v1" in s] + assert len(checks) == 1 + assert "*.remap" in checks[0] + assert scripts.index(checks[0]) == 1 # after prepare, before the build + + +@pytest.fixture +def lfs_check(monkeypatch, capsys, tmp_path): + script = next( + s + for s in deploy_script(monkeypatch, capsys) + if "git-lfs.github.com/spec/v1" in s + ) + release = tmp_path / "release" + (release / "src/configs").mkdir(parents=True) + script = script.replace(RELEASE, str(release)) + assert str(release) in script + + def run(): + return subprocess.run( + ["bash", "-c", script], capture_output=True, text=True, check=False + ) + + return release, run + + +def test_lfs_check_rejects_only_pointer_remaps(lfs_check): + release, run = lfs_check + (release / "src/configs/good.remap").write_bytes(b"\x00\x01binary") + (release / "src/configs/bad.remap").write_text(LFS_POINTER) + (release / "src/configs/notes.txt").write_text(LFS_POINTER) + result = run() + assert result.returncode != 0 + assert str(release / "src/configs/bad.remap") in result.stderr + assert "good.remap" not in result.stderr + assert "notes.txt" not in result.stderr + + +def test_lfs_check_passes_without_pointers(lfs_check): + release, run = lfs_check + (release / "src/configs/good.remap").write_bytes(b"\x00\x01binary") + result = run() + assert result.returncode == 0, result.stderr From ac93b44f0e66fe855f4e69a35da67b06de27cfcb Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 10:52:03 -0500 Subject: [PATCH 06/10] camera_streamer: add a stop-daemon command to the deploy tool The SIPL backend needs nvargus-daemon stopped so it can open the cameras. stop-daemon stops only that service, succeeds when it is already stopped, fails if it is still active afterwards, refuses while a sender started by run is alive, and never disables or uninstalls it. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../argus_sender/deploy/deploy_sender.py | 35 +++++++++++++++++++ .../argus_sender/tests/test_deploy_sender.py | 11 ++++++ 2 files changed, 46 insertions(+) diff --git a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py index 6737fcb89c..e07d58f522 100755 --- a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py +++ b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py @@ -10,6 +10,7 @@ deploy_sender.py user@orin deploy [--with-sipl] [--name NAME] deploy_sender.py user@orin run --name NAME --config FILE [--seconds N] [--env NAME=VALUE] deploy_sender.py user@orin stop stop the senders that `run` started + deploy_sender.py user@orin stop-daemon stop nvargus-daemon so SIPL capture can own the cameras deploy_sender.py user@orin reset [--reboot] restart nvargus-daemon (or reboot) to free a stuck camera deploy_sender.py user@orin remove-producer uninstall the sensing camera producer service and its releases deploy_sender.py user@orin linktest [--mb N] TCP throughput from the target to this machine @@ -20,6 +21,8 @@ refused rather than overwritten. `run` starts a built release for a bounded time with a config from this machine and opens the cameras; it prints a note, and does not stop anything, if another camera process is running. `run` succeeds when the sender ends at its time limit and fails with the sender's own exit status otherwise; `deploy` fails if any unit test fails. +`stop-daemon` stops nvargus-daemon (it returns at boot and holds the cameras that SIPL capture needs free); it is a no-op +when the daemon is already stopped, never disables or uninstalls it, and refuses while a sender started by `run` is alive. `reset` is the one command that changes the target's state: it needs passwordless sudo there, refuses while a sender started by `run` is alive, and otherwise restarts nvargus-daemon (or, with --reboot, reboots and waits for ssh to return). `remove-producer` also needs passwordless sudo: it stops and disables isaaccapture-producer.service, deletes its unit file @@ -185,6 +188,33 @@ def cmd_stop(target: str, dry_run: bool) -> None: ) +def cmd_stop_daemon(target: str, dry_run: bool) -> None: + """Stop nvargus-daemon so a SIPL sender can open the cameras; it is not disabled and starts again at boot. + + Needs passwordless sudo on the target (`sudo -n`). Succeeds when the daemon is already stopped and fails if it is + still active afterwards. Refuses while a sender started by `run` is alive, because it may be using the daemon. + """ + remote( + target, + r""" +if pgrep -f "^\./build/argus_sender --config src/configs/run-config.yaml" >/dev/null; then + echo "a sender from ~/camera-streamer/releases is running; run 'stop' first" >&2 + exit 1 +fi +if ! systemctl is-active --quiet nvargus-daemon.service; then echo "nvargus-daemon is not running"; exit 0; fi +sudo -n true || { echo "passwordless sudo is required on the target" >&2; exit 1; } +sudo -n systemctl stop nvargus-daemon.service || { echo "could not stop nvargus-daemon" >&2; exit 1; } +for _ in $(seq 1 20); do + systemctl is-active --quiet nvargus-daemon.service || { echo "nvargus-daemon stopped"; exit 0; } + sleep 1 +done +echo "nvargus-daemon is still active after 20 s" >&2 +exit 1 +""", + dry_run, + ) + + def cmd_reset(target: str, reboot: bool, dry_run: bool) -> None: """Free a camera that will not initialise: restart nvargus-daemon, or reboot the target. @@ -397,6 +427,9 @@ def main(argv: list[str] | None = None) -> int: sub.add_parser( "stop", help="stop the senders started from ~/camera-streamer/releases" ) + sub.add_parser( + "stop-daemon", help="stop nvargus-daemon so SIPL capture can own the cameras" + ) reset = sub.add_parser( "reset", help="restart nvargus-daemon (or reboot) to free a stuck camera" ) @@ -435,6 +468,8 @@ def main(argv: list[str] | None = None) -> int: cmd_check(args.target, args.dry_run) elif args.command == "stop": cmd_stop(args.target, args.dry_run) + elif args.command == "stop-daemon": + cmd_stop_daemon(args.target, args.dry_run) elif args.command == "reset": cmd_reset(args.target, args.reboot, args.dry_run) elif args.command == "remove-producer": diff --git a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py index ed82c391dd..cd3de27da9 100644 --- a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py +++ b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py @@ -61,6 +61,17 @@ def test_stop_and_reset_patterns_match_the_started_command(capsys): assert f'pgrep -f "{started}"' in reboot +def test_stop_daemon_only_stops_the_daemon_and_checks_the_result(capsys): + started = f"^\\./build/argus_sender --config {RUN_CONFIG}" + (script,) = remote_scripts(dry_run_lines(capsys, "stop-daemon")) + assert f'pgrep -f "{started}"' in script + assert "sudo -n systemctl stop nvargus-daemon.service" in script + assert "is not running" in script + assert "still active" in script + for broader in ("disable", "restart", "reboot", "rm "): + assert broader not in script + + def test_remove_producer_stops_before_removing(capsys): (script,) = remote_scripts(dry_run_lines(capsys, "remove-producer")) removal = script.index("rm -rf /opt/isaaccapture/releases") From c8679462bb000f6b48786522713d54a32b95b4ec Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 10:58:50 -0500 Subject: [PATCH 07/10] camera_streamer: drop the reboot and producer-removal commands from the deploy tool reset --reboot and remove-producer are station recovery and migration operations, not part of building and trying the sender. reset keeps restarting nvargus-daemon. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../argus_sender/deploy/deploy_sender.py | 98 ++----------------- .../argus_sender/tests/test_deploy_sender.py | 17 ---- 2 files changed, 10 insertions(+), 105 deletions(-) diff --git a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py index e07d58f522..30d7adce18 100755 --- a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py +++ b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py @@ -11,8 +11,7 @@ deploy_sender.py user@orin run --name NAME --config FILE [--seconds N] [--env NAME=VALUE] deploy_sender.py user@orin stop stop the senders that `run` started deploy_sender.py user@orin stop-daemon stop nvargus-daemon so SIPL capture can own the cameras - deploy_sender.py user@orin reset [--reboot] restart nvargus-daemon (or reboot) to free a stuck camera - deploy_sender.py user@orin remove-producer uninstall the sensing camera producer service and its releases + deploy_sender.py user@orin reset restart nvargus-daemon to free a stuck camera deploy_sender.py user@orin linktest [--mb N] TCP throughput from the target to this machine `deploy` copies the committed source of argus_sender (git archive of HEAD) into a new release directory under @@ -23,10 +22,8 @@ sender ends at its time limit and fails with the sender's own exit status otherwise; `deploy` fails if any unit test fails. `stop-daemon` stops nvargus-daemon (it returns at boot and holds the cameras that SIPL capture needs free); it is a no-op when the daemon is already stopped, never disables or uninstalls it, and refuses while a sender started by `run` is alive. -`reset` is the one command that changes the target's state: it needs passwordless sudo there, refuses while a sender started -by `run` is alive, and otherwise restarts nvargus-daemon (or, with --reboot, reboots and waits for ssh to return). -`remove-producer` also needs passwordless sudo: it stops and disables isaaccapture-producer.service, deletes its unit file -and /opt/isaaccapture/{current,releases}, and leaves /opt/isaaccapture/camera-platform.json and everything else alone. +`reset` also changes the target's state: it needs passwordless sudo there, refuses while a sender started +by `run` is alive, and otherwise restarts nvargus-daemon. """ from __future__ import annotations @@ -36,7 +33,6 @@ import shlex import subprocess import sys -import time from pathlib import Path RELEASES = "~/camera-streamer/releases" @@ -215,24 +211,20 @@ def cmd_stop_daemon(target: str, dry_run: bool) -> None: ) -def cmd_reset(target: str, reboot: bool, dry_run: bool) -> None: - """Free a camera that will not initialise: restart nvargus-daemon, or reboot the target. +def cmd_reset(target: str, dry_run: bool) -> None: + """Free a camera that will not initialise by restarting nvargus-daemon. Needs passwordless sudo on the target (`sudo -n`), so a missing password fails at once instead of hanging. Refuses while a sender started by `run` is alive, because restarting the daemon under it would kill its capture. """ - guard = r""" + remote( + target, + r""" if pgrep -f "^\./build/argus_sender --config src/configs/run-config.yaml" >/dev/null; then echo "a sender from ~/camera-streamer/releases is running; run 'stop' first" >&2 exit 1 fi sudo -n true || { echo "passwordless sudo is required on the target" >&2; exit 1; } -""" - if not reboot: - remote( - target, - guard - + r""" sudo -n systemctl restart nvargus-daemon.service for _ in $(seq 1 20); do systemctl is-active --quiet nvargus-daemon.service && { echo "nvargus-daemon is active"; exit 0; } @@ -240,66 +232,6 @@ def cmd_reset(target: str, reboot: bool, dry_run: bool) -> None: done echo "nvargus-daemon did not become active within 20 s" >&2 exit 1 -""", - dry_run, - ) - return - # The connection drops as the target goes down, so the reboot command's own exit status says nothing. - remote_ignoring_exit = ssh_command( - target, "bash -lc " + shlex.quote(guard + "sudo -n systemctl reboot") - ) - if dry_run: - print("DRY-RUN", shlex.join(remote_ignoring_exit)) - return - result = subprocess.run(remote_ignoring_exit, check=False) - if result.returncode not in (0, 255): - raise DeployError(f"could not start the reboot (exit code {result.returncode})") - # Wait for the target to go away, then for ssh to answer again, so the next command does not race the boot. - time.sleep(20) - deadline = time.monotonic() + 300 - while time.monotonic() < deadline: - if ( - subprocess.run( - ssh_command(target, "true"), check=False, capture_output=True - ).returncode - == 0 - ): - print("target is back") - return - time.sleep(5) - raise DeployError("the target did not answer ssh within 5 minutes of the reboot") - - -def cmd_remove_producer(target: str, dry_run: bool) -> None: - """Uninstall the sensing camera producer so the cameras belong to this sender alone. - - Removes only what the producer's own deploy creates: the service, its unit file and the release tree. The SIPL platform - file next to them is shared with this sender and stays. - """ - remote( - target, - r""" -sudo -n true || { echo "passwordless sudo is required on the target" >&2; exit 1; } -unit=isaaccapture-producer.service -echo "before:"; ls -la /opt/isaaccapture /etc/systemd/system/$unit 2>&1 | sed 's/^/ /' -# An installed unit must be confirmed stopped before its files go: a failed stop would otherwise leave a running -# producer with its unit and release tree deleted under it. An absent unit has nothing to stop. -if systemctl list-unit-files "$unit" --no-legend | grep -q .; then - sudo -n systemctl stop $unit || { echo "could not stop $unit; nothing was removed" >&2; exit 1; } - if systemctl is-active --quiet $unit || pgrep -x camera_plugin_s >/dev/null; then - echo "the producer is still running after the stop; nothing was removed" >&2 - exit 1 - fi - sudo -n systemctl disable $unit || { echo "could not disable $unit; nothing was removed" >&2; exit 1; } -fi -sudo -n rm -f /etc/systemd/system/$unit /opt/isaaccapture/current /opt/isaaccapture/current.new -sudo -n rm -rf /opt/isaaccapture/releases -sudo -n systemctl daemon-reload -sudo -n systemctl reset-failed $unit 2>/dev/null || true -echo "after:"; ls -la /opt/isaaccapture 2>&1 | sed 's/^/ /' -systemctl list-unit-files "$unit" --no-legend | grep -q . && { echo "$unit is still installed" >&2; exit 1; } -pgrep -x camera_plugin_s >/dev/null && { echo "camera_plugin_sensing is still running" >&2; exit 1; } -echo "producer removed" """, dry_run, ) @@ -430,15 +362,7 @@ def main(argv: list[str] | None = None) -> int: sub.add_parser( "stop-daemon", help="stop nvargus-daemon so SIPL capture can own the cameras" ) - reset = sub.add_parser( - "reset", help="restart nvargus-daemon (or reboot) to free a stuck camera" - ) - reset.add_argument( - "--reboot", action="store_true", help="reboot the target instead" - ) - sub.add_parser( - "remove-producer", help="uninstall the sensing camera producer service" - ) + sub.add_parser("reset", help="restart nvargus-daemon to free a stuck camera") link = sub.add_parser("linktest", help="TCP throughput from the target to this PC") link.add_argument("--mb", type=int, default=32, help="megabytes to send") run = sub.add_parser("run", help="run a deployed release for a bounded time") @@ -471,9 +395,7 @@ def main(argv: list[str] | None = None) -> int: elif args.command == "stop-daemon": cmd_stop_daemon(args.target, args.dry_run) elif args.command == "reset": - cmd_reset(args.target, args.reboot, args.dry_run) - elif args.command == "remove-producer": - cmd_remove_producer(args.target, args.dry_run) + cmd_reset(args.target, args.dry_run) elif args.command == "linktest": cmd_linktest(args.target, args.mb, args.dry_run) elif args.command == "run": diff --git a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py index cd3de27da9..6366a37dec 100644 --- a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py +++ b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py @@ -57,8 +57,6 @@ def test_stop_and_reset_patterns_match_the_started_command(capsys): assert stop.count(f'pgrep -f "{started}"') == 2 (reset,) = remote_scripts(dry_run_lines(capsys, "reset")) assert f'pgrep -f "{started}"' in reset - (reboot,) = remote_scripts(dry_run_lines(capsys, "reset", "--reboot")) - assert f'pgrep -f "{started}"' in reboot def test_stop_daemon_only_stops_the_daemon_and_checks_the_result(capsys): @@ -72,21 +70,6 @@ def test_stop_daemon_only_stops_the_daemon_and_checks_the_result(capsys): assert broader not in script -def test_remove_producer_stops_before_removing(capsys): - (script,) = remote_scripts(dry_run_lines(capsys, "remove-producer")) - removal = script.index("rm -rf /opt/isaaccapture/releases") - guards = [ - "list-unit-files", - "could not stop", - "still running after the stop", - "could not disable", - ] - positions = [script.index(guard) for guard in guards] - assert positions == sorted(positions) - assert positions[-1] < removal - assert script.index("rm -f /etc/systemd/system/") < removal - - def deploy_script(monkeypatch, capsys): monkeypatch.setattr(deploy_sender, "repo_root", lambda: Path("/repo")) monkeypatch.setattr( From c2415d15d8fdfc3053b6df7be8cf4209737c39c2 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 11:58:48 -0500 Subject: [PATCH 08/10] camera_streamer: make stop-daemon trust only a definite stopped state A failed or unrecognised systemctl state query is no longer taken as the daemon being stopped. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../argus_sender/deploy/deploy_sender.py | 11 +++++++---- .../argus_sender/tests/test_deploy_sender.py | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py index 30d7adce18..e8d0109003 100755 --- a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py +++ b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py @@ -188,23 +188,26 @@ def cmd_stop_daemon(target: str, dry_run: bool) -> None: """Stop nvargus-daemon so a SIPL sender can open the cameras; it is not disabled and starts again at boot. Needs passwordless sudo on the target (`sudo -n`). Succeeds when the daemon is already stopped and fails if it is - still active afterwards. Refuses while a sender started by `run` is alive, because it may be using the daemon. + still active afterwards or its state cannot be read. Refuses while a sender started by `run` is alive, because it + may be using the daemon. """ remote( target, r""" +state() { systemctl is-active nvargus-daemon.service 2>/dev/null || true; } +stopped() { case "$(state)" in inactive|failed) return 0;; *) return 1;; esac; } if pgrep -f "^\./build/argus_sender --config src/configs/run-config.yaml" >/dev/null; then echo "a sender from ~/camera-streamer/releases is running; run 'stop' first" >&2 exit 1 fi -if ! systemctl is-active --quiet nvargus-daemon.service; then echo "nvargus-daemon is not running"; exit 0; fi +if stopped; then echo "nvargus-daemon is not running"; exit 0; fi sudo -n true || { echo "passwordless sudo is required on the target" >&2; exit 1; } sudo -n systemctl stop nvargus-daemon.service || { echo "could not stop nvargus-daemon" >&2; exit 1; } for _ in $(seq 1 20); do - systemctl is-active --quiet nvargus-daemon.service || { echo "nvargus-daemon stopped"; exit 0; } + stopped && { echo "nvargus-daemon stopped"; exit 0; } sleep 1 done -echo "nvargus-daemon is still active after 20 s" >&2 +echo "nvargus-daemon is still $(state) after 20 s" >&2 exit 1 """, dry_run, diff --git a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py index 6366a37dec..f493e3e0ee 100644 --- a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py +++ b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py @@ -65,7 +65,10 @@ def test_stop_daemon_only_stops_the_daemon_and_checks_the_result(capsys): assert f'pgrep -f "{started}"' in script assert "sudo -n systemctl stop nvargus-daemon.service" in script assert "is not running" in script - assert "still active" in script + assert ( + "inactive|failed" in script + ) # only a definite stopped state counts; a query error is not "stopped" + assert "after 20 s" in script for broader in ("disable", "restart", "reboot", "rm "): assert broader not in script From 41655b5f84407e6cff80f4fbe4befab782a4f78e Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 12:39:42 -0500 Subject: [PATCH 09/10] camera_streamer: make the deploy tool's run report its real result The note about another camera process matched the shell running the script itself and printed on every run; it now matches a process named argus_sender. A login shell also runs the target's logout files on exit, so with errexit still on a failing last command there replaced the exit status chosen by run; errexit is turned off before the status is decided, and a run that ends at its time limit exits 0. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../argus_sender/deploy/deploy_sender.py | 5 ++++- .../argus_sender/tests/test_deploy_sender.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py index e8d0109003..5f83483c36 100755 --- a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py +++ b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py @@ -261,7 +261,7 @@ def cmd_run( if systemctl is-active --quiet camera-argus-sender.service 2>/dev/null; then echo "note: camera-argus-sender.service is active and may hold the cameras" >&2 fi -if pgrep -f '/argus_sender' >/dev/null; then +if pgrep -x argus_sender >/dev/null; then echo "note: another camera process is running" >&2 fi """ @@ -280,6 +280,9 @@ def cmd_run( # SIGINT first: the sender drains its encoder and returns every camera buffer on SIGINT/SIGTERM. status=0 {env_prefix}timeout --signal=INT --kill-after=15 {seconds} ./build/argus_sender --config {RUN_CONFIG} || status=$? +# errexit off for the rest: a login shell also runs the target's logout files on exit, and a failing last command in +# one would otherwise replace the exit status chosen here. +set +e # 124 is the time limit this command set; any other nonzero status is the sender's own and fails the run. if [[ $status -eq 124 ]]; then echo "sender stopped at its {seconds} s time limit"; exit 0; fi if [[ $status -ne 0 ]]; then echo "sender exited with status $status" >&2; exit $status; fi diff --git a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py index f493e3e0ee..6214942ca1 100644 --- a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py +++ b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py @@ -51,6 +51,19 @@ def test_run_uploads_to_and_starts_with_the_run_config(capsys, tmp_path): assert f"./build/argus_sender --config {RUN_CONFIG}" in script +def test_run_checks_for_other_senders_without_matching_itself_and_keeps_its_exit_status( + monkeypatch, capsys, tmp_path +): + config = tmp_path / "c.yaml" + config.write_text("x: 1\n") + lines = dry_run_lines(capsys, "run", "--name", "rel", "--config", str(config)) + (script,) = remote_scripts(lines[1:]) + # An exact process-name match; a command-line match would also find the shell whose command line holds this script. + assert "pgrep -x argus_sender" in script + assert "pgrep -f '/argus_sender'" not in script + assert script.index("set +e") < script.index("exit 0") + + def test_stop_and_reset_patterns_match_the_started_command(capsys): started = f"^\\./build/argus_sender --config {RUN_CONFIG}" (stop,) = remote_scripts(dry_run_lines(capsys, "stop")) From f9439e74f4292827b41bb856353b442bb4de4009 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 16:23:42 -0500 Subject: [PATCH 10/10] camera_streamer: make run say when the sender was stopped before its time limit timeout's timer follows the wall clock, so a forward step of the target's clock makes it exit 124 early, which run reported as reaching the limit. Elapsed time now comes from /proc/uptime, which a step does not move, and a 124 before about the requested time is reported as an early stop and fails. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../argus_sender/deploy/deploy_sender.py | 13 +++++++++++-- .../argus_sender/tests/test_deploy_sender.py | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py index 5f83483c36..135847a986 100755 --- a/examples/camera_streamer/argus_sender/deploy/deploy_sender.py +++ b/examples/camera_streamer/argus_sender/deploy/deploy_sender.py @@ -279,12 +279,21 @@ def cmd_run( cd $d # SIGINT first: the sender drains its encoder and returns every camera buffer on SIGINT/SIGTERM. status=0 +started=$(cut -d' ' -f1 /proc/uptime) {env_prefix}timeout --signal=INT --kill-after=15 {seconds} ./build/argus_sender --config {RUN_CONFIG} || status=$? +elapsed=$(awk -v s="$started" -v e="$(cut -d' ' -f1 /proc/uptime)" 'BEGIN {{ printf "%d", e - s }}') # errexit off for the rest: a login shell also runs the target's logout files on exit, and a failing last command in # one would otherwise replace the exit status chosen here. set +e -# 124 is the time limit this command set; any other nonzero status is the sender's own and fails the run. -if [[ $status -eq 124 ]]; then echo "sender stopped at its {seconds} s time limit"; exit 0; fi +# timeout's timer follows the wall clock, so a forward step of this host's clock (for example from a chrony restart) +# makes it exit 124 early. Elapsed time comes from /proc/uptime, which a step does not move: 124 means the time limit +# only if the run lasted about that long. +if [[ $status -eq 124 ]]; then + if (( elapsed + 2 >= {seconds} )); then echo "sender stopped at its {seconds} s time limit"; exit 0; fi + echo "sender was stopped after $elapsed s, before its {seconds} s limit (the wall clock stepped or timeout was signalled)" >&2 + exit 124 +fi +# Any other nonzero status is the sender's own and fails the run. if [[ $status -ne 0 ]]; then echo "sender exited with status $status" >&2; exit $status; fi """, dry_run, diff --git a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py index 6214942ca1..f48d2717d8 100644 --- a/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py +++ b/examples/camera_streamer/argus_sender/tests/test_deploy_sender.py @@ -62,6 +62,10 @@ def test_run_checks_for_other_senders_without_matching_itself_and_keeps_its_exit assert "pgrep -x argus_sender" in script assert "pgrep -f '/argus_sender'" not in script assert script.index("set +e") < script.index("exit 0") + # A 124 from timeout counts as the time limit only after about that long; a signalled timeout is a failure. + assert "/proc/uptime" in script # a wall-clock step must not hide an early stop + assert "elapsed + 2 >=" in script + assert "before its 60 s limit" in script def test_stop_and_reset_patterns_match_the_started_command(capsys):