From f6c3accdf35602b5b0dc3897f42c07fe29a6397b Mon Sep 17 00:00:00 2001 From: shitikyan Date: Fri, 26 Jun 2026 14:46:58 +0400 Subject: [PATCH 1/5] fix(tlsnotary): bake wstcp into the image + publish the proxy port range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TLSNotary verifications failed with nginx 502 / CloseEvent 1006 because the wstcp websocket→TCP proxy could never be reached, on two layers: 1. The runtime image ships no Rust toolchain, so ensureWstcp()'s on-demand `cargo install wstcp` fallback silently failed and the proxy never spawned. Bake the binary in via a dedicated `rust` build stage and copy it to the exact path proxyManager.ts probes ($HOME/.cargo/bin/wstcp). 2. The proxy binds a dynamic port in 55000-57000 inside the container, but docker-compose published none of them — so nginx forwarding /tlsn// to 127.0.0.1: hit a dead upstream. Publish a narrow, localhost-bound, env-configurable window; the allocation range (PORT_MIN/MAX, now env-overridable) and the published range read the same vars so they cannot drift. Manually binding one host port was a band-aid that broke on the next session's different port. This fixes both layers durably. Co-Authored-By: Claude Opus 4.8 (1M context) --- Dockerfile | 15 +++++++++++++++ docker-compose.yml | 13 +++++++++++++ src/features/tlsnotary/constants.ts | 14 ++++++++++---- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ff3b697bf..49355326e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -140,6 +140,16 @@ RUN if [ "$PRUNE_MODULES" = "true" ]; then \ # Stage 2: runtime # Minimal image with only what the node needs at run time. Runs as non-root. # ============================================================================= +# ── wstcp (TLSNotary websocket↔TCP proxy) ───────────────────────────── +# The TLSNotary flow spawns `wstcp` on demand to relay the browser's +# WebSocket to the target TLS server (see proxyManager.ts). The runtime +# image ships no Rust toolchain, so the on-demand `cargo install wstcp` +# fallback in ensureWstcp() cannot run — bake the binary in instead. +# Without it the proxy never binds its port and every verification fails +# with an nginx 502 / CloseEvent 1006 on the prover. +FROM rust:1-slim AS wstcp +RUN cargo install wstcp --root /wstcp + FROM oven/bun:1.3-debian AS runtime # OCI image metadata. @@ -178,6 +188,11 @@ RUN chmod 0755 /app/scripts/docker-entrypoint.sh \ && chown demos:demos /app /app/data /app/logs /app/state \ && chmod 0755 /app /app/data /app/logs /app/state +# TLSNotary proxy binary, baked in so the on-demand proxy can spawn +# without a Rust toolchain. Lands at $HOME/.cargo/bin/wstcp (HOME=/app) — +# the exact path proxyManager.ts::ensureWstcp() probes via `test -x`. +COPY --from=wstcp --chown=demos:demos /wstcp/bin/wstcp /app/.cargo/bin/wstcp + # Build-time provenance. These ARGs are populated by the build driver # (compose passes `git rev-parse HEAD` + `git rev-parse --abbrev-ref HEAD` # + `git diff --quiet; echo $?` + an ISO timestamp). They land in the diff --git a/docker-compose.yml b/docker-compose.yml index ed774c77a..5086c850c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,6 +173,13 @@ services: TLSNOTARY_MODE: ${TLSNOTARY_MODE:-docker} TLSNOTARY_FATAL: ${TLSNOTARY_FATAL:-false} TLSNOTARY_SIGNING_KEY: ${TLSNOTARY_SIGNING_KEY:-} + # wstcp proxy port window. The node spawns dynamic wstcp proxies in + # this range and the host reverse proxy forwards /tlsn// to them, + # so the SAME range must be host-published in `ports:` below — kept + # narrow to avoid a 2000-port mapping. Both stanzas read these vars so + # the allocation range and the published range can never drift. + TLSNOTARY_PROXY_PORT_MIN: ${TLSNOTARY_PROXY_PORT_MIN:-55000} + TLSNOTARY_PROXY_PORT_MAX: ${TLSNOTARY_PROXY_PORT_MAX:-55063} # Logging & misc LOG_LEVEL: ${LOG_LEVEL:-info} PROD: ${PROD:-false} @@ -198,6 +205,12 @@ services: - "${RPC_PORT:-53550}:${RPC_PORT:-53550}" - "${OMNI_PORT:-53551}:${OMNI_PORT:-53551}" - "${RPC_SIGNALING_PORT:-3005}:${RPC_SIGNALING_PORT:-3005}" + # wstcp TLSNotary proxy range, bound to 127.0.0.1 so ONLY the host's + # reverse proxy can reach it (not the public internet). The node + # allocates dynamic proxy ports here and nginx forwards + # /tlsn// to them — without this mapping every verification + # 502s. MUST match TLSNOTARY_PROXY_PORT_MIN/MAX in `environment:`. + - "127.0.0.1:${TLSNOTARY_PROXY_PORT_MIN:-55000}-${TLSNOTARY_PROXY_PORT_MAX:-55063}:${TLSNOTARY_PROXY_PORT_MIN:-55000}-${TLSNOTARY_PROXY_PORT_MAX:-55063}" # MCP (Model Context Protocol) intentionally NOT host-published. # The server binds `localhost` inside the container (src/index.ts) # and the SDK has no built-in authentication — publishing this port diff --git a/src/features/tlsnotary/constants.ts b/src/features/tlsnotary/constants.ts index f3a990711..e6e145b49 100644 --- a/src/features/tlsnotary/constants.ts +++ b/src/features/tlsnotary/constants.ts @@ -49,10 +49,16 @@ export const SIGNING_KEY_FILE_MODE = 0o600 * Configuration constants for port allocation and proxy lifecycle */ export const PORT_CONFIG = { - /** Minimum port number in the allocation range */ - PORT_MIN: 55000, - /** Maximum port number in the allocation range */ - PORT_MAX: 57000, + /** + * Minimum/maximum port for wstcp proxy allocation. Overridable via env so + * the published host range (docker-compose `ports:`) can be narrowed to a + * window the reverse proxy can actually reach — the proxies bind dynamic + * ports in this range and nginx forwards `/tlsn//` to them, so the + * range MUST be host-reachable or every verification 502s. Defaults keep + * the historical 55000-57000 behaviour. + */ + PORT_MIN: Number(process.env.TLSNOTARY_PROXY_PORT_MIN) || 55000, + PORT_MAX: Number(process.env.TLSNOTARY_PROXY_PORT_MAX) || 57000, /** Idle timeout before a proxy is considered stale (30 seconds) */ IDLE_TIMEOUT_MS: 30000, /** Maximum number of spawn retry attempts */ From 4ecc70b8cfeccc59b6eb39d64a58b8c009a5ac2d Mon Sep 17 00:00:00 2001 From: shitikyan Date: Fri, 26 Jun 2026 15:09:41 +0400 Subject: [PATCH 2/5] fix(tlsnotary): single PORT_CONFIG source so the env range drives allocation portAllocator.ts kept its own hardcoded PORT_CONFIG (55000-57000) and proxyManager imported PORT_CONFIG from it, so the env-overridable values in constants.ts were never used for allocation: the allocator would walk past the published window (e.g. 55064) and hand nginx an unreachable port, re-creating the 502 after the first 64 allocations. Point both consumers at constants.ts (the env-driven copy) and drop the duplicate. Addresses CodeRabbit/Greptile review on #949. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/features/tlsnotary/portAllocator.ts | 15 ++++----------- src/features/tlsnotary/proxyManager.ts | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/features/tlsnotary/portAllocator.ts b/src/features/tlsnotary/portAllocator.ts index 749826ddb..c08b26fcf 100644 --- a/src/features/tlsnotary/portAllocator.ts +++ b/src/features/tlsnotary/portAllocator.ts @@ -10,17 +10,10 @@ // REVIEW: TLSNotary port pool management for wstcp proxy instances import * as net from "net" import log from "@/utilities/logger" - -/** - * Configuration constants for port allocation - */ -export const PORT_CONFIG = { - PORT_MIN: 55000, - PORT_MAX: 57000, - IDLE_TIMEOUT_MS: 30000, // 30 seconds - MAX_SPAWN_RETRIES: 3, - SPAWN_TIMEOUT_MS: 5000, // 5 seconds to wait for wstcp to start -} +// Single source of truth for the proxy port window, so the env-overridable +// range (TLSNOTARY_PROXY_PORT_MIN/MAX) actually drives allocation — a local +// hardcoded copy here would silently ignore the published host range. +import { PORT_CONFIG } from "./constants" /** * Port pool state interface diff --git a/src/features/tlsnotary/proxyManager.ts b/src/features/tlsnotary/proxyManager.ts index 7110b2287..a954ca23b 100644 --- a/src/features/tlsnotary/proxyManager.ts +++ b/src/features/tlsnotary/proxyManager.ts @@ -36,12 +36,12 @@ import { promisify } from "util" import log from "@/utilities/logger" import { getSharedState } from "@/utilities/sharedState" import { - PORT_CONFIG, initPortPool, allocatePort, releasePort, type PortPoolState, } from "./portAllocator" +import { PORT_CONFIG } from "./constants" const execAsync = promisify(exec) From b56b20cef616b625375f84ba7f3be5400952d23d Mon Sep 17 00:00:00 2001 From: shitikyan Date: Fri, 26 Jun 2026 16:36:28 +0400 Subject: [PATCH 3/5] chore(tlsnotary): pin wstcp version + --locked for reproducible builds `cargo install wstcp` pulled the latest release at build time. Pin 0.2.1 and pass --locked so the image always builds the same wstcp and an upstream release can't change behaviour or break the build. Addresses CodeRabbit review on #949. Co-Authored-By: Claude Opus 4.8 (1M context) --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 49355326e..16e713d30 100644 --- a/Dockerfile +++ b/Dockerfile @@ -148,7 +148,9 @@ RUN if [ "$PRUNE_MODULES" = "true" ]; then \ # Without it the proxy never binds its port and every verification fails # with an nginx 502 / CloseEvent 1006 on the prover. FROM rust:1-slim AS wstcp -RUN cargo install wstcp --root /wstcp +# Pin the version + --locked so the image is reproducible and an upstream +# wstcp release can't silently change behaviour or break the build. +RUN cargo install wstcp --version 0.2.1 --locked --root /wstcp FROM oven/bun:1.3-debian AS runtime From 2fa26f7feae62b50aa24508091394076ff19a23f Mon Sep 17 00:00:00 2001 From: shitikyan Date: Fri, 26 Jun 2026 16:38:36 +0400 Subject: [PATCH 4/5] fix(tlsnotary): drop --locked from wstcp install (breaks the build) `--locked` forces wstcp 0.2.1's bundled Cargo.lock, whose pinned deps no longer compile on the current rust:1-slim toolchain (cargo exit 101). Keep the version pin for reproducibility but let cargo resolve compatible dependency versions. Validated: the stage builds and yields wstcp 0.2.1. Co-Authored-By: Claude Opus 4.8 (1M context) --- Dockerfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 16e713d30..1b9b880f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -148,9 +148,12 @@ RUN if [ "$PRUNE_MODULES" = "true" ]; then \ # Without it the proxy never binds its port and every verification fails # with an nginx 502 / CloseEvent 1006 on the prover. FROM rust:1-slim AS wstcp -# Pin the version + --locked so the image is reproducible and an upstream -# wstcp release can't silently change behaviour or break the build. -RUN cargo install wstcp --version 0.2.1 --locked --root /wstcp +# Pin the version so an upstream wstcp release can't silently change +# behaviour or break the build. `--locked` is intentionally omitted: the +# crate's bundled Cargo.lock pins dependency versions that no longer +# compile on the current toolchain, so we let cargo resolve compatible +# deps for this exact wstcp version. +RUN cargo install wstcp --version 0.2.1 --root /wstcp FROM oven/bun:1.3-debian AS runtime From 81b8fe91fbb9c5c4284259857f02772fa8ba66f4 Mon Sep 17 00:00:00 2001 From: shitikyan Date: Fri, 26 Jun 2026 16:59:12 +0400 Subject: [PATCH 5/5] fix(tlsnotary): publish the wstcp proxy range on the devnet stack too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The devnet override replaces the node `ports:` list (`!override`), which dropped the proxy-range publish from the base compose — so devnet verifications would still 502. Add a devnet-specific window (55100-55163, +100 offset from mainnet so the two stacks don't collide on host ports), with the allocation env and the published range driven by the same values. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.devnet.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker-compose.devnet.yml b/docker-compose.devnet.yml index 7c623e91d..aeef53181 100644 --- a/docker-compose.devnet.yml +++ b/docker-compose.devnet.yml @@ -56,6 +56,11 @@ services: container_name: demos-node-devnet environment: TLSNOTARY_PORT: 7147 + # wstcp proxy window, offset +100 from mainnet (55000-55063) so the + # two stacks don't fight over the same host ports. The allocation + # range and the published range below read these same values. + TLSNOTARY_PROXY_PORT_MIN: 55100 + TLSNOTARY_PROXY_PORT_MAX: 55163 networks: !override - demos-network-devnet # node_data/node_logs/node_state are isolated by the top-level `name:` @@ -69,6 +74,11 @@ services: - "53651:${OMNI_PORT:-53551}" - "3105:${RPC_SIGNALING_PORT:-3005}" - "9190:9090" + # wstcp TLSNotary proxy range (devnet window), localhost-bound so only + # the host reverse proxy reaches it. MUST match TLSNOTARY_PROXY_PORT_MIN/ + # MAX above. Host:container are equal (no offset) — the node advertises + # the container-internal port and nginx forwards /tlsn// to it. + - "127.0.0.1:55100-55163:55100-55163" prometheus: container_name: demos-prometheus-devnet