feat(observability): supervisor OTLP telemetry relay

AGENTS.md

@@ -57,7 +57,7 @@ Do not rely on this file for a full inventory. The detailed public and contribut
 | `crates/openshell-supervisor-middleware/` | Middleware runtime | Generic middleware registry, remote service integration, and chain execution |
 | `crates/openshell-supervisor-middleware-builtins/` | Built-in middleware | First-party in-process middleware implementations |
 | `crates/openshell-supervisor-network/` | Network supervisor | Proxying, L7 enforcement, policy evaluation, and provider credential injection |
-| `crates/openshell-supervisor-process/` | Process supervisor | Process lifecycle, namespace, and bypass monitoring |
+| `crates/openshell-supervisor-process/` | Process supervisor | SSH access, gateway session, log forwarding, and the OTLP telemetry relay |
 | `crates/openshell-vfio/` | VFIO support | PCI and GPU passthrough preparation and lifecycle |
 | `python/openshell/` | Python SDK | Python bindings and CLI packaging |
 | `sdk/typescript/` | TypeScript SDK | Native Connect client, curated sandbox API, and generated protobuf types |

Cargo.toml

@@ -71,6 +71,7 @@ tracing-appender = "0.2"
 opentelemetry = "0.32"
 opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
 opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] }
+opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace", "with-serde"] }
 tracing-opentelemetry = { version = "0.33", default-features = false, features = ["tracing-log"] }
 
 # Metrics

crates/openshell-core/src/sandbox_env.rs

@@ -266,11 +266,67 @@ pub const DEFAULT_SANDBOX_GID: u32 = 1000;
 /// OCI only for the former contract.
 pub const OCI_IMAGE_USER: &str = "OPENSHELL_OCI_IMAGE_USER";
 
+/// Standard OpenTelemetry environment variable for the OTLP exporter endpoint.
+///
+/// The gateway sets it in the sandbox environment when
+/// `[openshell.gateway.otlp]` is configured, pointing agent SDKs at
+/// [`OTLP_RELAY_ENDPOINT`]. A value the user declared explicitly wins.
+pub const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT";
+
+/// Standard OpenTelemetry environment variable for the OTLP exporter protocol.
+///
+/// Set to `http/protobuf` alongside [`OTEL_EXPORTER_OTLP_ENDPOINT`].
+pub const OTEL_EXPORTER_OTLP_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_PROTOCOL";
+
+/// Reserved destination agent processes export OTLP to.
+///
+/// This address is never routed. A workload `connect()` to any non-loopback
+/// address is intercepted by the sandbox seccomp broker and staged for the
+/// supervisor, which recognises this destination and serves the OTLP
+/// receiver on the staged stream itself instead of dialing upstream. So the
+/// value is a label the supervisor switches on, and it must stay outside
+/// loopback (loopback connects complete locally without mediation) and
+/// outside `198.18.0.0/15`, which the policy DNS runtime uses for synthetic
+/// answers. `192.0.0.8` is the IPv4 dummy address (RFC 7600): reserved,
+/// unroutable, and never assigned to a real service.
+pub const OTLP_RELAY_ADDR: &str = "192.0.0.8:4318";
+
+/// [`OTLP_RELAY_ADDR`] as the URL injected through
+/// [`OTEL_EXPORTER_OTLP_ENDPOINT`].
+pub const OTLP_RELAY_ENDPOINT: &str = "http://192.0.0.8:4318";
+
 // The corporate upstream-proxy configuration deliberately has no reserved
 // environment variables: it travels on the supervisor's argv
 // (`--upstream-proxy` and friends), which a sandbox image cannot forge the
 // way it could bake `ENV` values.
 
+#[cfg(test)]
+mod otlp_relay_address_tests {
+    use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+
+    use super::{OTLP_RELAY_ADDR, OTLP_RELAY_ENDPOINT};
+
+    #[test]
+    fn relay_address_is_an_unroutable_non_loopback_label() {
+        let addr: SocketAddr = OTLP_RELAY_ADDR.parse().expect("relay address parses");
+        assert_eq!(addr.port(), 4318, "OTLP HTTP default port");
+        let IpAddr::V4(ip) = addr.ip() else {
+            panic!("relay address must be IPv4");
+        };
+        assert!(
+            !ip.is_loopback(),
+            "loopback would bypass seccomp mediation and never reach the supervisor"
+        );
+        assert_eq!(ip, Ipv4Addr::new(192, 0, 0, 8), "RFC 7600 dummy address");
+        let [a, b, ..] = ip.octets();
+        assert!(
+            !(a == 198 && (b == 18 || b == 19)),
+            "198.18.0.0/15 is the policy DNS synthetic answer pool"
+        );
+        assert_eq!(OTLP_RELAY_ENDPOINT, format!("http://{OTLP_RELAY_ADDR}"));
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/openshell-gateway/src/vm.rs

@@ -810,6 +810,7 @@ mod tests {
             Some(&OtlpConfig {
                 endpoint: "http://collector.internal:4317".to_string(),
                 service_name: Some("custom-gateway".to_string()),
+                agent_endpoint: None,
             }),
             "production-us-west",
         );

crates/openshell-ocsf/src/lib.rs

@@ -64,6 +64,7 @@ pub use builders::{
 
 // --- Tracing layers ---
 pub use tracing_layers::{
-    OCSF_TARGET, OcsfJsonlLayer, OcsfShorthandLayer, clear_current_event, clone_current_event,
-    emit_ocsf_event, emit_ocsf_event_routed, set_current_event,
+    OCSF_TARGET, OcsfJsonlLayer, OcsfRelayLayer, OcsfRelaySink, OcsfShorthandLayer,
+    clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed,
+    set_current_event,
 };

crates/openshell-ocsf/src/tracing_layers/mod.rs

@@ -9,11 +9,13 @@
 
 pub(crate) mod event_bridge;
 mod jsonl_layer;
+mod relay_layer;
 mod shorthand_layer;
 
 pub use event_bridge::{
     OCSF_TARGET, clear_current_event, clone_current_event, emit_ocsf_event, emit_ocsf_event_routed,
     set_current_event,
 };
 pub use jsonl_layer::OcsfJsonlLayer;
+pub use relay_layer::{OcsfRelayLayer, OcsfRelaySink};
 pub use shorthand_layer::OcsfShorthandLayer;

crates/openshell-ocsf/src/tracing_layers/relay_layer.rs

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Tracing layer that captures OCSF events and forwards them as JSON bytes
+//! through a telemetry buffer sender for relay to the gateway.
+
+use std::sync::Arc;
+
+use tracing::Subscriber;
+use tracing_subscriber::Layer;
+use tracing_subscriber::layer::Context;
+
+use super::event_bridge::{OCSF_TARGET, clone_current_event};
+
+/// Callback trait for delivering serialized OCSF events.
+pub trait OcsfRelaySink: Send + Sync + 'static {
+    fn send(&self, json_bytes: Vec<u8>);
+}
+
+/// A tracing layer that captures OCSF events and serializes them to JSON
+/// for relay through the telemetry transport.
+pub struct OcsfRelayLayer {
+    sink: Arc<dyn OcsfRelaySink>,
+}
+
+impl OcsfRelayLayer {
+    pub fn new(sink: Arc<dyn OcsfRelaySink>) -> Self {
+        Self { sink }
+    }
+}
+
+impl<S: Subscriber> Layer<S> for OcsfRelayLayer {
+    fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) {
+        if event.metadata().target() != OCSF_TARGET {
+            return;
+        }
+
+        let Some(ocsf_event) = clone_current_event() else {
+            return;
+        };
+
+        if let Ok(json) = serde_json::to_vec(&ocsf_event) {
+            self.sink.send(json);
+        }
+    }
+}

crates/openshell-otel/src/lib.rs

@@ -5,7 +5,7 @@
 
 mod driver;
 mod grpc;
-mod propagation;
+pub mod propagation;
 
 pub use driver::{
     BoxGrpcStream, ComputeDriverTracing, DriverTracingConfig, DriverTracingHandle,

crates/openshell-otel/src/propagation.rs

@@ -54,6 +54,22 @@ impl Injector for MetadataMapInjector<'_> {
     }
 }
 
+/// Writes OpenTelemetry propagation fields to HTTP headers.
+#[derive(Debug)]
+pub struct HeaderMapInjector<'a>(pub &'a mut HeaderMap);
+
+impl Injector for HeaderMapInjector<'_> {
+    fn set(&mut self, key: &str, value: String) {
+        let Ok(key) = key.parse::<http::header::HeaderName>() else {
+            return;
+        };
+        let Ok(value) = value.parse() else {
+            return;
+        };
+        self.0.insert(key, value);
+    }
+}
+
 #[derive(Debug)]
 struct TraceContextMapInjector<'a>(&'a mut BTreeMap<String, String>);
 
@@ -75,6 +91,15 @@ pub fn current_trace_context_carrier() -> Option<BTreeMap<String, String>> {
     carrier.contains_key("traceparent").then_some(carrier)
 }
 
+/// Inject W3C traceparent into HTTP headers if not already present.
+pub fn inject_traceparent_if_missing(headers: &mut HeaderMap) {
+    if headers.contains_key("traceparent") {
+        return;
+    }
+    let context = tracing::Span::current().context();
+    TraceContextPropagator::new().inject_context(&context, &mut HeaderMapInjector(headers));
+}
+
 /// Injects the active W3C trace context into an outbound tonic request.
 #[derive(Debug, Clone, Copy)]
 pub struct TraceContextInterceptor;

crates/openshell-server/Cargo.toml

@@ -75,6 +75,7 @@ tracing-appender = { workspace = true }
 # OpenTelemetry (OTLP trace export, opt-in via [openshell.gateway.otlp])
 opentelemetry = { workspace = true }
 opentelemetry_sdk = { workspace = true }
+opentelemetry-proto = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 
 # Metrics

crates/openshell-server/src/config_file.rs

@@ -288,6 +288,19 @@ pub struct OtlpConfig {
     /// `service.name` resource attribute. Defaults to `openshell-gateway`.
     #[serde(default)]
     pub service_name: Option<String>,
+
+    /// OTLP/gRPC collector endpoint for relayed agent traces. When absent,
+    /// agent traces go to `endpoint`, so one collector serves both lanes
+    /// unless the operator splits them.
+    #[serde(default)]
+    pub agent_endpoint: Option<String>,
+}
+
+impl OtlpConfig {
+    /// The collector endpoint for the agent-trace lane.
+    pub fn agent_lane_endpoint(&self) -> &str {
+        self.agent_endpoint.as_deref().unwrap_or(&self.endpoint)
+    }
 }
 
 /// `[openshell.supervisor]` section.
@@ -921,6 +934,21 @@ service_name = "openshell-gateway-dev"
             "http://otel-collector.observability.svc:4317"
         );
         assert_eq!(otlp.service_name.as_deref(), Some("openshell-gateway-dev"));
+        assert_eq!(otlp.agent_lane_endpoint(), otlp.endpoint);
+    }
+
+    #[test]
+    fn otlp_agent_endpoint_splits_the_agent_lane() {
+        let toml = r#"
+[openshell.gateway.otlp]
+endpoint = "http://infra-collector:4317"
+agent_endpoint = "http://agent-collector:4317"
+"#;
+        let tmp = write_tmp(toml);
+        let file = load(tmp.path()).expect("valid otlp config parses");
+        let otlp = file.openshell.gateway.otlp.expect("otlp config");
+        assert_eq!(otlp.endpoint, "http://infra-collector:4317");
+        assert_eq!(otlp.agent_lane_endpoint(), "http://agent-collector:4317");
     }
 
     #[test]

crates/openshell-server/src/grpc/provider_readiness_tests.rs

@@ -71,6 +71,7 @@ fn hello() -> SupervisorHello {
         sandbox_id: Uuid::new_v4().to_string(),
         instance_id: Uuid::new_v4().to_string(),
         connection_epoch: 0,
+        capabilities: Vec::new(),
         supports_provider_readiness: true,
     }
 }