From 3d0fff1077996b9092c78a3351ef8b1ff2c00ad2 Mon Sep 17 00:00:00 2001 From: Shiju Date: Thu, 1 Oct 2026 13:49:50 +0530 Subject: [PATCH 1/7] feat(server): separate image preparation and admission deadlines Give sandbox image preparation its own deadline and start the admission deadline after preparation finishes. Persist both phases across gateway restarts and show recovery guidance for the phase that expired. Preserve the current service authorization schema and regenerate the Go bindings with the preparation timestamps. Fixes #3952 Related to #3955 Signed-off-by: Shiju --- crates/openshell-cli/src/run.rs | 109 ++++- crates/openshell-core/src/config.rs | 5 + crates/openshell-server/src/cli.rs | 49 +++ crates/openshell-server/src/compute/mod.rs | 246 +++++++++-- .../src/compute/provisioning_deadline.rs | 211 +++++++++- crates/openshell-server/src/config_file.rs | 3 + crates/openshell-server/src/grpc/policy.rs | 147 ++++++- crates/openshell-server/src/grpc/sandbox.rs | 7 +- crates/openshell-server/src/lib.rs | 3 + crates/openshell-server/src/storage_proto.rs | 7 +- crates/openshell-tui/src/lib.rs | 27 +- docs/how-it-works/gateways/configuration.mdx | 4 + .../how-it-works/policies/manage-policies.mdx | 7 +- docs/how-it-works/sandboxes/overview.mdx | 22 +- proto/openshell.proto | 12 +- sdk/go/proto/openshellv1/openshell.pb.go | 386 ++++++++++-------- 16 files changed, 989 insertions(+), 256 deletions(-) diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 7432c0175f..65c5fc2f34 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1218,14 +1218,15 @@ pub async fn sandbox_create( SandboxPhase::Error => { drop(stream); drop(client); - let provisioning_timed_out = last_sandbox + let timed_out_provisioning = last_sandbox .status .as_ref() .and_then(|status| status.provisioning.as_ref()) - .is_some_and(|record| record.timeout_time.is_some()); - let create_result = if provisioning_timed_out { + .filter(|record| record.timeout_time.is_some()); + let create_result = if let Some(record) = timed_out_provisioning { Err(miette::miette!( - "{last_error_reason}\nSandbox '{sandbox_name}' was retained. Inspect it with `openshell sandbox get {sandbox_name}`; repair its configuration, then run `openshell sandbox start {sandbox_name}` after cleanup completes." + "{}", + retained_sandbox_timeout_message(&sandbox_name, &last_error_reason, record) )) } else if last_error_reason.is_empty() { Err(miette::miette!( @@ -1259,6 +1260,24 @@ pub async fn sandbox_create( } } +/// Use the persisted phase to select recovery guidance. Preparation may expire +/// before any policy is evaluated, so it must not tell the user to repair policy. +fn retained_sandbox_timeout_message( + sandbox_name: &str, + error_reason: &str, + record: &openshell_core::proto::SandboxProvisioning, +) -> String { + let recovery = if record.preparation_deadline.is_some() && record.admission_start_time.is_none() + { + "check image preparation and supervisor startup diagnostics and the gateway's `image_preparation_timeout_seconds` budget" + } else { + "repair its configuration" + }; + format!( + "{error_reason}\nSandbox '{sandbox_name}' was retained. Inspect it with `openshell sandbox get {sandbox_name}`; {recovery}, then run `openshell sandbox start {sandbox_name}` after cleanup completes." + ) +} + /// Resolved source for the `--from` flag on `sandbox create`. #[derive(Debug)] enum ResolvedSource { @@ -2904,6 +2923,15 @@ fn sandbox_to_json(sandbox: &Sandbox) -> serde_json::Value { "configuration_change_id": record.configuration_change_id, "configuration_change_time": record.configuration_change_time.as_ref().map(ToString::to_string), "first_rejection_time": record.first_rejection_time.as_ref().map(ToString::to_string), + "phase": if record.deadline.is_none() && record.timeout_time.is_none() { + "ready" + } else if record.preparation_deadline.is_some() && record.admission_start_time.is_none() { + "preparation" + } else { + "admission" + }, + "preparation_deadline": record.preparation_deadline.as_ref().map(ToString::to_string), + "admission_start_time": record.admission_start_time.as_ref().map(ToString::to_string), "deadline": record.deadline.as_ref().map(ToString::to_string), "timeout_time": record.timeout_time.as_ref().map(ToString::to_string), "cleanup_completed_time": record.cleanup_completed_time.as_ref().map(ToString::to_string), @@ -8110,6 +8138,79 @@ mod tests { ); } + #[test] + fn retained_sandbox_timeout_message_matches_expired_phase() { + let mut record = openshell_core::proto::SandboxProvisioning { + preparation_deadline: openshell_core::time::timestamp_from_millis(1_800_000).ok(), + timeout_time: openshell_core::time::timestamp_from_millis(1_800_000).ok(), + ..Default::default() + }; + let message = super::retained_sandbox_timeout_message( + "cold-image", + "ImagePreparationTimedOut: preparation expired", + &record, + ); + assert!(message.starts_with("ImagePreparationTimedOut: preparation expired\n")); + assert!(message.contains("Sandbox 'cold-image' was retained")); + assert!(message.contains("image preparation and supervisor startup diagnostics")); + assert!(message.contains("image_preparation_timeout_seconds")); + assert!(!message.contains("repair its configuration")); + assert!(message.contains("openshell sandbox get cold-image")); + assert!(message.contains("openshell sandbox start cold-image` after cleanup completes")); + + // An admission timeout retains preparation timestamps. Its completed + // transition must select configuration repair rather than a larger budget. + record.admission_start_time = openshell_core::time::timestamp_from_millis(600_000).ok(); + let admission_without_preparation = openshell_core::proto::SandboxProvisioning { + timeout_time: record.timeout_time, + ..Default::default() + }; + for admission_record in [&record, &admission_without_preparation] { + let message = super::retained_sandbox_timeout_message( + "invalid-policy", + "ProvisioningTimedOut: repair window expired", + admission_record, + ); + assert!(message.contains("repair its configuration")); + assert!(!message.contains("image_preparation_timeout_seconds")); + assert!( + message.contains("openshell sandbox start invalid-policy` after cleanup completes") + ); + } + } + + #[test] + fn provisioning_json_distinguishes_preparation_and_admission() { + let mut sandbox = Sandbox::default(); + sandbox.set_phase(SandboxPhase::Provisioning.into()); + let ceiling = openshell_core::time::timestamp_from_millis(1_800_000).ok(); + sandbox.status.as_mut().unwrap().provisioning = + Some(openshell_core::proto::SandboxProvisioning { + preparation_deadline: ceiling, + deadline: ceiling, + ..Default::default() + }); + let json = super::sandbox_to_json(&sandbox); + assert_eq!(json["provisioning"]["phase"], "preparation"); + assert_eq!( + json["provisioning"]["preparation_deadline"], + "1970-01-01T00:30:00Z" + ); + assert!(json["provisioning"]["admission_start_time"].is_null()); + sandbox + .status + .as_mut() + .unwrap() + .provisioning + .as_mut() + .unwrap() + .admission_start_time = openshell_core::time::timestamp_from_millis(600_000).ok(); + assert_eq!( + super::sandbox_to_json(&sandbox)["provisioning"]["phase"], + "admission" + ); + } + #[test] fn sandbox_json_exposes_repair_diagnostic_and_accepted_generation() { use openshell_core::proto::{ConfigurationAdmissionState, SandboxConfigurationAdmission}; diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 820b4a9476..35785f20f9 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -240,6 +240,10 @@ pub struct Config { /// TTL for SSH session tokens, in seconds. 0 disables expiry. pub ssh_session_ttl_secs: u64, + /// Absolute image preparation and initial supervisor startup budget for new + /// sandbox attempts, in seconds. Must be between 1 and 86400, inclusive. + pub image_preparation_timeout_seconds: u32, + /// Maximum gRPC requests allowed per rate-limit window. /// /// When paired with [`Self::grpc_rate_limit_window_secs`], positive values @@ -865,6 +869,7 @@ impl Config { credential_drivers: Vec::new(), default_credential_driver: None, ssh_session_ttl_secs: default_ssh_session_ttl_secs(), + image_preparation_timeout_seconds: 1800, grpc_rate_limit_requests: None, grpc_rate_limit_window_secs: None, service_routing: ServiceRoutingConfig::default(), diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 87b67f5e8a..49e6f05c7f 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -551,6 +551,18 @@ fn prepare_server_config_with_drivers( config.policy_validation_failure_mode = mode; } + if let Some(seconds) = file + .as_ref() + .and_then(|f| f.openshell.gateway.image_preparation_timeout_seconds) + { + if !(1..=86_400).contains(&seconds) { + return Err(miette::miette!( + "image_preparation_timeout_seconds must be between 1 and 86400" + )); + } + config.image_preparation_timeout_seconds = seconds; + } + if let Some(issuer) = args.oidc_issuer.clone() { config = config.with_oidc(openshell_core::OidcConfig { issuer, @@ -3251,6 +3263,7 @@ version = 2 [openshell.gateway] policy_validation_failure_mode = "retain_last_valid" +image_preparation_timeout_seconds = 2400 [openshell.drivers.docker] unknown_docker_key = true @@ -3276,6 +3289,7 @@ mem_mib = "not-a-number" super::prepare_server_config(&mut args, &matches).expect("server config is prepared"); assert_eq!(prepared.config.compute_driver.as_deref(), Some("podman")); + assert_eq!(prepared.config.image_preparation_timeout_seconds, 2400); assert_eq!( prepared.config.policy_validation_failure_mode, openshell_core::PolicyValidationFailureMode::RetainLastValid @@ -3284,4 +3298,39 @@ mem_mib = "not-a-number" assert!(file.openshell.drivers.contains_key("docker")); assert!(file.openshell.drivers.contains_key("vm")); } + + #[test] + fn server_config_rejects_unbounded_image_preparation() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let state = tempfile::tempdir().unwrap(); + let tls = tempfile::tempdir().unwrap(); + let _state = EnvVarGuard::set("XDG_STATE_HOME", state.path().to_str().unwrap()); + let _tls = EnvVarGuard::set("OPENSHELL_LOCAL_TLS_DIR", tls.path().to_str().unwrap()); + let config_path = state.path().join("gateway.toml"); + for seconds in [0, 86_401] { + std::fs::write(&config_path, format!( + "[openshell]\nversion = 2\n[openshell.gateway]\nimage_preparation_timeout_seconds = {seconds}\n" + )).unwrap(); + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--config", + config_path.to_str().unwrap(), + "--db-url", + "sqlite::memory:", + "--compute-driver", + "podman", + "--disable-tls", + ]); + let Err(error) = super::prepare_server_config(&mut args, &matches) else { + panic!("unbounded preparation must be rejected"); + }; + assert!( + error + .to_string() + .contains("image_preparation_timeout_seconds must be between 1 and 86400") + ); + } + } } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 088231f6e1..48b5655abd 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -644,6 +644,7 @@ pub struct ComputeRuntime { telemetry_compute_driver: TelemetryComputeDriver, driver_process: Option>, default_image: String, + image_preparation_timeout_seconds: u32, store: Arc, sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, @@ -745,6 +746,7 @@ impl ComputeRuntime { telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process, default_image, + image_preparation_timeout_seconds: 1800, store, sandbox_index, sandbox_watch_bus, @@ -826,6 +828,16 @@ impl ComputeRuntime { &self.default_image } + /// Validate the budget once at startup. Persisted attempts keep their + /// original deadline even if the operator changes this value on restart. + pub(crate) fn with_image_preparation_timeout(mut self, seconds: u32) -> Result { + if !(1..=86_400).contains(&seconds) { + return Err("image_preparation_timeout_seconds must be between 1 and 86400".into()); + } + self.image_preparation_timeout_seconds = seconds; + Ok(self) + } + #[must_use] pub fn driver_info_snapshots(&self) -> &[ComputeDriverInfoSnapshot] { std::slice::from_ref(&self.driver_info) @@ -1618,6 +1630,7 @@ impl ComputeRuntime { SandboxPhase::Starting, "Starting", "Sandbox start requested", + self.image_preparation_timeout_seconds, ); }, ) @@ -1699,7 +1712,7 @@ impl ComputeRuntime { .await.map_err(|error| Status::internal(error.to_string()))? .ok_or_else(|| Status::not_found("sandbox removed during startup"))?; if provisioning_deadline::timed_out(¤t) { - return Err(Status::deadline_exceeded("provisioning repair window expired")); + return Err(Status::deadline_exceeded("provisioning deadline expired")); } } } @@ -2104,7 +2117,13 @@ impl ComputeRuntime { &sandbox_id, expected_resource_version, move |sandbox| { - apply_lifecycle_phase(sandbox, phase, &reason, &message); + apply_lifecycle_phase( + sandbox, + phase, + &reason, + &message, + self.image_preparation_timeout_seconds, + ); }, ) .await @@ -3312,22 +3331,27 @@ impl ComputeRuntime { } }; let expected_runtime_identity = sandbox_compute_runtime_identity(&sandbox); + // Recovery retains the original attempt and deadline. A + // stalled driver must release this lifecycle gate after + // expiry so the deadline worker can reclaim its compute. if let Err(err) = self - .driver - .call( - openshell_otel::rpc::START_SANDBOX, - Some(&sandbox_id), - |driver| async move { - driver - .start_sandbox(Request::new(StartSandboxRequest { - sandbox_id: driver_sandbox_id, - name: sandbox_name, - launch_authentication: Vec::new(), - generation_id, - expected_runtime_identity, - })) - .await - }, + .await_provisioning_operation( + &sandbox, + self.driver.call( + openshell_otel::rpc::START_SANDBOX, + Some(&sandbox_id), + |driver| async move { + driver + .start_sandbox(Request::new(StartSandboxRequest { + sandbox_id: driver_sandbox_id, + name: sandbox_name, + launch_authentication: Vec::new(), + generation_id, + expected_runtime_identity, + })) + .await + }, + ), ) .await { @@ -6553,7 +6577,13 @@ fn is_recoverable_error_reason(sandbox: &Sandbox) -> bool { .is_some_and(|c| c.reason == CONDITION_RUNTIME_RESTART || c.reason == CONDITION_STOPPED) } -fn apply_lifecycle_phase(sandbox: &mut Sandbox, phase: SandboxPhase, reason: &str, message: &str) { +fn apply_lifecycle_phase( + sandbox: &mut Sandbox, + phase: SandboxPhase, + reason: &str, + message: &str, + image_preparation_timeout_seconds: u32, +) { sandbox.set_phase(phase as i32); if matches!(phase, SandboxPhase::Stopping | SandboxPhase::Starting) { let status = sandbox.status.get_or_insert_with(Default::default); @@ -6564,8 +6594,9 @@ fn apply_lifecycle_phase(sandbox: &mut Sandbox, phase: SandboxPhase, reason: &st set_next_restart_at_ms(status, 0); set_main_process_started_at_ms(status, 0); if phase == SandboxPhase::Starting { - status.provisioning = Some(provisioning_deadline::new_record( + status.provisioning = Some(provisioning_deadline::new_preparation_record( openshell_core::time::now_ms(), + image_preparation_timeout_seconds, )); status.configuration_admission = Some(openshell_core::proto::SandboxConfigurationAdmission { @@ -6940,6 +6971,7 @@ pub fn new_test_runtime_with_driver( telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process: None, default_image: "openshell/sandbox:test".to_string(), + image_preparation_timeout_seconds: 1800, store, sandbox_index: SandboxIndex::new(), sandbox_watch_bus: SandboxWatchBus::new(), @@ -7989,6 +8021,7 @@ mod tests { telemetry_compute_driver: TelemetryComputeDriver::custom(), driver_process: None, default_image: "openshell/sandbox:test".to_string(), + image_preparation_timeout_seconds: 1800, store, sandbox_index: SandboxIndex::new(), sandbox_watch_bus: SandboxWatchBus::new(), @@ -10612,18 +10645,19 @@ mod tests { assert!(!crate::policy_store::permits_initial_static_policy_repair( &blocked )); - // Simulate the supervisor's successful exact-generation admission report. + // The supervisor report records the preparation-to-admission transition + // together with acceptance of the exact configuration generation. runtime .store .update_message_cas::(sandbox.object_id(), 0, |sandbox| { - sandbox - .status - .as_mut() - .unwrap() - .configuration_admission - .as_mut() - .unwrap() - .state = openshell_core::proto::ConfigurationAdmissionState::Accepted.into(); + let status = sandbox.status.as_mut().unwrap(); + provisioning_deadline::record_admission_start( + status.provisioning.as_mut().unwrap(), + openshell_core::time::now_ms(), + ) + .unwrap(); + status.configuration_admission.as_mut().unwrap().state = + openshell_core::proto::ConfigurationAdmissionState::Accepted.into(); }) .await .unwrap(); @@ -15369,6 +15403,162 @@ mod tests { .unwrap() } + #[tokio::test] + async fn preparation_expiry_releases_stalled_start_recovery_for_cleanup() { + let driver = ControlledDriver::new(); + driver.block_start(); + let runtime = test_runtime(driver.clone()).await; + let now = openshell_core::time::now_ms(); + let preparation = provisioning_deadline::new_preparation_record(now, 1800); + let mut sandbox = sandbox_record("sb-recovery-ttl", "recovery-ttl", SandboxPhase::Starting); + sandbox.status.as_mut().unwrap().provisioning = Some(preparation.clone()); + runtime.store.put_message(&sandbox).await.unwrap(); + let recovered_runtime = runtime.clone(); + let mut recovery = tokio::spawn(async move { + recovered_runtime + .recover_persisted_lifecycle_transitions() + .await + }); + tokio::time::timeout(Duration::from_secs(1), driver.start_started.notified()) + .await + .unwrap(); + runtime + .reconcile_provisioning_deadlines(now + 1_800_000) + .await + .unwrap(); + let expired = runtime + .store + .get_message::("sb-recovery-ttl") + .await + .unwrap() + .unwrap(); + assert_eq!(expired.phase(), i32::from(SandboxPhase::Error)); + let status = expired.status.as_ref().unwrap(); + let record = status.provisioning.as_ref().unwrap(); + assert_eq!(record.attempt_id, preparation.attempt_id); + assert_eq!( + record.preparation_deadline, + preparation.preparation_deadline + ); + assert!( + status + .conditions + .iter() + .any(|condition| condition.reason == "ImagePreparationTimedOut") + ); + // The recovery RPC never receives its semaphore permit. The persisted + // expiry must cancel its waiter and release the lifecycle gate itself. + if let Ok(result) = tokio::time::timeout(Duration::from_secs(3), &mut recovery).await { + result.unwrap().unwrap(); + } else { + recovery.abort(); + let _ = recovery.await; + panic!("expired recovery kept the lifecycle gate while the driver was blocked"); + } + runtime + .reconcile_provisioning_deadlines(now + 1_800_001) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + let sandbox = runtime + .store + .get_message::("sb-recovery-ttl") + .await + .unwrap() + .unwrap(); + let record = sandbox + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap(); + if record.cleanup_completed_time.is_some() { + assert_eq!(record.attempt_id, preparation.attempt_id); + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(driver.stop_calls(), 1); + } + + #[tokio::test] + async fn preparation_timeout_retains_reason_and_uses_existing_cleanup() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let mut sandbox = sandbox_record("sb-prepare", "prepare", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record(0, 1800)); + runtime.store.put_message(&sandbox).await.unwrap(); + let sandbox = runtime + .store + .get_message::("sb-prepare") + .await + .unwrap() + .unwrap(); + let gate = runtime.lifecycle_gates.lock_for("sb-prepare").await; + let global = runtime.lock_global_for_lifecycle(&gate).await; + assert!( + runtime + .claim_provisioning_timeout(&sandbox, 600_000) + .await + .unwrap() + .is_none() + ); + let expired = runtime + .claim_provisioning_timeout(&sandbox, 1_800_000) + .await + .unwrap() + .unwrap(); + assert_eq!(expired.phase(), i32::from(SandboxPhase::Error)); + assert!( + expired + .status + .as_ref() + .unwrap() + .conditions + .iter() + .any(|condition| { condition.reason == "ImagePreparationTimedOut" }) + ); + let record = expired + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap(); + assert!(record.admission_start_time.is_none()); + assert!(record.preparation_deadline.is_some()); + assert!(record.cleanup_completed_time.is_none()); + drop(global); + runtime + .reclaim_provisioning_timeout(&expired, &gate) + .await + .unwrap(); + let reclaimed = runtime + .store + .get_message::("sb-prepare") + .await + .unwrap() + .unwrap(); + assert!( + reclaimed + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap() + .cleanup_completed_time + .is_some() + ); + assert_eq!(driver.stop_calls(), 1); + } + #[tokio::test] async fn provisioning_timeout_persists_error_before_cleanup_and_preserves_record() { let driver = ControlledDriver::new(); diff --git a/crates/openshell-server/src/compute/provisioning_deadline.rs b/crates/openshell-server/src/compute/provisioning_deadline.rs index 54d1f83682..dfbb2fce64 100644 --- a/crates/openshell-server/src/compute/provisioning_deadline.rs +++ b/crates/openshell-server/src/compute/provisioning_deadline.rs @@ -1,11 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Gateway-owned provisioning repair-window transitions. +//! Gateway-owned image preparation and admission repair deadlines. //! //! Callers must persist each transition under the sandbox lifecycle fence. Times //! are gateway-assigned Unix milliseconds, never supervisor-supplied values. -//! The timer remains armed after admission acceptance until compute becomes Ready. +//! Preparation has an absolute ceiling. The first authenticated supervisor +//! configuration report starts admission repair, which remains armed until Ready. use openshell_core::proto::SandboxProvisioning; use openshell_core::time::{timestamp_from_millis, timestamp_to_millis}; @@ -26,6 +27,8 @@ pub(super) struct ProvisioningDeadline { attempt_id: String, change: ConfigurationChange, first_rejection_at_ms: Option, + preparation_deadline_at_ms: Option, + admission_start_at_ms: Option, state: DeadlineState, } @@ -50,6 +53,21 @@ impl ProvisioningDeadline { if record.deadline.is_some() && record.timeout_time.is_some() { return Err("provisioning cannot be armed and expired".into()); } + let preparation_deadline_at_ms = record + .preparation_deadline + .as_ref() + .map(|value| millis(Some(value), "preparation deadline")) + .transpose()?; + let admission_start_at_ms = record + .admission_start_time + .as_ref() + .map(|value| millis(Some(value), "admission start time")) + .transpose()?; + if admission_start_at_ms.is_some_and(|started| { + preparation_deadline_at_ms.is_none_or(|ceiling| started >= ceiling) + }) { + return Err("admission must start before its preparation deadline".into()); + } let state = if record.timeout_time.is_some() { DeadlineState::Expired { expired_at_ms: millis(record.timeout_time.as_ref(), "timeout time")?, @@ -61,6 +79,13 @@ impl ProvisioningDeadline { } else { DeadlineState::Ready }; + if let Some(ceiling) = preparation_deadline_at_ms + && admission_start_at_ms.is_none() + && !matches!(state, DeadlineState::Expired { .. }) + && !matches!(state, DeadlineState::Armed { deadline_at_ms } if deadline_at_ms == ceiling) + { + return Err("preparation must retain its absolute deadline until admission".into()); + } Ok(Self { attempt_id: record.attempt_id.clone(), change: ConfigurationChange { @@ -72,6 +97,8 @@ impl ProvisioningDeadline { .as_ref() .map(|value| millis(Some(value), "rejection time")) .transpose()?, + preparation_deadline_at_ms, + admission_start_at_ms, state, }) } @@ -84,6 +111,12 @@ impl ProvisioningDeadline { record.first_rejection_time = self .first_rejection_at_ms .and_then(|value| timestamp_from_millis(value).ok()); + record.preparation_deadline = self + .preparation_deadline_at_ms + .and_then(|value| timestamp_from_millis(value).ok()); + record.admission_start_time = self + .admission_start_at_ms + .and_then(|value| timestamp_from_millis(value).ok()); record.deadline = self .deadline_at_ms() .and_then(|value| timestamp_from_millis(value).ok()); @@ -98,12 +131,38 @@ impl ProvisioningDeadline { attempt_id, change, first_rejection_at_ms: None, + preparation_deadline_at_ms: None, + admission_start_at_ms: None, state: DeadlineState::Armed { deadline_at_ms: now_ms.saturating_add(REPAIR_WINDOW_MS), }, } } + fn is_preparing(&self) -> bool { + self.preparation_deadline_at_ms.is_some() && self.admission_start_at_ms.is_none() + } + + /// Only an authenticated report for the current supervisor may call this. + /// Persist it with the report: duplicate delivery or restart must not grant + /// another admission window, and a late registration cannot revive compute. + fn start_admission(&mut self, attempt_id: &str, now_ms: i64) -> bool { + if attempt_id != self.attempt_id + || !self.is_preparing() + || now_ms < self.change.committed_at_ms + || self + .deadline_at_ms() + .is_none_or(|deadline| now_ms >= deadline) + { + return false; + } + self.admission_start_at_ms = Some(now_ms); + self.state = DeadlineState::Armed { + deadline_at_ms: now_ms.saturating_add(REPAIR_WINDOW_MS), + }; + true + } + pub fn deadline_at_ms(&self) -> Option { match self.state { DeadlineState::Armed { deadline_at_ms } => Some(deadline_at_ms), @@ -124,10 +183,12 @@ impl ProvisioningDeadline { { return false; } - self.state = DeadlineState::Armed { - deadline_at_ms: deadline_at_ms - .max(change.committed_at_ms.saturating_add(REPAIR_WINDOW_MS)), - }; + if !self.is_preparing() { + self.state = DeadlineState::Armed { + deadline_at_ms: deadline_at_ms + .max(change.committed_at_ms.saturating_add(REPAIR_WINDOW_MS)), + }; + } self.change = change; self.first_rejection_at_ms = None; true @@ -140,6 +201,7 @@ impl ProvisioningDeadline { return false; }; if !self.matches(attempt_id, change_id) + || self.is_preparing() || self.first_rejection_at_ms.is_some() || now_ms < self.change.committed_at_ms || now_ms >= deadline_at_ms @@ -173,6 +235,7 @@ impl ProvisioningDeadline { /// must not call this method. Late readiness requires an explicit retry. pub fn ready(&mut self, attempt_id: &str, change_id: &str, now_ms: i64) -> bool { if !self.matches(attempt_id, change_id) + || self.is_preparing() || self .deadline_at_ms() .is_none_or(|deadline| now_ms >= deadline) @@ -197,7 +260,8 @@ pub fn timed_out(sandbox: &openshell_core::proto::Sandbox) -> bool { .is_some_and(|record| record.timeout_time.is_some()) } -/// Create an independent attempt. Supervisor reconnects must never call this. +/// Adopt an existing untimed attempt without granting it a new preparation phase. +/// New create/start operations use `new_preparation_record` instead. pub fn new_record(now_ms: i64) -> SandboxProvisioning { let mut record = SandboxProvisioning::default(); ProvisioningDeadline::new( @@ -212,6 +276,26 @@ pub fn new_record(now_ms: i64) -> SandboxProvisioning { record } +/// Create an independent attempt with a fixed preparation budget. The gateway +/// validates the configured seconds before constructing the runtime. Reconnects +/// and driver progress must never call this function. +pub fn new_preparation_record(now_ms: i64, timeout_seconds: u32) -> SandboxProvisioning { + let mut record = new_record(now_ms); + let ceiling = now_ms.saturating_add(i64::from(timeout_seconds) * 1_000); + record.preparation_deadline = timestamp_from_millis(ceiling).ok(); + record.deadline.clone_from(&record.preparation_deadline); + record +} + +/// The caller has checked the report's authentication, instance fence and +/// configuration generation. Persist this transition in the same CAS as admission. +pub fn record_admission_start(record: &mut SandboxProvisioning, now_ms: i64) -> Result<(), String> { + let mut deadline = ProvisioningDeadline::from_record(record)?; + deadline.start_admission(&record.attempt_id, now_ms); + deadline.write_record(record); + Ok(()) +} + /// Apply the first accepted rejection under the same CAS as admission evidence. /// The report's generation and supervisor instance must already be validated. pub fn record_rejection(record: &mut SandboxProvisioning, now_ms: i64) -> Result<(), String> { @@ -250,6 +334,10 @@ pub(super) fn reconcile_readiness(sandbox: &mut openshell_core::proto::Sandbox, if deadline.ready(&record.attempt_id, &record.configuration_change_id, now_ms) { deadline.write_record(record); } else { + let awaiting_registration = deadline.is_preparing() + && deadline + .deadline_at_ms() + .is_some_and(|value| now_ms < value); status.phase = SandboxPhase::Provisioning.into(); status .conditions @@ -257,8 +345,18 @@ pub(super) fn reconcile_readiness(sandbox: &mut openshell_core::proto::Sandbox, status.conditions.push(SandboxCondition { r#type: "Ready".into(), status: "False".into(), - reason: "ProvisioningDeadlineElapsed".into(), - message: "Provisioning deadline elapsed; awaiting compute reclamation".into(), + reason: if awaiting_registration { + "ConfigurationPending" + } else { + "ProvisioningDeadlineElapsed" + } + .into(), + message: if awaiting_registration { + "Waiting for an authenticated supervisor configuration report" + } else { + "Provisioning deadline elapsed; awaiting compute reclamation" + } + .into(), ..Default::default() }); } @@ -442,6 +540,7 @@ impl super::ComputeRuntime { return Ok(None); }; let mut deadline = ProvisioningDeadline::from_record(record)?; + let preparation_expired = deadline.is_preparing(); if !deadline.expire(&record.attempt_id, &record.configuration_change_id, now_ms) { return Ok(None); } @@ -463,7 +562,9 @@ impl super::ComputeRuntime { .configuration_admission .as_ref() .map_or("", |admission| admission.error.as_str()); - let message = if diagnostic.is_empty() { + let message = if preparation_expired { + "Image preparation or initial supervisor startup exceeded its absolute deadline".to_string() + } else if diagnostic.is_empty() { "Provisioning repair window expired after 300 seconds".to_string() } else { format!( @@ -476,7 +577,11 @@ impl super::ComputeRuntime { status.conditions.push(SandboxCondition { r#type: "Ready".into(), status: "False".into(), - reason: "ProvisioningTimedOut".into(), + reason: if preparation_expired { + "ImagePreparationTimedOut" + } else { + "ProvisioningTimedOut" + }.into(), message, transition_time: timestamp_from_millis(now_ms).ok(), }); @@ -488,7 +593,8 @@ impl super::ComputeRuntime { self.sandbox_watch_bus.notify(current.object_id()); tracing::warn!( sandbox_id = current.object_id(), - "Sandbox provisioning repair window expired" + preparation_expired, + "Sandbox provisioning deadline expired" ); Ok(Some(updated)) } @@ -654,6 +760,87 @@ impl super::ComputeRuntime { mod tests { use super::*; + #[test] + fn existing_wire_record_does_not_gain_preparation_time() { + use prost::Message; + // Encoded before preparation timestamps existed: attempt a, change c, + // configuration at epoch 0, and an admission deadline at 300 seconds. + let bytes = [0x0a, 1, b'a', 0x12, 1, b'c', 0x1a, 0, 0x2a, 3, 8, 0xac, 2]; + let mut record = SandboxProvisioning::decode(bytes.as_slice()).unwrap(); + let before = record.clone(); + record_admission_start(&mut record, 299_999).unwrap(); + assert_eq!(record, before); + assert!(record.preparation_deadline.is_none()); + assert!(!allows_admission(&record, 300_000)); + } + + #[test] + fn preparation_over_five_minutes_gets_a_full_admission_repair_window() { + let record = new_preparation_record(0, 1800); + let mut timer = ProvisioningDeadline::from_record(&record).unwrap(); + let attempt = record.attempt_id; + let change_id = record.configuration_change_id; + assert!(!timer.expire(&attempt, &change_id, 600_000)); + assert!(!timer.ready(&attempt, &change_id, 600_000)); + assert!(timer.start_admission(&attempt, 600_000)); + assert_eq!(timer.deadline_at_ms(), Some(900_000)); + assert!(timer.rejected(&attempt, &change_id, 601_000)); + assert_eq!(timer.deadline_at_ms(), Some(901_000)); + assert!(!timer.start_admission(&attempt, 800_000)); + assert!(timer.configuration_changed(&attempt, change("updated", 800_000))); + assert_eq!(timer.deadline_at_ms(), Some(1_100_000)); + assert_eq!(timer.admission_start_at_ms, Some(600_000)); + assert_eq!(timer.preparation_deadline_at_ms, Some(1_800_000)); + } + + #[test] + fn preparation_config_changes_and_restart_preserve_the_absolute_ceiling() { + use prost::Message; + let mut record = new_preparation_record(0, 1800); + let mut timer = ProvisioningDeadline::from_record(&record).unwrap(); + let attempt = record.attempt_id.clone(); + assert!(timer.configuration_changed(&attempt, change("first", 600_000))); + assert!(timer.configuration_changed(&attempt, change("last", 1_799_000))); + assert!(!timer.configuration_changed(&attempt, change("last", 1_799_500))); + assert!(!timer.rejected(&attempt, "last", 1_799_500)); + timer.write_record(&mut record); + let bytes = record.encode_to_vec(); + let restored = SandboxProvisioning::decode(bytes.as_slice()).unwrap(); + let mut timer = ProvisioningDeadline::from_record(&restored).unwrap(); + assert_eq!(timer.deadline_at_ms(), Some(1_800_000)); + assert!(!timer.start_admission("previous-attempt", 1_799_999)); + assert!(!timer.start_admission(&attempt, 1_800_000)); + assert!(timer.expire(&attempt, "last", 1_800_000)); + assert!(!timer.start_admission(&attempt, 1_799_999)); + assert!(!timer.ready(&attempt, "last", 1_799_999)); + assert!(!timer.configuration_changed(&attempt, change("late", 1_799_999))); + } + + #[test] + fn admission_registration_roundtrip_does_not_restart_repair() { + use prost::Message; + let mut record = new_preparation_record(0, 1800); + record_admission_start(&mut record, 600_000).unwrap(); + let before = record.clone(); + let bytes = record.encode_to_vec(); + let mut restored = SandboxProvisioning::decode(bytes.as_slice()).unwrap(); + record_admission_start(&mut restored, 899_999).unwrap(); + assert_eq!(restored, before); + assert!(!allows_admission(&restored, 900_000)); + } + + #[test] + fn malformed_preparation_timing_cannot_grant_admission() { + let mut record = new_preparation_record(0, 1800); + record.deadline = timestamp_from_millis(1_800_001).ok(); + assert!(ProvisioningDeadline::from_record(&record).is_err()); + record.deadline = None; + assert!(ProvisioningDeadline::from_record(&record).is_err()); + record.deadline = record.preparation_deadline; + record.admission_start_time = timestamp_from_millis(1_800_000).ok(); + assert!(ProvisioningDeadline::from_record(&record).is_err()); + } + #[test] fn protobuf_roundtrip_retains_deadline_and_cleanup_progress() { use prost::Message; diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index e442607d9d..efa0182967 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -116,6 +116,9 @@ pub struct GatewayFileSection { // ── Sandbox / SSH ──────────────────────────────────────────────────── #[serde(default)] pub ssh_session_ttl_secs: Option, + /// Absolute preparation budget for new attempts; existing deadlines persist. + #[serde(default)] + pub image_preparation_timeout_seconds: Option, #[serde(default)] pub grpc_rate_limit_requests: Option, #[serde(default)] diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index ec65533bc5..bf115370b4 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -4600,7 +4600,7 @@ pub(super) async fn handle_report_sandbox_configuration( .map_err(Status::internal)?; if crate::compute::provisioning_deadline::timed_out(&sandbox) { return Err(Status::failed_precondition( - "provisioning repair window expired; explicitly start the sandbox after cleanup", + "provisioning deadline expired; explicitly start the sandbox after cleanup", )); } let current = sandbox @@ -4686,10 +4686,10 @@ pub(super) async fn handle_report_sandbox_configuration( .claim_provisioning_timeout(&sandbox, now_ms) .await .map_err(Status::internal)?; - return Err(Status::failed_precondition( - "provisioning repair window expired", - )); + return Err(Status::failed_precondition("provisioning deadline expired")); } + crate::compute::provisioning_deadline::record_admission_start(record, now_ms) + .map_err(Status::internal)?; if reported == ConfigurationAdmissionState::Rejected { crate::compute::provisioning_deadline::record_rejection(record, now_ms) .map_err(Status::internal)?; @@ -7906,6 +7906,143 @@ mod tests { request } + #[tokio::test] + async fn preparation_registration_starts_repair_once_after_a_cold_start() { + use crate::compute::provisioning_deadline::new_preparation_record; + use openshell_core::proto::{ + ConfigurationAdmissionState, ReportSandboxConfigurationRequest, + SandboxConfigurationAdmission, SandboxPhase, + }; + use openshell_core::time::timestamp_to_millis; + let state = test_server_state().await; + let sandbox_id = "sb-cold-registration"; + let mut sandbox = test_sandbox( + sandbox_id, + "cold-registration", + openshell_policy::restrictive_default_policy(), + Vec::new(), + ); + sandbox.set_phase(SandboxPhase::Provisioning.into()); + let preparation = new_preparation_record(current_time_ms() - 600_000, 1800); + sandbox.status.as_mut().unwrap().provisioning = Some(preparation.clone()); + state.store.put_message(&sandbox).await.unwrap(); + let instance_id = uuid::Uuid::new_v4().to_string(); + let request = || { + with_sandbox( + Request::new(ReportSandboxConfigurationRequest { + sandbox_id: sandbox_id.into(), + admission: Some(SandboxConfigurationAdmission { + instance_id: instance_id.clone(), + state: ConfigurationAdmissionState::Pending.into(), + ..Default::default() + }), + ..Default::default() + }), + sandbox_id, + ) + }; + // A duplicate report racing the initial registration must share one + // persisted transition; either may acquire the lifecycle fence first. + let (first, duplicate) = tokio::join!( + handle_report_sandbox_configuration(&state, request()), + handle_report_sandbox_configuration(&state, request()), + ); + first.unwrap(); + duplicate.unwrap(); + let saved = state + .store + .get_message::(sandbox_id) + .await + .unwrap() + .unwrap(); + let record = saved + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap(); + assert_eq!( + record.preparation_deadline, + preparation.preparation_deadline + ); + let start = timestamp_to_millis(record.admission_start_time.as_ref().unwrap()).unwrap(); + let deadline = timestamp_to_millis(record.deadline.as_ref().unwrap()).unwrap(); + assert_eq!(deadline - start, 300_000); + handle_report_sandbox_configuration(&state, request()) + .await + .unwrap(); + let repeated = state + .store + .get_message::(sandbox_id) + .await + .unwrap() + .unwrap(); + assert_eq!( + repeated + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap(), + record + ); + } + + #[tokio::test] + async fn preparation_expiry_rejects_registration_before_the_scanner_runs() { + use crate::compute::provisioning_deadline::new_preparation_record; + use openshell_core::proto::{ + ConfigurationAdmissionState, ReportSandboxConfigurationRequest, + SandboxConfigurationAdmission, SandboxPhase, + }; + let state = test_server_state().await; + let sandbox_id = "sb-expired-preparation"; + let mut sandbox = test_sandbox( + sandbox_id, + "expired-preparation", + openshell_policy::restrictive_default_policy(), + Vec::new(), + ); + sandbox.set_phase(SandboxPhase::Provisioning.into()); + sandbox.status.as_mut().unwrap().provisioning = Some(new_preparation_record(0, 1800)); + state.store.put_message(&sandbox).await.unwrap(); + let error = handle_report_sandbox_configuration( + &state, + with_sandbox( + Request::new(ReportSandboxConfigurationRequest { + sandbox_id: sandbox_id.into(), + admission: Some(SandboxConfigurationAdmission { + instance_id: uuid::Uuid::new_v4().to_string(), + state: ConfigurationAdmissionState::Pending.into(), + ..Default::default() + }), + ..Default::default() + }), + sandbox_id, + ), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + let expired = state + .store + .get_message::(sandbox_id) + .await + .unwrap() + .unwrap(); + assert_eq!(expired.phase(), i32::from(SandboxPhase::Error)); + let status = expired.status.unwrap(); + assert!(status.provisioning.unwrap().admission_start_time.is_none()); + assert!( + status + .conditions + .iter() + .any(|condition| condition.reason == "ImagePreparationTimedOut") + ); + } + #[tokio::test] async fn provisioning_timeout_rejects_supervisor_registration() { use openshell_core::proto::{ @@ -7944,7 +8081,7 @@ mod tests { .await .unwrap_err(); assert_eq!(error.code(), Code::FailedPrecondition); - assert!(error.message().contains("repair window expired")); + assert!(error.message().contains("provisioning deadline expired")); } #[tokio::test] diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 594593a698..5dbae2cc7b 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -605,7 +605,12 @@ async fn handle_create_sandbox_inner( .status .as_mut() .expect("status initialized") - .provisioning = Some(crate::compute::provisioning_deadline::new_record(now_ms)); + .provisioning = Some( + crate::compute::provisioning_deadline::new_preparation_record( + now_ms, + state.config.image_preparation_timeout_seconds, + ), + ); crate::compute::provisioning_deadline::refresh_configuration( &state.store, &mut sandbox, diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 72efa01fbf..b2b839d969 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -1711,6 +1711,9 @@ async fn build_compute_runtime( let runtime = runtime .with_admission_policy(admission) + .and_then(|runtime| { + runtime.with_image_preparation_timeout(config.image_preparation_timeout_seconds) + }) .map_err(Error::config)?; Ok(runtime.with_telemetry_compute_driver(telemetry_compute_driver)) } diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index e6d8730aa0..f92d6b1e5d 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -126,12 +126,15 @@ mod tests { // inventories; the provider-environment file map is public-only. The // request has no provider-file capability field: older supervisors ignore // the additive file map while retaining the rest of the response. + // Preparation timing adds two optional timestamps to SandboxProvisioning. + // Existing rows decode with both absent and retain their active deadline; + // no stored attempt gains another phase or time budget on upgrade. // Service authorization also extends both schemas additively. Legacy // payloads retain the safe Strip default. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "2e156c6ad3c8eb51bcd30dc13b173fe339b38207a1b1f98f7be2e0cad8e3bd45"; + "581125d215f2a1967eb73826c411c1e72a53fb3a30a20e85c51d96bbb518e078"; const DURABLE_SCHEMA_SHA256: &str = - "38165d9d76f49fcfe98a12f241e032838a2376c1d1a87ea2796fd33b9b1a3541"; + "c1e49c80c52a5c7458952b333e7ca9da2dd24478b41756b710ba29a5217b67d7"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = "761dea31a521b0650840fe2a823ad6e36a265ed323ba4506889781d630df0ee3"; // A persisted Sandbox without endpoint status retains its lifecycle fields; diff --git a/crates/openshell-tui/src/lib.rs b/crates/openshell-tui/src/lib.rs index f235c4e003..ab856d093f 100644 --- a/crates/openshell-tui/src/lib.rs +++ b/crates/openshell-tui/src/lib.rs @@ -2720,7 +2720,12 @@ fn sandbox_notes_for_view( } else { "compute cleanup pending" }; - let mut notes = format!("Provisioning timed out; {cleanup}"); + let mut notes = + if record.preparation_deadline.is_some() && record.admission_start_time.is_none() { + format!("Image preparation timed out; {cleanup}") + } else { + format!("Provisioning timed out; {cleanup}") + }; if !forwards.is_empty() { notes.push_str("; "); notes.push_str(&forwards); @@ -3340,6 +3345,26 @@ mod sandbox_notes_tests { ); } + #[test] + fn preparation_timeout_notes_identify_the_expired_phase() { + let sandbox = Sandbox { + status: Some(SandboxStatus { + provisioning: Some(openshell_core::proto::SandboxProvisioning { + preparation_deadline: openshell_core::time::timestamp_from_millis(1_800_000) + .ok(), + timeout_time: openshell_core::time::timestamp_from_millis(1_800_000).ok(), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + assert_eq!( + sandbox_notes(&sandbox, String::new()), + "Image preparation timed out; compute cleanup pending" + ); + } + #[test] fn configuration_rejection_precedes_forwards_and_clears_after_repair() { let condition = SandboxCondition { diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 4cfd902daa..3a11406440 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -139,6 +139,10 @@ credential_drivers = ["kubernetes-secrets"] ssh_session_ttl_secs = 3600 +# Absolute image preparation and initial supervisor startup budget. +# Applies to new attempts only; allowed values are 1 through 86400 seconds. +image_preparation_timeout_seconds = 1800 + # Reject invalid policy generations securely by default. Set # "retain_last_valid" only when availability takes priority. policy_validation_failure_mode = "fail_closed" diff --git a/docs/how-it-works/policies/manage-policies.mdx b/docs/how-it-works/policies/manage-policies.mdx index f7416a62fc..c20af7b67b 100644 --- a/docs/how-it-works/policies/manage-policies.mdx +++ b/docs/how-it-works/policies/manage-policies.mdx @@ -368,14 +368,11 @@ openshell sandbox get my-sandbox --output json The same repair window applies when the gateway refuses an image-policy upload or a policy update that adds baseline filesystem paths with `FAILED_PRECONDITION` or `INVALID_ARGUMENT`. After the gateway acknowledges the rejection report, startup waits two seconds, reads a fresh configuration, and retries. If your repair reaches the gateway before that report, startup immediately reads the repaired configuration. The sandbox log records the gateway's reason; repeated refusals of the same write, status code, and configuration are logged once. Authentication and authorization failures still stop startup. -You have 300 seconds to fix the configuration. Replace the policy with -`openshell policy set`, or fix the provider configuration. Until the workload -first starts, you can also change filesystem, Landlock, and process settings. -Each change to the policy, providers, or settings restarts the 300 seconds. +You have 300 seconds to fix the configuration after the supervisor starts admission. Image preparation has its own deadline and does not consume that repair time. Replace the policy with `openshell policy set`, or fix the provider configuration. Until the workload first starts, you can also change filesystem, Landlock, and process settings. Each effective change to the policy, providers, or settings restarts the 300 seconds; writing an unchanged value does not. If the time runs out, the sandbox moves to `Error` with the reason `ProvisioningTimedOut`. Fixing the configuration does not restart it. After you -fix it, start the sandbox again, which begins a new 300-second window: +fix it, start the sandbox again. Admission gets a new 300-second window after preparation: ```shell openshell sandbox start my-sandbox diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index e38f925327..d7ea7fecfb 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -939,30 +939,20 @@ Management operations remain available while startup is blocked. After repair, the supervisor completes startup without recreating the sandbox. Starting a stopped sandbox repeats configuration admission before launching its workload. -The gateway enforces a 300-second provisioning repair window, independently of -the CLI wait timeout. An effective policy, settings, provider, profile, or -attachment change resets the window from its stored change time. The first -failed configuration load for that change grants another full window. Repeated -failures and reconnects do not extend it; reaching `Ready` clears it. +Image preparation and initial supervisor startup have an absolute deadline of 1800 seconds by default, independently of the CLI wait timeout. Operators can set `image_preparation_timeout_seconds` in `[openshell.gateway]` to any value from 1 through 86400. Each new create or explicit start stores its deadline. Progress events, configuration edits, and gateway or driver restarts cannot extend it. Changing the gateway setting affects new attempts only. -When the window expires, the sandbox enters `Error` with reason -`ProvisioningTimedOut`. The gateway stops its workload and supervisor compute, -retaining the sandbox record, diagnostic, and restartable storage. Cleanup can -remain pending if the backend is unavailable; the gateway retries it. Inspect -`provisioning` in JSON output for the deadline, timeout, and cleanup timestamps. -TUI NOTES distinguishes pending cleanup from reclaimed compute. +The first authenticated configuration report from the current supervisor ends preparation and starts a separate 300-second admission repair window. A slow image download therefore does not consume time reserved for fixing policy. During admission, an effective policy, settings, provider, profile, or attachment change resets the window from its stored change time. The first failed configuration load for that change grants another full window. Repeated failures, duplicate reports, and reconnects do not extend it; reaching `Ready` clears it. -Repair the configuration, wait for cleanup to complete, then explicitly restart: +Preparation expiry sets the sandbox to `Error` with reason `ImagePreparationTimedOut`; admission repair expiry uses `ProvisioningTimedOut`. The gateway stops its workload and supervisor compute, retaining the sandbox record, diagnostic, and restartable storage. Cleanup can remain pending if the backend is unavailable; the gateway retries it. Inspect `provisioning` in JSON output for the active `phase`, `deadline`, original `preparation_deadline`, `admission_start_time`, and cleanup timestamps. TUI NOTES identifies preparation timeout and distinguishes pending cleanup from reclaimed compute. + +For `ImagePreparationTimedOut`, inspect image preparation and supervisor startup diagnostics and check whether the configured preparation budget is sufficient. For `ProvisioningTimedOut`, repair the rejected configuration. In either case, wait for cleanup to complete, then explicitly restart: ```shell openshell sandbox get my-sandbox --output json openshell sandbox start my-sandbox ``` -Editing configuration after expiry does not restart compute. A retry gets a new -300-second window, while static-policy restrictions from any previous activation -remain in force. Timed-out records are retained even for ephemeral creates; use -`sandbox delete` when you no longer need the diagnostic or stored state. +Editing configuration after expiry does not restart compute. An explicit retry gets a new preparation deadline and a separate admission repair window, while static-policy restrictions from any previous activation remain in force. Attempts already active when the gateway is upgraded retain their stored deadline. Timed-out records are retained even for ephemeral creates; use `sandbox delete` when you no longer need the diagnostic or stored state. | Phase | Description | | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/proto/openshell.proto b/proto/openshell.proto index 83ffbe75ad..356d0baacc 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -1184,7 +1184,7 @@ message SandboxStatus { SandboxConfigurationAdmission configuration_admission = 11; // Durable first-acceptance marker. Absent on legacy records; never reset by restart. optional bool configuration_activated = 12; - // Gateway-owned repair window. Retained after timeout for inspection and retry. + // Gateway-owned provisioning deadlines, retained after timeout for inspection and retry. SandboxProvisioning provisioning = 13; // Consecutive policy-driven restart number in the current crash loop. uint32 restart_count = 14; @@ -3769,7 +3769,7 @@ message SandboxProvisioning { string configuration_change_id = 2; google.protobuf.Timestamp configuration_change_time = 3; google.protobuf.Timestamp first_rejection_time = 4; - // Present only while the repair window is armed. + // Active preparation or admission-repair deadline. Absent after Ready/timeout. google.protobuf.Timestamp deadline = 5; google.protobuf.Timestamp timeout_time = 6; // Set only after both supervisor and workload compute have been reclaimed. @@ -3781,6 +3781,14 @@ message SandboxProvisioning { // Attachment edits have their own durable clock; status writes do not change it. string attachment_change_id = 10; google.protobuf.Timestamp attachment_change_time = 11; + // Absolute ceiling for image preparation and initial supervisor startup. + // Set once per new attempt; progress, configuration writes, and restarts + // cannot extend it. Absent on attempts created before preparation timing. + google.protobuf.Timestamp preparation_deadline = 12; + // First authenticated supervisor configuration report for this attempt. + // Starts the admission repair window. Absent while preparing, including + // after a preparation timeout. Duplicate reports never change this value. + google.protobuf.Timestamp admission_start_time = 13; } // Create-time request to expose one loopback HTTP service in a sandbox. diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 109d2ef454..16d62d565b 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -3118,7 +3118,7 @@ type SandboxStatus struct { ConfigurationAdmission *SandboxConfigurationAdmission `protobuf:"bytes,11,opt,name=configuration_admission,json=configurationAdmission,proto3" json:"configuration_admission,omitempty"` // Durable first-acceptance marker. Absent on legacy records; never reset by restart. ConfigurationActivated *bool `protobuf:"varint,12,opt,name=configuration_activated,json=configurationActivated,proto3,oneof" json:"configuration_activated,omitempty"` - // Gateway-owned repair window. Retained after timeout for inspection and retry. + // Gateway-owned provisioning deadlines, retained after timeout for inspection and retry. Provisioning *SandboxProvisioning `protobuf:"bytes,13,opt,name=provisioning,proto3" json:"provisioning,omitempty"` // Consecutive policy-driven restart number in the current crash loop. RestartCount uint32 `protobuf:"varint,14,opt,name=restart_count,json=restartCount,proto3" json:"restart_count,omitempty"` @@ -17637,7 +17637,7 @@ type SandboxProvisioning struct { ConfigurationChangeId string `protobuf:"bytes,2,opt,name=configuration_change_id,json=configurationChangeId,proto3" json:"configuration_change_id,omitempty"` ConfigurationChangeTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=configuration_change_time,json=configurationChangeTime,proto3" json:"configuration_change_time,omitempty"` FirstRejectionTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=first_rejection_time,json=firstRejectionTime,proto3" json:"first_rejection_time,omitempty"` - // Present only while the repair window is armed. + // Active preparation or admission-repair deadline. Absent after Ready/timeout. Deadline *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=deadline,proto3" json:"deadline,omitempty"` TimeoutTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=timeout_time,json=timeoutTime,proto3" json:"timeout_time,omitempty"` // Set only after both supervisor and workload compute have been reclaimed. @@ -17649,8 +17649,16 @@ type SandboxProvisioning struct { // Attachment edits have their own durable clock; status writes do not change it. AttachmentChangeId string `protobuf:"bytes,10,opt,name=attachment_change_id,json=attachmentChangeId,proto3" json:"attachment_change_id,omitempty"` AttachmentChangeTime *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=attachment_change_time,json=attachmentChangeTime,proto3" json:"attachment_change_time,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Absolute ceiling for image preparation and initial supervisor startup. + // Set once per new attempt; progress, configuration writes, and restarts + // cannot extend it. Absent on attempts created before preparation timing. + PreparationDeadline *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=preparation_deadline,json=preparationDeadline,proto3" json:"preparation_deadline,omitempty"` + // First authenticated supervisor configuration report for this attempt. + // Starts the admission repair window. Absent while preparing, including + // after a preparation timeout. Duplicate reports never change this value. + AdmissionStartTime *timestamppb.Timestamp `protobuf:"bytes,13,opt,name=admission_start_time,json=admissionStartTime,proto3" json:"admission_start_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SandboxProvisioning) Reset() { @@ -17760,6 +17768,20 @@ func (x *SandboxProvisioning) GetAttachmentChangeTime() *timestamppb.Timestamp { return nil } +func (x *SandboxProvisioning) GetPreparationDeadline() *timestamppb.Timestamp { + if x != nil { + return x.PreparationDeadline + } + return nil +} + +func (x *SandboxProvisioning) GetAdmissionStartTime() *timestamppb.Timestamp { + if x != nil { + return x.AdmissionStartTime + } + return nil +} + // Create-time request to expose one loopback HTTP service in a sandbox. type SandboxServiceExposure struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -19141,7 +19163,7 @@ const file_openshell_proto_rawDesc = "" + "\x04path\x18\x04 \x01(\tR\x04path\x12=\n" + "\vlast_result\x18\x05 \x01(\x0e2\x1c.openshell.v1.EndpointResultR\n" + "lastResult\x12H\n" + - "\x12last_reported_time\x18j \x01(\v2\x1a.google.protobuf.TimestampR\x10lastReportedTimeJ\x04\b\x06\x10\aR\x10last_reported_at\"\xce\x05\n" + + "\x12last_reported_time\x18j \x01(\v2\x1a.google.protobuf.TimestampR\x10lastReportedTimeJ\x04\b\x06\x10\aR\x10last_reported_at\"\xeb\x06\n" + "\x13SandboxProvisioning\x12\x1d\n" + "\n" + "attempt_id\x18\x01 \x01(\tR\tattemptId\x126\n" + @@ -19155,7 +19177,9 @@ const file_openshell_proto_rawDesc = "" + "\x12cleanup_retry_time\x18\t \x01(\v2\x1a.google.protobuf.TimestampR\x10cleanupRetryTime\x120\n" + "\x14attachment_change_id\x18\n" + " \x01(\tR\x12attachmentChangeId\x12P\n" + - "\x16attachment_change_time\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\x14attachmentChangeTime\"\xaa\x01\n" + + "\x16attachment_change_time\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\x14attachmentChangeTime\x12M\n" + + "\x14preparation_deadline\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\x13preparationDeadline\x12L\n" + + "\x14admission_start_time\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\x12admissionStartTime\"\xaa\x01\n" + "\x16SandboxServiceExposure\x12\x18\n" + "\aservice\x18\x01 \x01(\tR\aservice\x12\x1f\n" + "\vtarget_port\x18\x02 \x01(\rR\n" + @@ -20108,180 +20132,182 @@ var file_openshell_proto_depIdxs = []int32{ 275, // 316: openshell.v1.SandboxProvisioning.cleanup_completed_time:type_name -> google.protobuf.Timestamp 275, // 317: openshell.v1.SandboxProvisioning.cleanup_retry_time:type_name -> google.protobuf.Timestamp 275, // 318: openshell.v1.SandboxProvisioning.attachment_change_time:type_name -> google.protobuf.Timestamp - 19, // 319: openshell.v1.SandboxServiceExposure.authorization_mode:type_name -> openshell.v1.ServiceAuthorizationMode - 275, // 320: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp - 275, // 321: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp - 127, // 322: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry.value:type_name -> openshell.v1.ProviderProfileCredential - 156, // 323: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry.value:type_name -> openshell.v1.StaticCredentialBinding - 24, // 324: openshell.v1.OpenShell.Health:input_type -> openshell.v1.HealthRequest - 26, // 325: openshell.v1.OpenShell.GetCurrentUser:input_type -> openshell.v1.GetCurrentUserRequest - 28, // 326: openshell.v1.OpenShell.GetGatewayInfo:input_type -> openshell.v1.GetGatewayInfoRequest - 52, // 327: openshell.v1.OpenShell.CreateSandbox:input_type -> openshell.v1.CreateSandboxRequest - 60, // 328: openshell.v1.OpenShell.BeginRootfsTarStaging:input_type -> openshell.v1.BeginRootfsTarStagingRequest - 62, // 329: openshell.v1.OpenShell.GetSandbox:input_type -> openshell.v1.GetSandboxRequest - 63, // 330: openshell.v1.OpenShell.ListSandboxes:input_type -> openshell.v1.ListSandboxesRequest - 53, // 331: openshell.v1.OpenShell.CreateSandboxTemplate:input_type -> openshell.v1.CreateSandboxTemplateRequest - 54, // 332: openshell.v1.OpenShell.GetSandboxTemplate:input_type -> openshell.v1.GetSandboxTemplateRequest - 55, // 333: openshell.v1.OpenShell.ListSandboxTemplates:input_type -> openshell.v1.ListSandboxTemplatesRequest - 56, // 334: openshell.v1.OpenShell.DeleteSandboxTemplate:input_type -> openshell.v1.DeleteSandboxTemplateRequest - 64, // 335: openshell.v1.OpenShell.ListSandboxProviders:input_type -> openshell.v1.ListSandboxProvidersRequest - 65, // 336: openshell.v1.OpenShell.AttachSandboxProvider:input_type -> openshell.v1.AttachSandboxProviderRequest - 66, // 337: openshell.v1.OpenShell.DetachSandboxProvider:input_type -> openshell.v1.DetachSandboxProviderRequest - 82, // 338: openshell.v1.OpenShell.GetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest - 67, // 339: openshell.v1.OpenShell.DeleteSandbox:input_type -> openshell.v1.DeleteSandboxRequest - 68, // 340: openshell.v1.OpenShell.StopSandbox:input_type -> openshell.v1.StopSandboxRequest - 69, // 341: openshell.v1.OpenShell.StartSandbox:input_type -> openshell.v1.StartSandboxRequest - 87, // 342: openshell.v1.OpenShell.CreateSshSession:input_type -> openshell.v1.CreateSshSessionRequest - 89, // 343: openshell.v1.OpenShell.ExposeService:input_type -> openshell.v1.ExposeServiceRequest - 90, // 344: openshell.v1.OpenShell.GetService:input_type -> openshell.v1.GetServiceRequest - 91, // 345: openshell.v1.OpenShell.ListServices:input_type -> openshell.v1.ListServicesRequest - 93, // 346: openshell.v1.OpenShell.DeleteService:input_type -> openshell.v1.DeleteServiceRequest - 97, // 347: openshell.v1.OpenShell.RevokeSshSession:input_type -> openshell.v1.RevokeSshSessionRequest - 99, // 348: openshell.v1.OpenShell.ExecSandbox:input_type -> openshell.v1.ExecSandboxRequest - 105, // 349: openshell.v1.OpenShell.ForwardTcp:input_type -> openshell.v1.TcpForwardFrame - 106, // 350: openshell.v1.OpenShell.ExecSandboxInteractive:input_type -> openshell.v1.ExecSandboxInput - 113, // 351: openshell.v1.OpenShell.CreateProvider:input_type -> openshell.v1.CreateProviderRequest - 114, // 352: openshell.v1.OpenShell.GetProvider:input_type -> openshell.v1.GetProviderRequest - 115, // 353: openshell.v1.OpenShell.ListProviders:input_type -> openshell.v1.ListProvidersRequest - 120, // 354: openshell.v1.OpenShell.ListProviderProfiles:input_type -> openshell.v1.ListProviderProfilesRequest - 121, // 355: openshell.v1.OpenShell.GetProviderProfile:input_type -> openshell.v1.GetProviderProfileRequest - 145, // 356: openshell.v1.OpenShell.ImportProviderProfiles:input_type -> openshell.v1.ImportProviderProfilesRequest - 147, // 357: openshell.v1.OpenShell.UpdateProviderProfiles:input_type -> openshell.v1.UpdateProviderProfilesRequest - 149, // 358: openshell.v1.OpenShell.LintProviderProfiles:input_type -> openshell.v1.LintProviderProfilesRequest - 116, // 359: openshell.v1.OpenShell.UpdateProvider:input_type -> openshell.v1.UpdateProviderRequest - 133, // 360: openshell.v1.OpenShell.GetProviderRefreshStatus:input_type -> openshell.v1.GetProviderRefreshStatusRequest - 135, // 361: openshell.v1.OpenShell.ConfigureProviderRefresh:input_type -> openshell.v1.ConfigureProviderRefreshRequest - 137, // 362: openshell.v1.OpenShell.RotateProviderCredential:input_type -> openshell.v1.RotateProviderCredentialRequest - 139, // 363: openshell.v1.OpenShell.DeleteProviderRefresh:input_type -> openshell.v1.DeleteProviderRefreshRequest - 117, // 364: openshell.v1.OpenShell.DeleteProvider:input_type -> openshell.v1.DeleteProviderRequest - 152, // 365: openshell.v1.OpenShell.DeleteProviderProfile:input_type -> openshell.v1.DeleteProviderProfileRequest - 290, // 366: openshell.v1.OpenShell.GetSandboxConfig:input_type -> openshell.sandbox.v1.GetSandboxConfigRequest - 291, // 367: openshell.v1.OpenShell.GetGatewayConfig:input_type -> openshell.sandbox.v1.GetGatewayConfigRequest - 160, // 368: openshell.v1.OpenShell.UpdateConfig:input_type -> openshell.v1.UpdateConfigRequest - 170, // 369: openshell.v1.OpenShell.GetSandboxPolicyStatus:input_type -> openshell.v1.GetSandboxPolicyStatusRequest - 172, // 370: openshell.v1.OpenShell.ListSandboxPolicies:input_type -> openshell.v1.ListSandboxPoliciesRequest - 174, // 371: openshell.v1.OpenShell.ReportPolicyStatus:input_type -> openshell.v1.ReportPolicyStatusRequest - 247, // 372: openshell.v1.OpenShell.ReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest - 84, // 373: openshell.v1.OpenShell.ReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest - 177, // 374: openshell.v1.OpenShell.ReportSandboxConfiguration:input_type -> openshell.v1.ReportSandboxConfigurationRequest - 154, // 375: openshell.v1.OpenShell.GetSandboxProviderEnvironment:input_type -> openshell.v1.GetSandboxProviderEnvironmentRequest - 158, // 376: openshell.v1.OpenShell.ExchangeProviderSubjectToken:input_type -> openshell.v1.ExchangeProviderSubjectTokenRequest - 180, // 377: openshell.v1.OpenShell.GetSandboxLogs:input_type -> openshell.v1.GetSandboxLogsRequest - 181, // 378: openshell.v1.OpenShell.PushSandboxLogs:input_type -> openshell.v1.PushSandboxLogsRequest - 184, // 379: openshell.v1.OpenShell.ConnectSupervisor:input_type -> openshell.v1.SupervisorMessage - 191, // 380: openshell.v1.OpenShell.ReportMainProcessExit:input_type -> openshell.v1.ReportMainProcessExitRequest - 193, // 381: openshell.v1.OpenShell.FinalizeMainProcessExit:input_type -> openshell.v1.FinalizeMainProcessExitRequest - 199, // 382: openshell.v1.OpenShell.RelayStream:input_type -> openshell.v1.RelayFrame - 201, // 383: openshell.v1.OpenShell.PeerRelay:input_type -> openshell.v1.PeerRelayFrame - 84, // 384: openshell.v1.OpenShell.PeerReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest - 247, // 385: openshell.v1.OpenShell.PeerReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest - 82, // 386: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest - 109, // 387: openshell.v1.OpenShell.WatchSandbox:input_type -> openshell.v1.WatchSandboxRequest - 210, // 388: openshell.v1.OpenShell.SubmitPolicyAnalysis:input_type -> openshell.v1.SubmitPolicyAnalysisRequest - 212, // 389: openshell.v1.OpenShell.GetDraftPolicy:input_type -> openshell.v1.GetDraftPolicyRequest - 214, // 390: openshell.v1.OpenShell.ApproveDraftChunk:input_type -> openshell.v1.ApproveDraftChunkRequest - 216, // 391: openshell.v1.OpenShell.RejectDraftChunk:input_type -> openshell.v1.RejectDraftChunkRequest - 219, // 392: openshell.v1.OpenShell.ApproveAllDraftChunks:input_type -> openshell.v1.ApproveAllDraftChunksRequest - 221, // 393: openshell.v1.OpenShell.EditDraftChunk:input_type -> openshell.v1.EditDraftChunkRequest - 223, // 394: openshell.v1.OpenShell.UndoDraftChunk:input_type -> openshell.v1.UndoDraftChunkRequest - 225, // 395: openshell.v1.OpenShell.ClearDraftChunks:input_type -> openshell.v1.ClearDraftChunksRequest - 227, // 396: openshell.v1.OpenShell.GetDraftHistory:input_type -> openshell.v1.GetDraftHistoryRequest - 20, // 397: openshell.v1.OpenShell.IssueSandboxToken:input_type -> openshell.v1.IssueSandboxTokenRequest - 22, // 398: openshell.v1.OpenShell.RefreshSandboxToken:input_type -> openshell.v1.RefreshSandboxTokenRequest - 230, // 399: openshell.v1.OpenShell.CreateWorkspace:input_type -> openshell.v1.CreateWorkspaceRequest - 232, // 400: openshell.v1.OpenShell.GetWorkspace:input_type -> openshell.v1.GetWorkspaceRequest - 234, // 401: openshell.v1.OpenShell.ListWorkspaces:input_type -> openshell.v1.ListWorkspacesRequest - 236, // 402: openshell.v1.OpenShell.DeleteWorkspace:input_type -> openshell.v1.DeleteWorkspaceRequest - 239, // 403: openshell.v1.OpenShell.AddWorkspaceMember:input_type -> openshell.v1.AddWorkspaceMemberRequest - 241, // 404: openshell.v1.OpenShell.RemoveWorkspaceMember:input_type -> openshell.v1.RemoveWorkspaceMemberRequest - 243, // 405: openshell.v1.OpenShell.ListWorkspaceMembers:input_type -> openshell.v1.ListWorkspaceMembersRequest - 25, // 406: openshell.v1.OpenShell.Health:output_type -> openshell.v1.HealthResponse - 27, // 407: openshell.v1.OpenShell.GetCurrentUser:output_type -> openshell.v1.GetCurrentUserResponse - 29, // 408: openshell.v1.OpenShell.GetGatewayInfo:output_type -> openshell.v1.GetGatewayInfoResponse - 70, // 409: openshell.v1.OpenShell.CreateSandbox:output_type -> openshell.v1.SandboxResponse - 61, // 410: openshell.v1.OpenShell.BeginRootfsTarStaging:output_type -> openshell.v1.BeginRootfsTarStagingResponse - 70, // 411: openshell.v1.OpenShell.GetSandbox:output_type -> openshell.v1.SandboxResponse - 71, // 412: openshell.v1.OpenShell.ListSandboxes:output_type -> openshell.v1.ListSandboxesResponse - 57, // 413: openshell.v1.OpenShell.CreateSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse - 57, // 414: openshell.v1.OpenShell.GetSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse - 58, // 415: openshell.v1.OpenShell.ListSandboxTemplates:output_type -> openshell.v1.ListSandboxTemplatesResponse - 59, // 416: openshell.v1.OpenShell.DeleteSandboxTemplate:output_type -> openshell.v1.DeleteSandboxTemplateResponse - 72, // 417: openshell.v1.OpenShell.ListSandboxProviders:output_type -> openshell.v1.ListSandboxProvidersResponse - 73, // 418: openshell.v1.OpenShell.AttachSandboxProvider:output_type -> openshell.v1.AttachSandboxProviderResponse - 74, // 419: openshell.v1.OpenShell.DetachSandboxProvider:output_type -> openshell.v1.DetachSandboxProviderResponse - 83, // 420: openshell.v1.OpenShell.GetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse - 86, // 421: openshell.v1.OpenShell.DeleteSandbox:output_type -> openshell.v1.DeleteSandboxResponse - 70, // 422: openshell.v1.OpenShell.StopSandbox:output_type -> openshell.v1.SandboxResponse - 70, // 423: openshell.v1.OpenShell.StartSandbox:output_type -> openshell.v1.SandboxResponse - 88, // 424: openshell.v1.OpenShell.CreateSshSession:output_type -> openshell.v1.CreateSshSessionResponse - 96, // 425: openshell.v1.OpenShell.ExposeService:output_type -> openshell.v1.ServiceEndpointResponse - 96, // 426: openshell.v1.OpenShell.GetService:output_type -> openshell.v1.ServiceEndpointResponse - 92, // 427: openshell.v1.OpenShell.ListServices:output_type -> openshell.v1.ListServicesResponse - 94, // 428: openshell.v1.OpenShell.DeleteService:output_type -> openshell.v1.DeleteServiceResponse - 98, // 429: openshell.v1.OpenShell.RevokeSshSession:output_type -> openshell.v1.RevokeSshSessionResponse - 103, // 430: openshell.v1.OpenShell.ExecSandbox:output_type -> openshell.v1.ExecSandboxEvent - 105, // 431: openshell.v1.OpenShell.ForwardTcp:output_type -> openshell.v1.TcpForwardFrame - 103, // 432: openshell.v1.OpenShell.ExecSandboxInteractive:output_type -> openshell.v1.ExecSandboxEvent - 118, // 433: openshell.v1.OpenShell.CreateProvider:output_type -> openshell.v1.ProviderResponse - 118, // 434: openshell.v1.OpenShell.GetProvider:output_type -> openshell.v1.ProviderResponse - 119, // 435: openshell.v1.OpenShell.ListProviders:output_type -> openshell.v1.ListProvidersResponse - 144, // 436: openshell.v1.OpenShell.ListProviderProfiles:output_type -> openshell.v1.ListProviderProfilesResponse - 143, // 437: openshell.v1.OpenShell.GetProviderProfile:output_type -> openshell.v1.ProviderProfileResponse - 146, // 438: openshell.v1.OpenShell.ImportProviderProfiles:output_type -> openshell.v1.ImportProviderProfilesResponse - 148, // 439: openshell.v1.OpenShell.UpdateProviderProfiles:output_type -> openshell.v1.UpdateProviderProfilesResponse - 150, // 440: openshell.v1.OpenShell.LintProviderProfiles:output_type -> openshell.v1.LintProviderProfilesResponse - 118, // 441: openshell.v1.OpenShell.UpdateProvider:output_type -> openshell.v1.ProviderResponse - 134, // 442: openshell.v1.OpenShell.GetProviderRefreshStatus:output_type -> openshell.v1.GetProviderRefreshStatusResponse - 136, // 443: openshell.v1.OpenShell.ConfigureProviderRefresh:output_type -> openshell.v1.ConfigureProviderRefreshResponse - 138, // 444: openshell.v1.OpenShell.RotateProviderCredential:output_type -> openshell.v1.RotateProviderCredentialResponse - 140, // 445: openshell.v1.OpenShell.DeleteProviderRefresh:output_type -> openshell.v1.DeleteProviderRefreshResponse - 151, // 446: openshell.v1.OpenShell.DeleteProvider:output_type -> openshell.v1.DeleteProviderResponse - 153, // 447: openshell.v1.OpenShell.DeleteProviderProfile:output_type -> openshell.v1.DeleteProviderProfileResponse - 292, // 448: openshell.v1.OpenShell.GetSandboxConfig:output_type -> openshell.sandbox.v1.GetSandboxConfigResponse - 293, // 449: openshell.v1.OpenShell.GetGatewayConfig:output_type -> openshell.sandbox.v1.GetGatewayConfigResponse - 169, // 450: openshell.v1.OpenShell.UpdateConfig:output_type -> openshell.v1.UpdateConfigResponse - 171, // 451: openshell.v1.OpenShell.GetSandboxPolicyStatus:output_type -> openshell.v1.GetSandboxPolicyStatusResponse - 173, // 452: openshell.v1.OpenShell.ListSandboxPolicies:output_type -> openshell.v1.ListSandboxPoliciesResponse - 175, // 453: openshell.v1.OpenShell.ReportPolicyStatus:output_type -> openshell.v1.ReportPolicyStatusResponse - 248, // 454: openshell.v1.OpenShell.ReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse - 85, // 455: openshell.v1.OpenShell.ReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse - 178, // 456: openshell.v1.OpenShell.ReportSandboxConfiguration:output_type -> openshell.v1.ReportSandboxConfigurationResponse - 157, // 457: openshell.v1.OpenShell.GetSandboxProviderEnvironment:output_type -> openshell.v1.GetSandboxProviderEnvironmentResponse - 159, // 458: openshell.v1.OpenShell.ExchangeProviderSubjectToken:output_type -> openshell.v1.ExchangeProviderSubjectTokenResponse - 183, // 459: openshell.v1.OpenShell.GetSandboxLogs:output_type -> openshell.v1.GetSandboxLogsResponse - 182, // 460: openshell.v1.OpenShell.PushSandboxLogs:output_type -> openshell.v1.PushSandboxLogsResponse - 185, // 461: openshell.v1.OpenShell.ConnectSupervisor:output_type -> openshell.v1.GatewayMessage - 192, // 462: openshell.v1.OpenShell.ReportMainProcessExit:output_type -> openshell.v1.ReportMainProcessExitResponse - 194, // 463: openshell.v1.OpenShell.FinalizeMainProcessExit:output_type -> openshell.v1.FinalizeMainProcessExitResponse - 199, // 464: openshell.v1.OpenShell.RelayStream:output_type -> openshell.v1.RelayFrame - 201, // 465: openshell.v1.OpenShell.PeerRelay:output_type -> openshell.v1.PeerRelayFrame - 85, // 466: openshell.v1.OpenShell.PeerReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse - 248, // 467: openshell.v1.OpenShell.PeerReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse - 83, // 468: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse - 110, // 469: openshell.v1.OpenShell.WatchSandbox:output_type -> openshell.v1.SandboxStreamEvent - 211, // 470: openshell.v1.OpenShell.SubmitPolicyAnalysis:output_type -> openshell.v1.SubmitPolicyAnalysisResponse - 213, // 471: openshell.v1.OpenShell.GetDraftPolicy:output_type -> openshell.v1.GetDraftPolicyResponse - 215, // 472: openshell.v1.OpenShell.ApproveDraftChunk:output_type -> openshell.v1.ApproveDraftChunkResponse - 217, // 473: openshell.v1.OpenShell.RejectDraftChunk:output_type -> openshell.v1.RejectDraftChunkResponse - 220, // 474: openshell.v1.OpenShell.ApproveAllDraftChunks:output_type -> openshell.v1.ApproveAllDraftChunksResponse - 222, // 475: openshell.v1.OpenShell.EditDraftChunk:output_type -> openshell.v1.EditDraftChunkResponse - 224, // 476: openshell.v1.OpenShell.UndoDraftChunk:output_type -> openshell.v1.UndoDraftChunkResponse - 226, // 477: openshell.v1.OpenShell.ClearDraftChunks:output_type -> openshell.v1.ClearDraftChunksResponse - 229, // 478: openshell.v1.OpenShell.GetDraftHistory:output_type -> openshell.v1.GetDraftHistoryResponse - 21, // 479: openshell.v1.OpenShell.IssueSandboxToken:output_type -> openshell.v1.IssueSandboxTokenResponse - 23, // 480: openshell.v1.OpenShell.RefreshSandboxToken:output_type -> openshell.v1.RefreshSandboxTokenResponse - 231, // 481: openshell.v1.OpenShell.CreateWorkspace:output_type -> openshell.v1.CreateWorkspaceResponse - 233, // 482: openshell.v1.OpenShell.GetWorkspace:output_type -> openshell.v1.GetWorkspaceResponse - 235, // 483: openshell.v1.OpenShell.ListWorkspaces:output_type -> openshell.v1.ListWorkspacesResponse - 237, // 484: openshell.v1.OpenShell.DeleteWorkspace:output_type -> openshell.v1.DeleteWorkspaceResponse - 240, // 485: openshell.v1.OpenShell.AddWorkspaceMember:output_type -> openshell.v1.AddWorkspaceMemberResponse - 242, // 486: openshell.v1.OpenShell.RemoveWorkspaceMember:output_type -> openshell.v1.RemoveWorkspaceMemberResponse - 244, // 487: openshell.v1.OpenShell.ListWorkspaceMembers:output_type -> openshell.v1.ListWorkspaceMembersResponse - 406, // [406:488] is the sub-list for method output_type - 324, // [324:406] is the sub-list for method input_type - 324, // [324:324] is the sub-list for extension type_name - 324, // [324:324] is the sub-list for extension extendee - 0, // [0:324] is the sub-list for field type_name + 275, // 319: openshell.v1.SandboxProvisioning.preparation_deadline:type_name -> google.protobuf.Timestamp + 275, // 320: openshell.v1.SandboxProvisioning.admission_start_time:type_name -> google.protobuf.Timestamp + 19, // 321: openshell.v1.SandboxServiceExposure.authorization_mode:type_name -> openshell.v1.ServiceAuthorizationMode + 275, // 322: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp + 275, // 323: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp + 127, // 324: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry.value:type_name -> openshell.v1.ProviderProfileCredential + 156, // 325: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry.value:type_name -> openshell.v1.StaticCredentialBinding + 24, // 326: openshell.v1.OpenShell.Health:input_type -> openshell.v1.HealthRequest + 26, // 327: openshell.v1.OpenShell.GetCurrentUser:input_type -> openshell.v1.GetCurrentUserRequest + 28, // 328: openshell.v1.OpenShell.GetGatewayInfo:input_type -> openshell.v1.GetGatewayInfoRequest + 52, // 329: openshell.v1.OpenShell.CreateSandbox:input_type -> openshell.v1.CreateSandboxRequest + 60, // 330: openshell.v1.OpenShell.BeginRootfsTarStaging:input_type -> openshell.v1.BeginRootfsTarStagingRequest + 62, // 331: openshell.v1.OpenShell.GetSandbox:input_type -> openshell.v1.GetSandboxRequest + 63, // 332: openshell.v1.OpenShell.ListSandboxes:input_type -> openshell.v1.ListSandboxesRequest + 53, // 333: openshell.v1.OpenShell.CreateSandboxTemplate:input_type -> openshell.v1.CreateSandboxTemplateRequest + 54, // 334: openshell.v1.OpenShell.GetSandboxTemplate:input_type -> openshell.v1.GetSandboxTemplateRequest + 55, // 335: openshell.v1.OpenShell.ListSandboxTemplates:input_type -> openshell.v1.ListSandboxTemplatesRequest + 56, // 336: openshell.v1.OpenShell.DeleteSandboxTemplate:input_type -> openshell.v1.DeleteSandboxTemplateRequest + 64, // 337: openshell.v1.OpenShell.ListSandboxProviders:input_type -> openshell.v1.ListSandboxProvidersRequest + 65, // 338: openshell.v1.OpenShell.AttachSandboxProvider:input_type -> openshell.v1.AttachSandboxProviderRequest + 66, // 339: openshell.v1.OpenShell.DetachSandboxProvider:input_type -> openshell.v1.DetachSandboxProviderRequest + 82, // 340: openshell.v1.OpenShell.GetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest + 67, // 341: openshell.v1.OpenShell.DeleteSandbox:input_type -> openshell.v1.DeleteSandboxRequest + 68, // 342: openshell.v1.OpenShell.StopSandbox:input_type -> openshell.v1.StopSandboxRequest + 69, // 343: openshell.v1.OpenShell.StartSandbox:input_type -> openshell.v1.StartSandboxRequest + 87, // 344: openshell.v1.OpenShell.CreateSshSession:input_type -> openshell.v1.CreateSshSessionRequest + 89, // 345: openshell.v1.OpenShell.ExposeService:input_type -> openshell.v1.ExposeServiceRequest + 90, // 346: openshell.v1.OpenShell.GetService:input_type -> openshell.v1.GetServiceRequest + 91, // 347: openshell.v1.OpenShell.ListServices:input_type -> openshell.v1.ListServicesRequest + 93, // 348: openshell.v1.OpenShell.DeleteService:input_type -> openshell.v1.DeleteServiceRequest + 97, // 349: openshell.v1.OpenShell.RevokeSshSession:input_type -> openshell.v1.RevokeSshSessionRequest + 99, // 350: openshell.v1.OpenShell.ExecSandbox:input_type -> openshell.v1.ExecSandboxRequest + 105, // 351: openshell.v1.OpenShell.ForwardTcp:input_type -> openshell.v1.TcpForwardFrame + 106, // 352: openshell.v1.OpenShell.ExecSandboxInteractive:input_type -> openshell.v1.ExecSandboxInput + 113, // 353: openshell.v1.OpenShell.CreateProvider:input_type -> openshell.v1.CreateProviderRequest + 114, // 354: openshell.v1.OpenShell.GetProvider:input_type -> openshell.v1.GetProviderRequest + 115, // 355: openshell.v1.OpenShell.ListProviders:input_type -> openshell.v1.ListProvidersRequest + 120, // 356: openshell.v1.OpenShell.ListProviderProfiles:input_type -> openshell.v1.ListProviderProfilesRequest + 121, // 357: openshell.v1.OpenShell.GetProviderProfile:input_type -> openshell.v1.GetProviderProfileRequest + 145, // 358: openshell.v1.OpenShell.ImportProviderProfiles:input_type -> openshell.v1.ImportProviderProfilesRequest + 147, // 359: openshell.v1.OpenShell.UpdateProviderProfiles:input_type -> openshell.v1.UpdateProviderProfilesRequest + 149, // 360: openshell.v1.OpenShell.LintProviderProfiles:input_type -> openshell.v1.LintProviderProfilesRequest + 116, // 361: openshell.v1.OpenShell.UpdateProvider:input_type -> openshell.v1.UpdateProviderRequest + 133, // 362: openshell.v1.OpenShell.GetProviderRefreshStatus:input_type -> openshell.v1.GetProviderRefreshStatusRequest + 135, // 363: openshell.v1.OpenShell.ConfigureProviderRefresh:input_type -> openshell.v1.ConfigureProviderRefreshRequest + 137, // 364: openshell.v1.OpenShell.RotateProviderCredential:input_type -> openshell.v1.RotateProviderCredentialRequest + 139, // 365: openshell.v1.OpenShell.DeleteProviderRefresh:input_type -> openshell.v1.DeleteProviderRefreshRequest + 117, // 366: openshell.v1.OpenShell.DeleteProvider:input_type -> openshell.v1.DeleteProviderRequest + 152, // 367: openshell.v1.OpenShell.DeleteProviderProfile:input_type -> openshell.v1.DeleteProviderProfileRequest + 290, // 368: openshell.v1.OpenShell.GetSandboxConfig:input_type -> openshell.sandbox.v1.GetSandboxConfigRequest + 291, // 369: openshell.v1.OpenShell.GetGatewayConfig:input_type -> openshell.sandbox.v1.GetGatewayConfigRequest + 160, // 370: openshell.v1.OpenShell.UpdateConfig:input_type -> openshell.v1.UpdateConfigRequest + 170, // 371: openshell.v1.OpenShell.GetSandboxPolicyStatus:input_type -> openshell.v1.GetSandboxPolicyStatusRequest + 172, // 372: openshell.v1.OpenShell.ListSandboxPolicies:input_type -> openshell.v1.ListSandboxPoliciesRequest + 174, // 373: openshell.v1.OpenShell.ReportPolicyStatus:input_type -> openshell.v1.ReportPolicyStatusRequest + 247, // 374: openshell.v1.OpenShell.ReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest + 84, // 375: openshell.v1.OpenShell.ReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest + 177, // 376: openshell.v1.OpenShell.ReportSandboxConfiguration:input_type -> openshell.v1.ReportSandboxConfigurationRequest + 154, // 377: openshell.v1.OpenShell.GetSandboxProviderEnvironment:input_type -> openshell.v1.GetSandboxProviderEnvironmentRequest + 158, // 378: openshell.v1.OpenShell.ExchangeProviderSubjectToken:input_type -> openshell.v1.ExchangeProviderSubjectTokenRequest + 180, // 379: openshell.v1.OpenShell.GetSandboxLogs:input_type -> openshell.v1.GetSandboxLogsRequest + 181, // 380: openshell.v1.OpenShell.PushSandboxLogs:input_type -> openshell.v1.PushSandboxLogsRequest + 184, // 381: openshell.v1.OpenShell.ConnectSupervisor:input_type -> openshell.v1.SupervisorMessage + 191, // 382: openshell.v1.OpenShell.ReportMainProcessExit:input_type -> openshell.v1.ReportMainProcessExitRequest + 193, // 383: openshell.v1.OpenShell.FinalizeMainProcessExit:input_type -> openshell.v1.FinalizeMainProcessExitRequest + 199, // 384: openshell.v1.OpenShell.RelayStream:input_type -> openshell.v1.RelayFrame + 201, // 385: openshell.v1.OpenShell.PeerRelay:input_type -> openshell.v1.PeerRelayFrame + 84, // 386: openshell.v1.OpenShell.PeerReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest + 247, // 387: openshell.v1.OpenShell.PeerReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest + 82, // 388: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest + 109, // 389: openshell.v1.OpenShell.WatchSandbox:input_type -> openshell.v1.WatchSandboxRequest + 210, // 390: openshell.v1.OpenShell.SubmitPolicyAnalysis:input_type -> openshell.v1.SubmitPolicyAnalysisRequest + 212, // 391: openshell.v1.OpenShell.GetDraftPolicy:input_type -> openshell.v1.GetDraftPolicyRequest + 214, // 392: openshell.v1.OpenShell.ApproveDraftChunk:input_type -> openshell.v1.ApproveDraftChunkRequest + 216, // 393: openshell.v1.OpenShell.RejectDraftChunk:input_type -> openshell.v1.RejectDraftChunkRequest + 219, // 394: openshell.v1.OpenShell.ApproveAllDraftChunks:input_type -> openshell.v1.ApproveAllDraftChunksRequest + 221, // 395: openshell.v1.OpenShell.EditDraftChunk:input_type -> openshell.v1.EditDraftChunkRequest + 223, // 396: openshell.v1.OpenShell.UndoDraftChunk:input_type -> openshell.v1.UndoDraftChunkRequest + 225, // 397: openshell.v1.OpenShell.ClearDraftChunks:input_type -> openshell.v1.ClearDraftChunksRequest + 227, // 398: openshell.v1.OpenShell.GetDraftHistory:input_type -> openshell.v1.GetDraftHistoryRequest + 20, // 399: openshell.v1.OpenShell.IssueSandboxToken:input_type -> openshell.v1.IssueSandboxTokenRequest + 22, // 400: openshell.v1.OpenShell.RefreshSandboxToken:input_type -> openshell.v1.RefreshSandboxTokenRequest + 230, // 401: openshell.v1.OpenShell.CreateWorkspace:input_type -> openshell.v1.CreateWorkspaceRequest + 232, // 402: openshell.v1.OpenShell.GetWorkspace:input_type -> openshell.v1.GetWorkspaceRequest + 234, // 403: openshell.v1.OpenShell.ListWorkspaces:input_type -> openshell.v1.ListWorkspacesRequest + 236, // 404: openshell.v1.OpenShell.DeleteWorkspace:input_type -> openshell.v1.DeleteWorkspaceRequest + 239, // 405: openshell.v1.OpenShell.AddWorkspaceMember:input_type -> openshell.v1.AddWorkspaceMemberRequest + 241, // 406: openshell.v1.OpenShell.RemoveWorkspaceMember:input_type -> openshell.v1.RemoveWorkspaceMemberRequest + 243, // 407: openshell.v1.OpenShell.ListWorkspaceMembers:input_type -> openshell.v1.ListWorkspaceMembersRequest + 25, // 408: openshell.v1.OpenShell.Health:output_type -> openshell.v1.HealthResponse + 27, // 409: openshell.v1.OpenShell.GetCurrentUser:output_type -> openshell.v1.GetCurrentUserResponse + 29, // 410: openshell.v1.OpenShell.GetGatewayInfo:output_type -> openshell.v1.GetGatewayInfoResponse + 70, // 411: openshell.v1.OpenShell.CreateSandbox:output_type -> openshell.v1.SandboxResponse + 61, // 412: openshell.v1.OpenShell.BeginRootfsTarStaging:output_type -> openshell.v1.BeginRootfsTarStagingResponse + 70, // 413: openshell.v1.OpenShell.GetSandbox:output_type -> openshell.v1.SandboxResponse + 71, // 414: openshell.v1.OpenShell.ListSandboxes:output_type -> openshell.v1.ListSandboxesResponse + 57, // 415: openshell.v1.OpenShell.CreateSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse + 57, // 416: openshell.v1.OpenShell.GetSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse + 58, // 417: openshell.v1.OpenShell.ListSandboxTemplates:output_type -> openshell.v1.ListSandboxTemplatesResponse + 59, // 418: openshell.v1.OpenShell.DeleteSandboxTemplate:output_type -> openshell.v1.DeleteSandboxTemplateResponse + 72, // 419: openshell.v1.OpenShell.ListSandboxProviders:output_type -> openshell.v1.ListSandboxProvidersResponse + 73, // 420: openshell.v1.OpenShell.AttachSandboxProvider:output_type -> openshell.v1.AttachSandboxProviderResponse + 74, // 421: openshell.v1.OpenShell.DetachSandboxProvider:output_type -> openshell.v1.DetachSandboxProviderResponse + 83, // 422: openshell.v1.OpenShell.GetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse + 86, // 423: openshell.v1.OpenShell.DeleteSandbox:output_type -> openshell.v1.DeleteSandboxResponse + 70, // 424: openshell.v1.OpenShell.StopSandbox:output_type -> openshell.v1.SandboxResponse + 70, // 425: openshell.v1.OpenShell.StartSandbox:output_type -> openshell.v1.SandboxResponse + 88, // 426: openshell.v1.OpenShell.CreateSshSession:output_type -> openshell.v1.CreateSshSessionResponse + 96, // 427: openshell.v1.OpenShell.ExposeService:output_type -> openshell.v1.ServiceEndpointResponse + 96, // 428: openshell.v1.OpenShell.GetService:output_type -> openshell.v1.ServiceEndpointResponse + 92, // 429: openshell.v1.OpenShell.ListServices:output_type -> openshell.v1.ListServicesResponse + 94, // 430: openshell.v1.OpenShell.DeleteService:output_type -> openshell.v1.DeleteServiceResponse + 98, // 431: openshell.v1.OpenShell.RevokeSshSession:output_type -> openshell.v1.RevokeSshSessionResponse + 103, // 432: openshell.v1.OpenShell.ExecSandbox:output_type -> openshell.v1.ExecSandboxEvent + 105, // 433: openshell.v1.OpenShell.ForwardTcp:output_type -> openshell.v1.TcpForwardFrame + 103, // 434: openshell.v1.OpenShell.ExecSandboxInteractive:output_type -> openshell.v1.ExecSandboxEvent + 118, // 435: openshell.v1.OpenShell.CreateProvider:output_type -> openshell.v1.ProviderResponse + 118, // 436: openshell.v1.OpenShell.GetProvider:output_type -> openshell.v1.ProviderResponse + 119, // 437: openshell.v1.OpenShell.ListProviders:output_type -> openshell.v1.ListProvidersResponse + 144, // 438: openshell.v1.OpenShell.ListProviderProfiles:output_type -> openshell.v1.ListProviderProfilesResponse + 143, // 439: openshell.v1.OpenShell.GetProviderProfile:output_type -> openshell.v1.ProviderProfileResponse + 146, // 440: openshell.v1.OpenShell.ImportProviderProfiles:output_type -> openshell.v1.ImportProviderProfilesResponse + 148, // 441: openshell.v1.OpenShell.UpdateProviderProfiles:output_type -> openshell.v1.UpdateProviderProfilesResponse + 150, // 442: openshell.v1.OpenShell.LintProviderProfiles:output_type -> openshell.v1.LintProviderProfilesResponse + 118, // 443: openshell.v1.OpenShell.UpdateProvider:output_type -> openshell.v1.ProviderResponse + 134, // 444: openshell.v1.OpenShell.GetProviderRefreshStatus:output_type -> openshell.v1.GetProviderRefreshStatusResponse + 136, // 445: openshell.v1.OpenShell.ConfigureProviderRefresh:output_type -> openshell.v1.ConfigureProviderRefreshResponse + 138, // 446: openshell.v1.OpenShell.RotateProviderCredential:output_type -> openshell.v1.RotateProviderCredentialResponse + 140, // 447: openshell.v1.OpenShell.DeleteProviderRefresh:output_type -> openshell.v1.DeleteProviderRefreshResponse + 151, // 448: openshell.v1.OpenShell.DeleteProvider:output_type -> openshell.v1.DeleteProviderResponse + 153, // 449: openshell.v1.OpenShell.DeleteProviderProfile:output_type -> openshell.v1.DeleteProviderProfileResponse + 292, // 450: openshell.v1.OpenShell.GetSandboxConfig:output_type -> openshell.sandbox.v1.GetSandboxConfigResponse + 293, // 451: openshell.v1.OpenShell.GetGatewayConfig:output_type -> openshell.sandbox.v1.GetGatewayConfigResponse + 169, // 452: openshell.v1.OpenShell.UpdateConfig:output_type -> openshell.v1.UpdateConfigResponse + 171, // 453: openshell.v1.OpenShell.GetSandboxPolicyStatus:output_type -> openshell.v1.GetSandboxPolicyStatusResponse + 173, // 454: openshell.v1.OpenShell.ListSandboxPolicies:output_type -> openshell.v1.ListSandboxPoliciesResponse + 175, // 455: openshell.v1.OpenShell.ReportPolicyStatus:output_type -> openshell.v1.ReportPolicyStatusResponse + 248, // 456: openshell.v1.OpenShell.ReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse + 85, // 457: openshell.v1.OpenShell.ReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse + 178, // 458: openshell.v1.OpenShell.ReportSandboxConfiguration:output_type -> openshell.v1.ReportSandboxConfigurationResponse + 157, // 459: openshell.v1.OpenShell.GetSandboxProviderEnvironment:output_type -> openshell.v1.GetSandboxProviderEnvironmentResponse + 159, // 460: openshell.v1.OpenShell.ExchangeProviderSubjectToken:output_type -> openshell.v1.ExchangeProviderSubjectTokenResponse + 183, // 461: openshell.v1.OpenShell.GetSandboxLogs:output_type -> openshell.v1.GetSandboxLogsResponse + 182, // 462: openshell.v1.OpenShell.PushSandboxLogs:output_type -> openshell.v1.PushSandboxLogsResponse + 185, // 463: openshell.v1.OpenShell.ConnectSupervisor:output_type -> openshell.v1.GatewayMessage + 192, // 464: openshell.v1.OpenShell.ReportMainProcessExit:output_type -> openshell.v1.ReportMainProcessExitResponse + 194, // 465: openshell.v1.OpenShell.FinalizeMainProcessExit:output_type -> openshell.v1.FinalizeMainProcessExitResponse + 199, // 466: openshell.v1.OpenShell.RelayStream:output_type -> openshell.v1.RelayFrame + 201, // 467: openshell.v1.OpenShell.PeerRelay:output_type -> openshell.v1.PeerRelayFrame + 85, // 468: openshell.v1.OpenShell.PeerReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse + 248, // 469: openshell.v1.OpenShell.PeerReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse + 83, // 470: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse + 110, // 471: openshell.v1.OpenShell.WatchSandbox:output_type -> openshell.v1.SandboxStreamEvent + 211, // 472: openshell.v1.OpenShell.SubmitPolicyAnalysis:output_type -> openshell.v1.SubmitPolicyAnalysisResponse + 213, // 473: openshell.v1.OpenShell.GetDraftPolicy:output_type -> openshell.v1.GetDraftPolicyResponse + 215, // 474: openshell.v1.OpenShell.ApproveDraftChunk:output_type -> openshell.v1.ApproveDraftChunkResponse + 217, // 475: openshell.v1.OpenShell.RejectDraftChunk:output_type -> openshell.v1.RejectDraftChunkResponse + 220, // 476: openshell.v1.OpenShell.ApproveAllDraftChunks:output_type -> openshell.v1.ApproveAllDraftChunksResponse + 222, // 477: openshell.v1.OpenShell.EditDraftChunk:output_type -> openshell.v1.EditDraftChunkResponse + 224, // 478: openshell.v1.OpenShell.UndoDraftChunk:output_type -> openshell.v1.UndoDraftChunkResponse + 226, // 479: openshell.v1.OpenShell.ClearDraftChunks:output_type -> openshell.v1.ClearDraftChunksResponse + 229, // 480: openshell.v1.OpenShell.GetDraftHistory:output_type -> openshell.v1.GetDraftHistoryResponse + 21, // 481: openshell.v1.OpenShell.IssueSandboxToken:output_type -> openshell.v1.IssueSandboxTokenResponse + 23, // 482: openshell.v1.OpenShell.RefreshSandboxToken:output_type -> openshell.v1.RefreshSandboxTokenResponse + 231, // 483: openshell.v1.OpenShell.CreateWorkspace:output_type -> openshell.v1.CreateWorkspaceResponse + 233, // 484: openshell.v1.OpenShell.GetWorkspace:output_type -> openshell.v1.GetWorkspaceResponse + 235, // 485: openshell.v1.OpenShell.ListWorkspaces:output_type -> openshell.v1.ListWorkspacesResponse + 237, // 486: openshell.v1.OpenShell.DeleteWorkspace:output_type -> openshell.v1.DeleteWorkspaceResponse + 240, // 487: openshell.v1.OpenShell.AddWorkspaceMember:output_type -> openshell.v1.AddWorkspaceMemberResponse + 242, // 488: openshell.v1.OpenShell.RemoveWorkspaceMember:output_type -> openshell.v1.RemoveWorkspaceMemberResponse + 244, // 489: openshell.v1.OpenShell.ListWorkspaceMembers:output_type -> openshell.v1.ListWorkspaceMembersResponse + 408, // [408:490] is the sub-list for method output_type + 326, // [326:408] is the sub-list for method input_type + 326, // [326:326] is the sub-list for extension type_name + 326, // [326:326] is the sub-list for extension extendee + 0, // [0:326] is the sub-list for field type_name } func init() { file_openshell_proto_init() } From fbefe47e0b7dceb3064ec5f44fa0360041d4dbde Mon Sep 17 00:00:00 2001 From: Shiju Date: Thu, 1 Oct 2026 14:38:06 +0530 Subject: [PATCH 2/7] fix(cli): simplify preparation timeout fallback selection Use lazy Option fallbacks while preserving timeout messages and retained sandbox behavior. Signed-off-by: Shiju --- crates/openshell-cli/src/run.rs | 35 +++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 65c5fc2f34..719e61767c 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1223,21 +1223,26 @@ pub async fn sandbox_create( .as_ref() .and_then(|status| status.provisioning.as_ref()) .filter(|record| record.timeout_time.is_some()); - let create_result = if let Some(record) = timed_out_provisioning { - Err(miette::miette!( - "{}", - retained_sandbox_timeout_message(&sandbox_name, &last_error_reason, record) - )) - } else if last_error_reason.is_empty() { - Err(miette::miette!( - "sandbox entered error phase while provisioning" - )) - } else { - Err(miette::miette!( - "sandbox entered error phase while provisioning: {}", - last_error_reason - )) - }; + let create_result = timed_out_provisioning.map_or_else( + || { + if last_error_reason.is_empty() { + Err(miette::miette!( + "sandbox entered error phase while provisioning" + )) + } else { + Err(miette::miette!( + "sandbox entered error phase while provisioning: {}", + last_error_reason + )) + } + }, + |record| { + Err(miette::miette!( + "{}", + retained_sandbox_timeout_message(&sandbox_name, &last_error_reason, record) + )) + }, + ); finalize_sandbox_create_session( &effective_server, &sandbox_name, From 14cd8f85563fd9b12a39f689ffb3b65b00d4581c Mon Sep 17 00:00:00 2001 From: Shiju Date: Thu, 1 Oct 2026 17:24:08 +0530 Subject: [PATCH 3/7] docs(server): clarify admission timer prerequisites Signed-off-by: Shiju --- crates/openshell-server/src/compute/provisioning_deadline.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/openshell-server/src/compute/provisioning_deadline.rs b/crates/openshell-server/src/compute/provisioning_deadline.rs index dfbb2fce64..56cb421e94 100644 --- a/crates/openshell-server/src/compute/provisioning_deadline.rs +++ b/crates/openshell-server/src/compute/provisioning_deadline.rs @@ -287,8 +287,9 @@ pub fn new_preparation_record(now_ms: i64, timeout_seconds: u32) -> SandboxProvi record } -/// The caller has checked the report's authentication, instance fence and -/// configuration generation. Persist this transition in the same CAS as admission. +/// The caller has authenticated the supervisor and checked its instance fence. +/// A Pending registration may start timing before configuration validation. +/// Persist this transition in the same CAS as the supervisor report. pub fn record_admission_start(record: &mut SandboxProvisioning, now_ms: i64) -> Result<(), String> { let mut deadline = ProvisioningDeadline::from_record(record)?; deadline.start_admission(&record.attempt_id, now_ms); From e052713307fd762663eecbd66be875314929ac88 Mon Sep 17 00:00:00 2001 From: Shiju Date: Sat, 3 Oct 2026 07:36:02 +0530 Subject: [PATCH 4/7] fix(compute): enforce deadlines during initial sandbox create Release stalled create operations after preparation expires and preserve the timeout diagnosis across late driver results. Keep failed-create cleanup bound to its original attempt so another replica can retry safely. Signed-off-by: Shiju --- crates/openshell-server/src/compute/mod.rs | 643 +++++++++++++++++++-- 1 file changed, 584 insertions(+), 59 deletions(-) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 48b5655abd..330d98f7ec 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -1121,40 +1121,58 @@ impl ComputeRuntime { spec.await_main_process_attachment = await_main_process_attachment; spec.launch_authentication = launch_authentication.unwrap_or_default(); } - match self - .driver - .call( - openshell_otel::rpc::CREATE_SANDBOX, - Some(sandbox.object_id()), - |driver| async move { - driver - .create_sandbox(Request::new(CreateSandboxRequest { - sandbox: Some(driver_sandbox), - })) - .await - }, + let result = self + .await_provisioning_operation( + &sandbox, + self.driver.call( + openshell_otel::rpc::CREATE_SANDBOX, + Some(sandbox.object_id()), + |driver| async move { + driver + .create_sandbox(Request::new(CreateSandboxRequest { + sandbox: Some(driver_sandbox), + })) + .await + }, + ), ) - .await + .await; + // Transfer the upload only after the driver actually accepts create, + // including a success that arrives after the preparation deadline. + if result.is_ok() + && let Some(staged) = staged.as_mut() { + staged.disarm(); + } + let global_guard = self.lock_global_for_lifecycle(&lifecycle_guard).await; + // The scanner can expire preparation while create owns the lifecycle + // gate. Every driver outcome must observe that durable decision before + // deleting records, publishing status, or compensating a failed create. + let current = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|error| Status::internal(format!("fetch created sandbox failed: {error}")))? + .ok_or_else(|| Status::not_found("sandbox removed during create"))?; + if sandbox_provisioning_attempt_id(¤t) != sandbox_provisioning_attempt_id(&sandbox) { + return Err(Status::aborted( + "sandbox provisioning attempt changed during create", + )); + } + sandbox = current; + if provisioning_deadline::timed_out(&sandbox) { + return Err(Status::deadline_exceeded( + "image preparation deadline expired", + )); + } + match result { Ok(response) => { let runtime_identity = response.into_inner().runtime_identity; - // The driver now owns the staged archive and removes the - // request directory once it has built the disk. - if let Some(staged) = staged.as_mut() { - staged.disarm(); - } - let global_guard = self.lock_global_for_lifecycle(&lifecycle_guard).await; if self.supports_sandbox_authentication() && runtime_identity.is_empty() { let status = Status::internal("compute driver did not return a runtime identity"); return Err(self - .compensate_failed_create( - &sandbox_id, - sandbox.object_name(), - lifecycle_guard, - global_guard, - status, - ) + .compensate_failed_create(&sandbox, lifecycle_guard, global_guard, status) .await); } if self.supports_sandbox_authentication() { @@ -1175,8 +1193,7 @@ impl ComputeRuntime { )); return Err(self .compensate_failed_create( - &sandbox_id, - sandbox.object_name(), + &sandbox, lifecycle_guard, global_guard, status, @@ -1189,46 +1206,64 @@ impl ComputeRuntime { self.sandbox_watch_bus.notify(sandbox.object_id()); Ok(sandbox) } - Err(status) if status.code() == Code::AlreadyExists => { - let _ = self - .store - .delete(Sandbox::object_type(), sandbox.object_id()) - .await; - self.sandbox_index.remove_sandbox(sandbox.object_id()); - Err(Status::already_exists("sandbox already exists")) - } - Err(status) if status.code() == Code::FailedPrecondition => { - let _ = self - .store - .delete(Sandbox::object_type(), sandbox.object_id()) - .await; - self.sandbox_index.remove_sandbox(sandbox.object_id()); - Err(Status::failed_precondition(status.message().to_string())) - } - Err(err) => { - let _ = self + Err(status) => { + // Another replica can expire this attempt after our read. + // Remove only the version inspected above, never a newer row. + match self .store - .delete(Sandbox::object_type(), sandbox.object_id()) - .await; - self.sandbox_index.remove_sandbox(sandbox.object_id()); - Err(Status::internal(format!( - "create sandbox failed: {}", - err.message() - ))) + .delete_if( + Sandbox::object_type(), + &sandbox_id, + sandbox_resource_version(&sandbox), + ) + .await + { + Ok(_) => self.sandbox_index.remove_sandbox(&sandbox_id), + Err(crate::persistence::PersistenceError::Conflict { .. }) => { + if let Some(current) = self + .store + .get_message::(&sandbox_id) + .await + .map_err(|error| Status::internal(error.to_string()))? + && provisioning_deadline::timed_out(¤t) + { + return Err(Status::deadline_exceeded( + "image preparation deadline expired", + )); + } + return Err(Status::aborted( + "sandbox changed during failed create cleanup", + )); + } + Err(error) => { + return Err(Status::internal(format!("clean up failed create: {error}"))); + } + } + match status.code() { + Code::AlreadyExists => Err(Status::already_exists("sandbox already exists")), + Code::FailedPrecondition => { + Err(Status::failed_precondition(status.message().to_string())) + } + _ => Err(Status::internal(format!( + "create sandbox failed: {}", + status.message() + ))), + } } } } async fn compensate_failed_create( &self, - sandbox_id: &str, - sandbox_name: &str, + created: &Sandbox, lifecycle_guard: SandboxLifecycleGuard, global_guard: tokio::sync::OwnedMutexGuard<()>, original: Status, ) -> Status { + let sandbox_id = created.object_id(); + let sandbox_name = created.object_name(); let transition = match self - .begin_sandbox_delete_with_initial_snapshot(sandbox_id, None) + .begin_sandbox_delete_with_initial_snapshot(sandbox_id, None, Some(created)) .await { Ok(BeginDelete::Started(transition)) => *transition, @@ -1241,6 +1276,9 @@ impl ComputeRuntime { ), ); } + Err(error) if matches!(error.code(), Code::DeadlineExceeded | Code::Aborted) => { + return error; + } Err(error) => { drop(global_guard); let delete_result = self @@ -1914,7 +1952,10 @@ impl ComputeRuntime { let current_generation = sandbox_runtime_generation(¤t)?; let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); - if current_generation != expected_generation || !allowed_phases.contains(&phase) + if current_generation != expected_generation + || sandbox_provisioning_attempt_id(¤t) + != sandbox_provisioning_attempt_id(starting) + || !allowed_phases.contains(&phase) { return Err(format!( "sandbox changed lifecycle ownership while persisting runtime identity (phase: {phase:?})" @@ -2265,7 +2306,7 @@ impl ComputeRuntime { // `Deleting` row used to fence recovery, and the prior row used only // for exact-version rollback after an ambiguous driver failure. let transition = match self - .begin_sandbox_delete_with_initial_snapshot(&target.sandbox_id, Some(current)) + .begin_sandbox_delete_with_initial_snapshot(&target.sandbox_id, Some(current), None) .await? { BeginDelete::AlreadyDeleting => { @@ -2344,6 +2385,7 @@ impl ComputeRuntime { &self, sandbox_id: &str, mut initial_snapshot: Option, + failed_create: Option<&Sandbox>, ) -> Result { let operation = "set sandbox phase to Deleting"; @@ -2358,6 +2400,24 @@ impl ComputeRuntime { .ok_or_else(|| Status::not_found("sandbox not found"))?, }; + // Failed-create compensation owns only its original attempt. + // Preserve its timeout diagnosis and any replacement attempt on + // every CAS retry. Explicit deletion may remove either state. + if let Some(created) = failed_create { + if sandbox_provisioning_attempt_id(&sandbox) + != sandbox_provisioning_attempt_id(created) + { + return Err(Status::aborted( + "sandbox provisioning attempt changed during create cleanup", + )); + } + if provisioning_deadline::timed_out(&sandbox) { + return Err(Status::deadline_exceeded( + "image preparation deadline expired", + )); + } + } + if SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown) == SandboxPhase::Deleting { @@ -5978,6 +6038,14 @@ fn decode_sandbox_record(record: &ObjectRecord) -> Result { Sandbox::decode(record.payload.as_slice()).map_err(|e| e.to_string()) } +fn sandbox_provisioning_attempt_id(sandbox: &Sandbox) -> Option<&str> { + sandbox + .status + .as_ref() + .and_then(|status| status.provisioning.as_ref()) + .map(|record| record.attempt_id.as_str()) +} + fn sandbox_resource_version(sandbox: &Sandbox) -> u64 { sandbox .metadata @@ -7549,8 +7617,10 @@ mod tests { delete_requests: TestMutex>, delete_outcome: TestMutex, create_started: Notify, + create_finished: Notify, create_release: Semaphore, create_blocked: AtomicBool, + create_error: TestMutex>, stop_started: Notify, stop_finished: Notify, stop_release: Semaphore, @@ -7588,8 +7658,10 @@ mod tests { delete_requests: TestMutex::new(Vec::new()), delete_outcome: TestMutex::new(ControlledDeleteOutcome::Ok(true)), create_started: Notify::new(), + create_finished: Notify::new(), create_release: Semaphore::new(0), create_blocked: AtomicBool::new(false), + create_error: TestMutex::new(None), stop_started: Notify::new(), stop_finished: Notify::new(), stop_release: Semaphore::new(0), @@ -7843,6 +7915,15 @@ mod tests { .expect("create release semaphore closed") .forget(); } + self.create_finished.notify_one(); + if let Some(error) = self + .create_error + .lock() + .expect("create error lock poisoned") + .clone() + { + return Err(error); + } Ok(tonic::Response::new(CreateSandboxResponse { runtime_identity: self .runtime_identity @@ -11700,7 +11781,7 @@ mod tests { .unwrap(); let transition = runtime - .begin_sandbox_delete_with_initial_snapshot("sb-1", Some(stale_snapshot)) + .begin_sandbox_delete_with_initial_snapshot("sb-1", Some(stale_snapshot), None) .await .unwrap(); let BeginDelete::Started(transition) = transition else { @@ -15403,6 +15484,450 @@ mod tests { .unwrap() } + fn stage_create_test_upload( + runtime: &mut ComputeRuntime, + sandbox: &mut Sandbox, + root: &Path, + ) -> PathBuf { + runtime.rootfs_tar_staging = Arc::new(rootfs_tar::RootfsTarStagingRegistry::new( + Some(root.to_path_buf()), + 1024, + )); + runtime.admission.allow_driver_config = true; + let slot = runtime + .rootfs_tar_staging + .begin("default", "test", "rootfs.tar", 7) + .unwrap(); + std::fs::write(&slot.upload_path, b"archive").unwrap(); + sandbox + .spec + .get_or_insert_with(SandboxSpec::default) + .template + .get_or_insert_with(SandboxTemplate::default) + .driver_config = Some(prost_types::Struct { + fields: [( + runtime.driver_info.name.clone(), + struct_value([("rootfs_tar_staging_token", string_value(&slot.token))]), + )] + .into_iter() + .collect(), + }); + slot.upload_path + } + + #[tokio::test] + async fn initial_create_preparation_expiry_releases_cleanup_gate() { + let driver = ControlledDriver::new(); + driver.block_create(); + let mut runtime = test_runtime(driver.clone()).await; + let now = openshell_core::time::now_ms(); + let preparation = provisioning_deadline::new_preparation_record(now, 1); + let mut sandbox = sandbox_record("sb-create-ttl", "create-ttl", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = Some(preparation.clone()); + let upload_root = tempfile::tempdir().unwrap(); + let upload = stage_create_test_upload(&mut runtime, &mut sandbox, upload_root.path()); + let creating_runtime = runtime.clone(); + let mut create = + tokio::spawn( + async move { creating_runtime.create_sandbox(sandbox, None, false).await }, + ); + tokio::time::timeout(Duration::from_secs(1), driver.create_started.notified()) + .await + .unwrap(); + + runtime + .reconcile_provisioning_deadlines(now + 1_000) + .await + .unwrap(); + assert_eq!( + driver.stop_calls(), + 0, + "create still owns the lifecycle gate" + ); + let result = match tokio::time::timeout(Duration::from_secs(3), &mut create).await { + Ok(result) => result.unwrap(), + Err(_) => { + create.abort(); + let _ = create.await; + panic!( + "expired initial create kept the lifecycle gate while the driver was blocked" + ); + } + }; + assert_eq!(result.unwrap_err().code(), Code::DeadlineExceeded); + tokio::time::timeout(Duration::from_secs(1), async { + while upload.exists() { + tokio::task::yield_now().await; + } + }) + .await + .expect("unaccepted upload must be cleaned up"); + // No create permit was released. Cleanup must acquire the lifecycle + // gate after the deadline waiter cancels the blocked operation. + runtime + .reconcile_provisioning_deadlines(now + 1_001) + .await + .unwrap(); + let retained = tokio::time::timeout(Duration::from_secs(1), async { + loop { + let sandbox = runtime + .store + .get_message::("sb-create-ttl") + .await + .unwrap() + .expect("retained timeout record"); + if sandbox + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap() + .cleanup_completed_time + .is_some() + { + break sandbox; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(driver.stop_calls(), 1); + assert_eq!(driver.delete_calls(), 0); + assert_eq!(retained.phase(), i32::from(SandboxPhase::Error)); + let status = retained.status.as_ref().unwrap(); + let record = status.provisioning.as_ref().unwrap(); + assert_eq!(record.attempt_id, preparation.attempt_id); + assert_eq!( + record.preparation_deadline, + preparation.preparation_deadline + ); + assert!( + status + .conditions + .iter() + .any(|c| c.reason == "ImagePreparationTimedOut") + ); + } + + #[tokio::test] + async fn failed_create_cleanup_preserves_timeout_after_cas_conflict() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let now = openshell_core::time::now_ms(); + let mut sandbox = sandbox_record( + "sb-compensation-ttl", + "compensation-ttl", + SandboxPhase::Provisioning, + ); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record(now, 1)); + runtime.store.put_message(&sandbox).await.unwrap(); + let stale = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + let lifecycle_guard = runtime.lifecycle_gates.lock_for(sandbox.object_id()).await; + runtime + .reconcile_provisioning_deadlines(now + 1_000) + .await + .unwrap(); + let _global_guard = runtime.lock_global_for_lifecycle(&lifecycle_guard).await; + let result = runtime + .begin_sandbox_delete_with_initial_snapshot( + sandbox.object_id(), + Some(stale.clone()), + Some(&stale), + ) + .await; + assert!(matches!(result, Err(error) if error.code() == Code::DeadlineExceeded)); + let retained = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert!(provisioning_deadline::timed_out(&retained)); + assert_eq!(driver.delete_calls(), 0); + } + + #[tokio::test] + async fn late_initial_create_preserves_retry_from_another_replica() { + let driver = ControlledDriver::new(); + driver.block_create(); + *driver.create_error.lock().unwrap() = Some(Status::internal("late create failure")); + let runtime = test_runtime(driver.clone()).await; + let mut other = runtime.clone(); + other.sync_lock = Arc::new(Mutex::new(())); + other.lifecycle_gates = Arc::new(LifecycleGateRegistry::default()); + other.replica_id = "other-replica".into(); + let now = openshell_core::time::now_ms(); + let mut sandbox = sandbox_record( + "sb-create-retry", + "create-retry", + SandboxPhase::Provisioning, + ); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record(now, 1)); + let creating_runtime = runtime.clone(); + let create = + tokio::spawn( + async move { creating_runtime.create_sandbox(sandbox, None, false).await }, + ); + driver.create_started.notified().await; + other + .reconcile_provisioning_deadlines(now + 1_000) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let current = other + .store + .get_message::("sb-create-retry") + .await + .unwrap() + .unwrap(); + if current + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap() + .cleanup_completed_time + .is_some() + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + // A's driver has completed, but only A's local lock prevents it from + // processing the result. B uses the same store with its own locks. + let held = runtime.sync_lock.lock().await; + driver.release_create(); + driver.create_finished.notified().await; + let retry = other + .start_sandbox("default", "create-retry") + .await + .unwrap(); + drop(held); + let result = tokio::time::timeout(Duration::from_secs(2), create) + .await + .unwrap() + .unwrap(); + assert_eq!(result.unwrap_err().code(), Code::Aborted); + let retained = runtime + .store + .get_message::("sb-create-retry") + .await + .unwrap() + .expect("new attempt must survive"); + assert_eq!(retained, retry); + assert_eq!(driver.delete_calls(), 0); + } + + #[tokio::test] + async fn failed_create_cleanup_preserves_retry_after_cas_conflict() { + let runtime = test_runtime(ControlledDriver::new()).await; + let mut sandbox = sandbox_record("sb-retry-cas", "retry-cas", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 1), + ); + runtime.store.put_message(&sandbox).await.unwrap(); + let stale = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + let retry = runtime + .store + .update_message_cas::( + sandbox.object_id(), + sandbox_resource_version(&stale), + |current| { + apply_lifecycle_phase( + current, + SandboxPhase::Starting, + "Starting", + "Sandbox start requested", + runtime.image_preparation_timeout_seconds, + ); + }, + ) + .await + .unwrap(); + assert!(sandbox_resource_version(&retry) > sandbox_resource_version(&stale)); + let result = runtime + .begin_sandbox_delete_with_initial_snapshot( + sandbox.object_id(), + Some(stale.clone()), + Some(&stale), + ) + .await; + assert!(matches!(result, Err(error) if error.code() == Code::Aborted)); + let retained = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(retained.status, retry.status); + } + + #[tokio::test] + async fn create_runtime_binding_rejects_retry_after_cas_conflict() { + let runtime = test_runtime(ControlledDriver::new()).await; + let mut sandbox = sandbox_record( + "sb-binding-retry", + "binding-retry", + SandboxPhase::Provisioning, + ); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 1), + ); + runtime.store.put_message(&sandbox).await.unwrap(); + let stale = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + let retry = runtime + .store + .update_message_cas::( + sandbox.object_id(), + sandbox_resource_version(&stale), + |current| { + apply_lifecycle_phase( + current, + SandboxPhase::Starting, + "Starting", + "Sandbox start requested", + runtime.image_preparation_timeout_seconds, + ); + current.set_phase(SandboxPhase::Ready.into()); + set_compute_runtime_binding(current, "retry-runtime"); + }, + ) + .await + .unwrap(); + assert_eq!( + sandbox_runtime_generation(&stale).unwrap(), + sandbox_runtime_generation(&retry).unwrap() + ); + let result = runtime + .persist_runtime_binding( + sandbox.object_id(), + &stale, + "test-driver", + "expired-create-runtime", + &[SandboxPhase::Provisioning, SandboxPhase::Ready], + ) + .await; + assert!( + result.is_err(), + "old create must not bind a replacement attempt" + ); + let retained = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(retained, retry); + } + + #[tokio::test] + async fn initial_create_preserves_timeout_when_driver_returns_late() { + // Exercise every former error-deletion branch and the authenticated + // success path that would otherwise compensate for missing identity. + for error in [ + None, + Some(Status::already_exists("late duplicate")), + Some(Status::failed_precondition("late rejection")), + Some(Status::internal("late failure")), + ] { + let driver = ControlledDriver::new(); + driver.block_create(); + let accepted_upload = error.is_none(); + *driver.create_error.lock().unwrap() = error; + let mut runtime = test_runtime(driver.clone()).await; + runtime.driver_info.supports_sandbox_authentication = true; + let now = openshell_core::time::now_ms(); + let preparation = provisioning_deadline::new_preparation_record(now, 1); + let mut sandbox = + sandbox_record("sb-late-create", "late-create", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = Some(preparation.clone()); + let upload_root = tempfile::tempdir().unwrap(); + let upload = stage_create_test_upload(&mut runtime, &mut sandbox, upload_root.path()); + let creating_runtime = runtime.clone(); + let create = + tokio::spawn( + async move { creating_runtime.create_sandbox(sandbox, None, false).await }, + ); + tokio::time::timeout(Duration::from_secs(1), driver.create_started.notified()) + .await + .unwrap(); + runtime + .reconcile_provisioning_deadlines(now + 1_000) + .await + .unwrap(); + driver.release_create(); + tokio::time::timeout(Duration::from_secs(1), driver.create_finished.notified()) + .await + .expect("driver result must reach the create path before cancellation"); + let result = tokio::time::timeout(Duration::from_secs(1), create) + .await + .unwrap() + .unwrap(); + assert_eq!(result.unwrap_err().code(), Code::DeadlineExceeded); + if accepted_upload { + assert!( + upload.exists(), + "successful driver owns the accepted upload" + ); + } else { + tokio::time::timeout(Duration::from_secs(1), async { + while upload.exists() { + tokio::task::yield_now().await; + } + }) + .await + .expect("failed create cleans up the upload"); + } + let retained = runtime + .store + .get_message::("sb-late-create") + .await + .unwrap() + .expect("late result must retain timeout diagnosis"); + assert!(provisioning_deadline::timed_out(&retained)); + let status = retained.status.as_ref().unwrap(); + assert_eq!( + status.provisioning.as_ref().unwrap().attempt_id, + preparation.attempt_id + ); + assert!( + status + .conditions + .iter() + .any(|c| c.reason == "ImagePreparationTimedOut") + ); + assert_eq!( + driver.delete_calls(), + 0, + "timeout cleanup must not enter create compensation" + ); + } + } + #[tokio::test] async fn preparation_expiry_releases_stalled_start_recovery_for_cleanup() { let driver = ControlledDriver::new(); From 02846c5e4f8e923fa74d5740f83194a4bbc4d9bb Mon Sep 17 00:00:00 2001 From: Shiju Date: Sat, 3 Oct 2026 07:51:35 +0530 Subject: [PATCH 5/7] test(server): satisfy deadline regression lints Drop the create-error mutex guard before matching its cloned value and use idiomatic iteration and timeout matching in the deadline fixtures. Signed-off-by: Shiju --- crates/openshell-server/src/compute/mod.rs | 28 ++++++++++------------ 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 330d98f7ec..fa6dd84926 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -7916,12 +7916,12 @@ mod tests { .forget(); } self.create_finished.notify_one(); - if let Some(error) = self + let create_error = self .create_error .lock() .expect("create error lock poisoned") - .clone() - { + .clone(); + if let Some(error) = create_error { return Err(error); } Ok(tonic::Response::new(CreateSandboxResponse { @@ -15505,11 +15505,10 @@ mod tests { .template .get_or_insert_with(SandboxTemplate::default) .driver_config = Some(prost_types::Struct { - fields: [( + fields: std::iter::once(( runtime.driver_info.name.clone(), struct_value([("rootfs_tar_staging_token", string_value(&slot.token))]), - )] - .into_iter() + )) .collect(), }); slot.upload_path @@ -15544,15 +15543,14 @@ mod tests { 0, "create still owns the lifecycle gate" ); - let result = match tokio::time::timeout(Duration::from_secs(3), &mut create).await { - Ok(result) => result.unwrap(), - Err(_) => { - create.abort(); - let _ = create.await; - panic!( - "expired initial create kept the lifecycle gate while the driver was blocked" - ); - } + let result = if let Ok(result) = + tokio::time::timeout(Duration::from_secs(3), &mut create).await + { + result.unwrap() + } else { + create.abort(); + let _ = create.await; + panic!("expired initial create kept the lifecycle gate while the driver was blocked"); }; assert_eq!(result.unwrap_err().code(), Code::DeadlineExceeded); tokio::time::timeout(Duration::from_secs(1), async { From a12d8b2927965f2f9a5d195ad7cb3522a33b79b0 Mon Sep 17 00:00:00 2001 From: Shiju Date: Sat, 3 Oct 2026 12:46:32 +0530 Subject: [PATCH 6/7] fix(server): retain ownership of pending provisioning operations Keep submitted create and start requests alive after caller cancellation, monitor failure, or preparation timeout. Persist request ownership before dispatch and retain staged uploads while the driver response is pending. Require timeout cleanup to stop compute after driver settlement without discarding an active cleanup claim. Fence late result handling against newer operations, preserve failed-start recovery, and defer automatic restart while another request owns the sandbox. Expose pending ownership in CLI JSON. Add ordered multi-replica and cancellation regressions for the review findings. Signed-off-by: Shiju --- crates/openshell-cli/src/run.rs | 24 + crates/openshell-server/src/compute/mod.rs | 2025 +++++++++++++---- .../src/compute/provisioning_deadline.rs | 25 +- .../src/compute/provisioning_operation.rs | 309 +++ .../src/compute/rootfs_tar.rs | 65 + crates/openshell-server/src/storage_proto.rs | 7 +- docs/how-it-works/sandboxes/overview.mdx | 4 + proto/openshell.proto | 10 + sdk/go/proto/openshellv1/openshell.pb.go | 34 +- 9 files changed, 2069 insertions(+), 434 deletions(-) create mode 100644 crates/openshell-server/src/compute/provisioning_operation.rs diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 719e61767c..353af3c4e9 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -2942,6 +2942,8 @@ fn sandbox_to_json(sandbox: &Sandbox) -> serde_json::Value { "cleanup_completed_time": record.cleanup_completed_time.as_ref().map(ToString::to_string), "cleanup_error": record.cleanup_error, "cleanup_retry_time": record.cleanup_retry_time.as_ref().map(ToString::to_string), + "driver_operation_pending": record.driver_operation_pending, + "driver_operation_id": record.driver_operation_id, })); serde_json::json!({ "id": sandbox.object_id(), @@ -8184,6 +8186,28 @@ mod tests { } } + #[test] + fn provisioning_json_exposes_pending_driver_operation() { + for pending in [true, false] { + let mut sandbox = Sandbox::default(); + sandbox.set_phase(SandboxPhase::Provisioning.into()); + sandbox.status.as_mut().unwrap().provisioning = + Some(openshell_core::proto::SandboxProvisioning { + driver_operation_pending: pending, + driver_operation_id: "operation-1".into(), + ..Default::default() + }); + assert_eq!( + super::sandbox_to_json(&sandbox)["provisioning"]["driver_operation_pending"], + pending + ); + assert_eq!( + super::sandbox_to_json(&sandbox)["provisioning"]["driver_operation_id"], + "operation-1" + ); + } + } + #[test] fn provisioning_json_distinguishes_preparation_and_admission() { let mut sandbox = Sandbox::default(); diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index fa6dd84926..0270157f88 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -6,6 +6,7 @@ pub mod driver_config; pub mod lease; pub mod provisioning_deadline; +mod provisioning_operation; pub mod rootfs_tar; use crate::grpc::policy::SANDBOX_SETTINGS_OBJECT_TYPE; @@ -73,6 +74,7 @@ pub type DriverWatchStream = pub type SharedComputeDriver = Arc + Send + Sync>; +use provisioning_operation::ProvisioningOperationError; use traced_driver::TracedDriver; const LIFECYCLE_SWEEP_PAGE_SIZE: u32 = 1000; @@ -156,6 +158,23 @@ mod traced_driver { .await } + /// Keep a submitted call alive independently of the request handler. + /// The owned future retains the driver, arguments, and tracing scope. + pub(super) fn call_owned( + &self, + rpc: openshell_otel::ComputeDriverRpc, + sandbox_id: Option<&str>, + call: impl FnOnce(SharedComputeDriver) -> Fut + Send + 'static, + ) -> impl Future> + Send + 'static + where + T: Send + 'static, + Fut: Future> + Send + 'static, + { + let driver = self.clone(); + let sandbox_id = sandbox_id.map(str::to_owned); + async move { driver.call(rpc, sandbox_id.as_deref(), call).await } + } + /// Open a driver watch while keeping the client span alive with the stream. pub(super) async fn watch(&self) -> Result, Status> { let span = self.span(openshell_otel::rpc::WATCH_SANDBOXES, None); @@ -1121,30 +1140,41 @@ impl ComputeRuntime { spec.await_main_process_attachment = await_main_process_attachment; spec.launch_authentication = launch_authentication.unwrap_or_default(); } - let result = self - .await_provisioning_operation( - &sandbox, - self.driver.call( - openshell_otel::rpc::CREATE_SANDBOX, - Some(sandbox.object_id()), - |driver| async move { - driver - .create_sandbox(Request::new(CreateSandboxRequest { - sandbox: Some(driver_sandbox), - })) - .await - }, - ), - ) - .await; - // Transfer the upload only after the driver actually accepts create, - // including a success that arrives after the preparation deadline. - if result.is_ok() - && let Some(staged) = staged.as_mut() - { - staged.disarm(); - } + let result = Box::pin(self.await_provisioning_operation( + &sandbox, + self.driver.call_owned( + openshell_otel::rpc::CREATE_SANDBOX, + Some(sandbox.object_id()), + move |driver| async move { + // The owned task keeps this input through monitor or + // caller cancellation. A crash leaves its sweep marker. + if let Some(staged) = staged.as_mut() { + staged.prepare_dispatch().await?; + staged.disarm(); + } + let response = driver + .create_sandbox(Request::new(CreateSandboxRequest { + sandbox: Some(driver_sandbox), + })) + .await; + if let Some(staged) = staged.as_mut() { + staged.finish_driver_operation(response.is_ok()).await; + } + response + }, + ), + )) + .await; let global_guard = self.lock_global_for_lifecycle(&lifecycle_guard).await; + // Result ownership comes from settlement, never from a newer row read + // after the driver returned. Recovery may reuse the same attempt. + let owned = match &result { + Ok((_, settled)) | Err(ProvisioningOperationError::Driver { settled, .. }) => settled, + Err( + ProvisioningOperationError::Monitor(status) + | ProvisioningOperationError::Unsettled(status), + ) => return Err(status.clone()), + }; // The scanner can expire preparation while create owns the lifecycle // gate. Every driver outcome must observe that durable decision before // deleting records, publishing status, or compensating a failed create. @@ -1154,11 +1184,7 @@ impl ComputeRuntime { .await .map_err(|error| Status::internal(format!("fetch created sandbox failed: {error}")))? .ok_or_else(|| Status::not_found("sandbox removed during create"))?; - if sandbox_provisioning_attempt_id(¤t) != sandbox_provisioning_attempt_id(&sandbox) { - return Err(Status::aborted( - "sandbox provisioning attempt changed during create", - )); - } + provisioning_operation::ensure_current_result(¤t, owned)?; sandbox = current; if provisioning_deadline::timed_out(&sandbox) { return Err(Status::deadline_exceeded( @@ -1166,7 +1192,7 @@ impl ComputeRuntime { )); } match result { - Ok(response) => { + Ok((response, _)) => { let runtime_identity = response.into_inner().runtime_identity; if self.supports_sandbox_authentication() && runtime_identity.is_empty() { let status = @@ -1182,7 +1208,13 @@ impl ComputeRuntime { &sandbox, self.configured_driver_name(), &runtime_identity, - &[SandboxPhase::Provisioning, SandboxPhase::Ready], + // A main-process exit can schedule automatic + // restart before this owned CREATE settles. + &[ + SandboxPhase::Provisioning, + SandboxPhase::Ready, + SandboxPhase::Starting, + ], ) .await; sandbox = match persisted { @@ -1206,7 +1238,15 @@ impl ComputeRuntime { self.sandbox_watch_bus.notify(sandbox.object_id()); Ok(sandbox) } - Err(status) => { + Err( + ProvisioningOperationError::Monitor(status) + | ProvisioningOperationError::Unsettled(status), + ) => { + // A monitor failure says nothing about the submitted create. + // Preserve its record even if the owner has since returned. + Err(status) + } + Err(ProvisioningOperationError::Driver { status, .. }) => { // Another replica can expire this attempt after our read. // Remove only the version inspected above, never a newer row. match self @@ -1276,28 +1316,24 @@ impl ComputeRuntime { ), ); } - Err(error) if matches!(error.code(), Code::DeadlineExceeded | Code::Aborted) => { + Err(error) + if matches!( + error.code(), + Code::DeadlineExceeded | Code::Aborted | Code::FailedPrecondition + ) => + { return error; } Err(error) => { - drop(global_guard); - let delete_result = self - .delete_backend_after_failed_create(sandbox_id, sandbox_name) - .await; - let cleanup_detail = match delete_result { - Ok(_) => String::new(), - Err(delete_error) => format!( - "; best-effort backend cleanup also failed: {}", - delete_error.message() - ), - }; + // A failed store read or CAS cannot establish cleanup + // ownership. Keep the record for authoritative cleanup; an + // unclaimed DELETE could destroy a newer same-ID runtime. return Status::new( original.code(), format!( - "{}; cleanup after successful create could not claim the sandbox record: {}{}", + "{}; cleanup after successful create could not claim the sandbox record: {}", original.message(), - error.message(), - cleanup_detail + error.message() ), ); } @@ -1400,6 +1436,7 @@ impl ComputeRuntime { )); } + provisioning_operation::ensure_operation_settled(¤t)?; let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); if matches!(phase, SandboxPhase::Stopped | SandboxPhase::Completed) || is_failed_main_process_result(¤t) @@ -1556,6 +1593,7 @@ impl ComputeRuntime { .await .map_err(|e| Status::internal(format!("fetch sandbox failed: {e}")))? .ok_or_else(|| Status::not_found("sandbox not found"))?; + provisioning_operation::ensure_operation_settled(&candidate)?; if provisioning_deadline::timed_out(&candidate) && candidate .status @@ -1592,6 +1630,7 @@ impl ComputeRuntime { let mut attempts = 0; let (previous, starting, launch_authentication) = loop { + provisioning_operation::ensure_operation_settled(¤t)?; let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); if phase == SandboxPhase::Ready { return Ok(current); @@ -1733,30 +1772,6 @@ impl ComputeRuntime { })? } - /// Release the lifecycle gate after the durable deadline expires, allowing - /// cleanup to cancel partial startup. Configuration repairs can extend this - /// deadline, so a fixed timeout around the driver RPC would expire too soon. - async fn await_provisioning_operation( - &self, - starting: &Sandbox, - operation: impl std::future::Future>, - ) -> Result { - tokio::pin!(operation); - loop { - tokio::select! { - result = &mut operation => return result, - () = tokio::time::sleep(Duration::from_secs(1)) => { - let current = self.store.get_message::(starting.object_id()) - .await.map_err(|error| Status::internal(error.to_string()))? - .ok_or_else(|| Status::not_found("sandbox removed during startup"))?; - if provisioning_deadline::timed_out(¤t) { - return Err(Status::deadline_exceeded("provisioning deadline expired")); - } - } - } - } - } - async fn complete_sandbox_start( &self, sandbox_id: String, @@ -1776,71 +1791,68 @@ impl ComputeRuntime { .map_err(Status::failed_precondition)? .into_string(); let expected_runtime_identity = sandbox_compute_runtime_identity(&previous); - let authentication_for_recreate = launch_authentication.clone(); - let mut result = self - .await_provisioning_operation( - &starting, - self.driver.call( - openshell_otel::rpc::START_SANDBOX, - Some(&sandbox_id), - |driver| { - let sandbox_id = sandbox_id.clone(); - let sandbox_name = sandbox_name.clone(); - async move { - driver - .start_sandbox(Request::new(StartSandboxRequest { - sandbox_id, - name: sandbox_name, - launch_authentication, - generation_id, - expected_runtime_identity, - })) - .await - } - }, - ), - ) - .await; - - if provisioning_deadline::timed_out(&previous) - && matches!(&result, Err(error) if error.code() == Code::NotFound) - { - // A partial provisioning attempt may have been canceled before a - // restartable backend object existed. Keep the API identity/spec. + // One owned operation covers both calls. A NotFound response alone + // does not end ownership while the recovery create can still run. + let recreate = if provisioning_deadline::timed_out(&previous) { let mut driver_sandbox = driver_sandbox_from_public(&starting, &self.driver_info.name) .map_err(|status| *status)?; if let Some(spec) = driver_sandbox.spec.as_mut() { - spec.launch_authentication = authentication_for_recreate; + spec.launch_authentication + .clone_from(&launch_authentication); } - result = self - .await_provisioning_operation( - &starting, - self.driver.call( - openshell_otel::rpc::CREATE_SANDBOX, - Some(&sandbox_id), - |driver| async move { - driver - .create_sandbox(Request::new(CreateSandboxRequest { - sandbox: Some(driver_sandbox), - })) - .await - .map(|response| { - tonic::Response::new( - openshell_core::proto::compute::v1::StartSandboxResponse { - runtime_identity: response - .into_inner() - .runtime_identity, - }, - ) - }) - }, - ), + Some(driver_sandbox) + } else { + None + }; + let driver = self.driver.clone(); + let operation_id = sandbox_id.clone(); + let result = Box::pin(self.await_provisioning_operation(&starting, async move { + let request_id = operation_id.clone(); + let response = driver + .call_owned( + openshell_otel::rpc::START_SANDBOX, + Some(&operation_id), + move |driver| async move { + driver + .start_sandbox(Request::new(StartSandboxRequest { + sandbox_id: request_id, + name: sandbox_name, + launch_authentication, + generation_id, + expected_runtime_identity, + })) + .await + }, ) .await; - } + match (response, recreate) { + (Err(error), Some(driver_sandbox)) if error.code() == Code::NotFound => { + driver + .call_owned( + openshell_otel::rpc::CREATE_SANDBOX, + Some(&operation_id), + move |driver| async move { + use openshell_core::proto::compute::v1::StartSandboxResponse; + let response = driver + .create_sandbox(Request::new(CreateSandboxRequest { + sandbox: Some(driver_sandbox), + })) + .await?; + Ok(tonic::Response::new(StartSandboxResponse { + runtime_identity: response.into_inner().runtime_identity, + })) + }, + ) + .await + } + (response, _) => response, + } + })) + .await; match result { - Ok(response) => { + Ok((response, starting)) => { + provisioning_operation::ensure_current_result(&starting, &starting)?; let runtime_identity = response.into_inner().runtime_identity; if self.supports_sandbox_authentication() && runtime_identity.is_empty() { let status = @@ -1883,18 +1895,29 @@ impl ComputeRuntime { } } } else { - self.store + let latest = self + .store .get_message::(&sandbox_id) .await .map_err(|e| Status::internal(format!("fetch sandbox failed: {e}")))? - .ok_or_else(|| Status::not_found("sandbox not found"))? + .ok_or_else(|| Status::not_found("sandbox not found"))?; + provisioning_operation::ensure_current_result(&latest, &starting)?; + latest }; self.sandbox_index.update_from_sandbox(&latest); self.sandbox_watch_bus.notify(&sandbox_id); Ok(latest) } - Err(err) => { - self.recover_failed_lifecycle(&lifecycle_guard, &starting, &previous, false) + Err( + ProvisioningOperationError::Monitor(error) + | ProvisioningOperationError::Unsettled(error), + ) => Err(error), + Err(ProvisioningOperationError::Driver { + status: err, + settled, + }) => { + provisioning_operation::ensure_current_result(&settled, &settled)?; + self.recover_failed_lifecycle(&lifecycle_guard, &settled, &previous, false) .await; Err(Status::new( err.code(), @@ -1912,6 +1935,8 @@ impl ComputeRuntime { runtime_identity: &str, allowed_phases: &[SandboxPhase], ) -> Result { + provisioning_operation::ensure_current_result(starting, starting) + .map_err(|error| error.to_string())?; let expected_generation = sandbox_runtime_generation(starting)?; let mut expected_resource_version = sandbox_resource_version(starting); @@ -1953,8 +1978,8 @@ impl ComputeRuntime { let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); if current_generation != expected_generation - || sandbox_provisioning_attempt_id(¤t) - != sandbox_provisioning_attempt_id(starting) + || provisioning_operation::ensure_current_result(¤t, starting) + .is_err() || !allowed_phases.contains(&phase) { return Err(format!( @@ -1983,40 +2008,51 @@ impl ComputeRuntime { previous: &Sandbox, original: Status, ) -> Status { - let sandbox_id = starting.object_id(); - let sandbox_name = starting.object_name(); + let sandbox_id = starting.object_id().to_string(); + let sandbox_name = starting.object_name().to_string(); + let request_id = sandbox_id.clone(); + // Claim compensation against the settled START operation before STOP. + // A newer operation or timeout must reject it without touching compute. let stop_result = self - .driver - .call( - openshell_otel::rpc::STOP_SANDBOX, - Some(sandbox_id), - |driver| { - let sandbox_id = sandbox_id.to_string(); - let sandbox_name = sandbox_name.to_string(); - async move { - driver - .stop_sandbox(Request::new(StopSandboxRequest { - sandbox_id, - name: sandbox_name, - })) - .await - } - }, + .await_provisioning_operation( + starting, + self.driver.call_owned( + openshell_otel::rpc::STOP_SANDBOX, + Some(&sandbox_id), + move |driver| { + let sandbox_id = request_id; + async move { + driver + .stop_sandbox(Request::new(StopSandboxRequest { + sandbox_id, + name: sandbox_name, + })) + .await + } + }, + ), ) .await; - if let Err(error) = stop_result { - return Status::new( - original.code(), - format!( - "{}; rollback after successful start failed: {}", - original.message(), - error.message() - ), - ); - } + let settled = match stop_result { + Ok((_, settled)) => settled, + Err( + error @ (ProvisioningOperationError::Monitor(_) + | ProvisioningOperationError::Unsettled(_)), + ) => return error.into(), + Err(error) => { + return Status::new( + original.code(), + format!( + "{}; rollback after successful start failed: {}", + original.message(), + Status::from(error).message() + ), + ); + } + }; let _global_guard = self.lock_global_for_lifecycle(lifecycle_guard).await; - if self.restore_lifecycle_snapshot(starting, previous).await { + if self.restore_lifecycle_snapshot(&settled, previous).await { original } else { Status::new( @@ -2172,8 +2208,23 @@ impl ComputeRuntime { } async fn restore_lifecycle_snapshot(&self, owned: &Sandbox, previous: &Sandbox) -> bool { + if provisioning_deadline::timed_out(owned) + || provisioning_deadline::driver_operation_pending(owned) + { + return false; + } let sandbox_id = owned.object_id().to_string(); - let previous = previous.clone(); + let mut previous = previous.clone(); + // Restore lifecycle state, not historical driver ownership. In + // particular a failed retry must not resurrect its prior timed-out + // attempt or reauthorize callbacks using an old operation ID. + previous + .status + .get_or_insert_with(Default::default) + .provisioning = owned + .status + .as_ref() + .and_then(|status| status.provisioning.clone()); match self .store .update_message_cas::( @@ -2399,23 +2450,13 @@ impl ComputeRuntime { .map_err(|e| Status::internal(format!("fetch sandbox failed: {e}")))? .ok_or_else(|| Status::not_found("sandbox not found"))?, }; + provisioning_operation::ensure_operation_settled(&sandbox)?; // Failed-create compensation owns only its original attempt. // Preserve its timeout diagnosis and any replacement attempt on - // every CAS retry. Explicit deletion may remove either state. + // every CAS retry. Explicit deletion requires a settled operation. if let Some(created) = failed_create { - if sandbox_provisioning_attempt_id(&sandbox) - != sandbox_provisioning_attempt_id(created) - { - return Err(Status::aborted( - "sandbox provisioning attempt changed during create cleanup", - )); - } - if provisioning_deadline::timed_out(&sandbox) { - return Err(Status::deadline_exceeded( - "image preparation deadline expired", - )); - } + provisioning_operation::ensure_current_result(&sandbox, created)?; } if SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown) @@ -2564,6 +2605,9 @@ impl ComputeRuntime { } let sandbox = decode_sandbox_record(&record)?; + if provisioning_deadline::driver_operation_pending(&sandbox) { + return Ok(false); + } self.cleanup_sandbox_owned_records(&sandbox).await?; match self @@ -3076,6 +3120,13 @@ impl ComputeRuntime { } }; + if provisioning_deadline::driver_operation_pending(&sandbox) { + warn!( + sandbox_id, + "Retaining pending driver operation during gateway recovery" + ); + continue; + } let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); let recoverable_error = phase == SandboxPhase::Error && is_recoverable_error_reason(&sandbox); @@ -3136,34 +3187,36 @@ impl ComputeRuntime { } }; let expected_runtime_identity = sandbox_compute_runtime_identity(&sandbox); - match self - .await_provisioning_operation( - &sandbox, - self.driver.call( - openshell_otel::rpc::START_SANDBOX, - Some(&sandbox_id), - |driver| { - let sandbox_id = sandbox_id.clone(); - let sandbox_name = sandbox_name.clone(); - let launch_authentication = launch_authentication.clone(); - let expected_runtime_identity = expected_runtime_identity.clone(); - async move { - driver - .start_sandbox(Request::new(StartSandboxRequest { - sandbox_id, - name: sandbox_name, - launch_authentication, - generation_id, - expected_runtime_identity, - })) - .await - } - }, - ), - ) - .await + let request_id = sandbox_id.clone(); + let request_name = sandbox_name.clone(); + match Box::pin(self.await_provisioning_operation( + &sandbox, + self.driver.call_owned( + openshell_otel::rpc::START_SANDBOX, + Some(&sandbox_id), + move |driver| { + let sandbox_id = request_id; + let sandbox_name = request_name; + let launch_authentication = launch_authentication.clone(); + let expected_runtime_identity = expected_runtime_identity.clone(); + async move { + driver + .start_sandbox(Request::new(StartSandboxRequest { + sandbox_id, + name: sandbox_name, + launch_authentication, + generation_id, + expected_runtime_identity, + })) + .await + } + }, + ), + )) + .await { - Ok(response) => { + Ok((response, settled)) => { + let sandbox = settled; let mut recovered_sandbox = sandbox.clone(); if self.supports_sandbox_authentication() { let runtime_identity = response.into_inner().runtime_identity; @@ -3193,7 +3246,7 @@ impl ComputeRuntime { ) .await { - Ok(updated) => recovered_sandbox = updated, + Ok(updated) => *recovered_sandbox = updated, Err(error) => { warn!( sandbox_id = %sandbox.object_id(), @@ -3230,7 +3283,18 @@ impl ComputeRuntime { recovered += 1; } } - Err(err) if err.code() == Code::NotFound => { + Err( + ProvisioningOperationError::Monitor(error) + | ProvisioningOperationError::Unsettled(error), + ) => { + warn!(sandbox_id, %error, "Gateway recovery stopped monitoring an owned driver operation"); + failed += 1; + } + Err(ProvisioningOperationError::Driver { + status: err, + settled, + }) if err.code() == Code::NotFound => { + let sandbox = settled; authentication_failed(sandbox.object_id()); // Backend resource is gone but the store still // remembers the sandbox. Mark Error so the UI @@ -3252,7 +3316,11 @@ impl ComputeRuntime { } missing += 1; } - Err(err) => { + Err(ProvisioningOperationError::Driver { + status: err, + settled, + }) => { + let sandbox = settled; authentication_failed(sandbox.object_id()); warn!( sandbox_id = %sandbox.object_id(), @@ -3306,6 +3374,13 @@ impl ComputeRuntime { continue; } }; + if provisioning_deadline::driver_operation_pending(&sandbox) { + warn!( + sandbox_id, + "Retaining pending driver operation during gateway recovery" + ); + continue; + } let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); match phase { SandboxPhase::Stopped | SandboxPhase::Completed => { @@ -3397,7 +3472,7 @@ impl ComputeRuntime { if let Err(err) = self .await_provisioning_operation( &sandbox, - self.driver.call( + self.driver.call_owned( openshell_otel::rpc::START_SANDBOX, Some(&sandbox_id), |driver| async move { @@ -3440,7 +3515,10 @@ impl ComputeRuntime { match self .store .update_message_cas::(&sandbox_id, 0, |s| { - if provisioning_deadline::timed_out(s) { + if provisioning_deadline::timed_out(s) + || provisioning_deadline::driver_operation_pending(s) + || !provisioning_operation::same_operation(s, sandbox) + { return; } s.set_phase(SandboxPhase::Error as i32); @@ -3752,6 +3830,14 @@ impl ComputeRuntime { } async fn restart_sandbox_runtime(&self, sandbox_id: &str) -> Result<(), String> { + use openshell_core::proto::compute::v1::StartSandboxResponse; + + enum RestartOutcome { + Started(StartSandboxResponse), + Superseded, + DriverError(&'static str, Status), + } + let lifecycle_guard = self.lifecycle_gates.lock_for(sandbox_id).await; let global_guard = self.lock_global_for_lifecycle(&lifecycle_guard).await; let Some(current) = self @@ -3763,12 +3849,18 @@ impl ComputeRuntime { return Ok(()); }; let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); - let now_ms = openshell_core::time::now_ms(); let due = current .status .as_ref() - .is_some_and(|status| restart_is_due(status, now_ms)); - if phase != SandboxPhase::Starting || !is_automatic_restart_transition(¤t) || !due { + .is_some_and(|status| restart_is_due(status, openshell_core::time::now_ms())); + if phase != SandboxPhase::Starting + || !is_automatic_restart_transition(¤t) + || !due + || provisioning_deadline::driver_operation_pending(¤t) + || provisioning_deadline::timed_out(¤t) + { + // Main-process exit may schedule restart while CREATE is pending. + // Keep that schedule intact so settlement makes it eligible again. return Ok(()); } @@ -3786,10 +3878,13 @@ impl ComputeRuntime { .transpose() .map_err(|status| status.to_string())? }; - let claimed = if already_claimed { + let claimed = if already_claimed && sandbox_provisioning_attempt_id(¤t).is_none() { current.clone() } else { - self.store + // Claim the restart schedule and driver ownership in one CAS. A + // pending check followed by an unclaimed STOP races another replica. + match self + .store .update_message_cas::( sandbox_id, sandbox_resource_version(¤t), @@ -3799,10 +3894,8 @@ impl ComputeRuntime { { identity.write(&mut metadata.annotations); } + provisioning_operation::claim_record(sandbox); let status = sandbox.status.get_or_insert_with(Default::default); - // Zero durably claims this due attempt across gateway - // replicas. The readiness watchdog starts only after the - // old runtime has stopped. set_next_restart_at_ms(status, 0); upsert_ready_condition( &mut sandbox.status, @@ -3819,77 +3912,97 @@ impl ComputeRuntime { }, ) .await - .map_err(|err| err.to_string())? + { + Ok(claimed) => claimed, + Err(crate::persistence::PersistenceError::Conflict { .. }) => return Ok(()), + Err(error) => return Err(error.to_string()), + } }; self.sandbox_index.update_from_sandbox(&claimed); self.sandbox_watch_bus.notify(sandbox_id); drop(global_guard); - let stop_result = self - .driver - .call( - openshell_otel::rpc::STOP_SANDBOX, - Some(sandbox_id), - |driver| { - let sandbox_id = sandbox_id.to_string(); - let sandbox_name = sandbox_name.clone(); - async move { - driver - .stop_sandbox(Request::new(StopSandboxRequest { - sandbox_id, - name: sandbox_name, - })) - .await - } - }, - ) - .await; - if let Err(status) = stop_result { - return self - .record_restart_driver_failure(&lifecycle_guard, sandbox_id, "stop", status) - .await; - } - - self.cleanup_stopped_sandbox_sessions(&claimed).await?; - - let Some(armed) = self.arm_restart_readiness_watchdog(&claimed).await? else { - // A concurrent stop or delete changed durable intent while the - // old runtime was stopping. Do not recreate it. - return Ok(()); - }; - - let launch_authentication = serialize_persisted_launch_authentication(authority, &armed) - .map_err(|status| status.to_string())?; - let generation_id = sandbox_runtime_generation(&armed)?.into_string(); + let runtime = self.clone(); + let owned = claimed.clone(); + let operation_id = sandbox_id.to_string(); + let operation_name = sandbox_name.clone(); let expected_runtime_identity = sandbox_compute_runtime_identity(¤t); + // One detached worker owns STOP and START. Caller cancellation cannot + // strand a raw START, and the monitor may release the local gate while + // the durable pending flag still protects the unresolved operation. + let result = self + .await_claimed_provisioning_operation(&claimed, async move { + let request_id = operation_id.clone(); + let request_name = operation_name.clone(); + let stop = runtime + .driver + .call_owned( + openshell_otel::rpc::STOP_SANDBOX, + Some(&operation_id), + move |driver| async move { + driver + .stop_sandbox(Request::new(StopSandboxRequest { + sandbox_id: request_id, + name: request_name, + })) + .await + }, + ) + .await; + if let Err(status) = stop { + return Ok(RestartOutcome::DriverError("stop", status)); + } + runtime + .cleanup_stopped_sandbox_sessions(&owned) + .await + .map_err(Status::internal)?; + let Some(armed) = runtime + .arm_restart_readiness_watchdog(&owned) + .await + .map_err(Status::internal)? + else { + return Ok(RestartOutcome::Superseded); + }; + let authority = runtime.restart_authority.get().and_then(Option::as_deref); + let launch_authentication = + serialize_persisted_launch_authentication(authority, &armed)?; + let generation_id = sandbox_runtime_generation(&armed) + .map_err(Status::failed_precondition)? + .into_string(); + let request_id = operation_id.clone(); + let start = runtime + .driver + .call_owned( + openshell_otel::rpc::START_SANDBOX, + Some(&operation_id), + move |driver| async move { + driver + .start_sandbox(Request::new(StartSandboxRequest { + sandbox_id: request_id, + name: operation_name, + launch_authentication, + generation_id, + expected_runtime_identity, + })) + .await + }, + ) + .await; + Ok(match start { + Ok(response) => RestartOutcome::Started(response.into_inner()), + Err(status) => RestartOutcome::DriverError("start", status), + }) + }) + .await + .map_err(|error| error.to_string())?; - let start_result = self - .driver - .call( - openshell_otel::rpc::START_SANDBOX, - Some(sandbox_id), - |driver| { - let sandbox_id = sandbox_id.to_string(); - let sandbox_name = sandbox_name.clone(); - async move { - driver - .start_sandbox(Request::new(StartSandboxRequest { - sandbox_id, - name: sandbox_name, - launch_authentication, - generation_id, - expected_runtime_identity, - })) - .await - } - }, - ) - .await; - let response = match start_result { - Ok(response) => response.into_inner(), - Err(status) => { + let (outcome, settled) = result; + let response = match outcome { + RestartOutcome::Started(response) => response, + RestartOutcome::Superseded => return Ok(()), + RestartOutcome::DriverError(stage, status) => { return self - .record_restart_driver_failure(&lifecycle_guard, sandbox_id, "start", status) + .record_restart_driver_failure(&lifecycle_guard, &settled, stage, status) .await; } }; @@ -3901,16 +4014,14 @@ impl ComputeRuntime { } self.persist_runtime_binding( sandbox_id, - &armed, + &settled, self.configured_driver_name(), &response.runtime_identity, &[SandboxPhase::Starting, SandboxPhase::Ready], ) .await?; } - - self.enforce_lifecycle_after_restart_start(&armed).await?; - + self.enforce_lifecycle_after_restart_start(&settled).await?; info!( sandbox_id, sandbox_name, "Sandbox runtime restarted; waiting for replacement supervisor" @@ -3931,6 +4042,9 @@ impl ComputeRuntime { for _ in 0..DELETE_PHASE_CAS_RETRY_LIMIT { let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); let claim_owned = phase == SandboxPhase::Starting + && provisioning_operation::same_operation(¤t, claimed) + && sandbox_runtime_generation(¤t) == sandbox_runtime_generation(claimed) + && !provisioning_deadline::timed_out(¤t) && is_automatic_restart_transition(¤t) && current .status @@ -3989,6 +4103,16 @@ impl ComputeRuntime { .get_message::(&sandbox_id) .await .map_err(|err| err.to_string())?; + if sandbox_provisioning_attempt_id(armed).is_some() + && latest.as_ref().is_none_or(|current| { + !provisioning_operation::same_operation(current, armed) + || sandbox_runtime_generation(current) != sandbox_runtime_generation(armed) + || provisioning_deadline::timed_out(current) + }) + { + // Newer operations and timeout reclamation own their own cleanup. + return Ok(()); + } let phase = latest.as_ref().map(|sandbox| { SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown) }); @@ -4059,10 +4183,11 @@ impl ComputeRuntime { async fn record_restart_driver_failure( &self, lifecycle_guard: &SandboxLifecycleGuard, - sandbox_id: &str, + settled: &Sandbox, operation: &str, driver_status: Status, ) -> Result<(), String> { + let sandbox_id = settled.object_id(); let _global_guard = self.lock_global_for_lifecycle(lifecycle_guard).await; let Some(current) = self .store @@ -4072,7 +4197,9 @@ impl ComputeRuntime { else { return Ok(()); }; - if !is_automatic_restart_transition(¤t) { + if !is_automatic_restart_transition(¤t) + || provisioning_operation::ensure_current_result(¤t, settled).is_err() + { return Ok(()); } let terminal = driver_status.code() == Code::NotFound; @@ -4883,8 +5010,9 @@ impl ComputeRuntime { .await .map_err(|e| e.to_string())?; if let Some(sandbox) = sandbox.as_ref() { - if provisioning_deadline::timed_out(sandbox) - && sandbox.phase() == i32::from(SandboxPhase::Error) + if provisioning_deadline::driver_operation_pending(sandbox) + || (provisioning_deadline::timed_out(sandbox) + && sandbox.phase() == i32::from(SandboxPhase::Error)) { return Ok(()); } @@ -4894,8 +5022,16 @@ impl ComputeRuntime { // processed sequentially, so this must not block on the driver // call itself, only on the (instant, non-blocking) decision to // make it. - self.spawn_driver_sandbox_cleanup(sandbox.object_id(), sandbox.object_name()); - self.cleanup_sandbox_owned_records(sandbox).await?; + if self + .remove_sandbox_record_if_version_locked( + sandbox_id, + sandbox_resource_version(sandbox), + ) + .await? + { + self.spawn_driver_sandbox_cleanup(sandbox.object_id(), sandbox.object_name()); + } + return Ok(()); } let _ = self @@ -4912,6 +5048,9 @@ impl ComputeRuntime { sandbox: &Sandbox, expected_resource_version: u64, ) -> Result<(), String> { + if provisioning_deadline::driver_operation_pending(sandbox) { + return Ok(()); + } let sandbox_id = sandbox.object_id(); self.remove_sandbox_record_if_version_locked(sandbox_id, expected_resource_version) .await?; @@ -5227,6 +5366,9 @@ impl ComputeRuntime { } let sandbox = decode_sandbox_record(¤t_record)?; + if provisioning_deadline::driver_operation_pending(&sandbox) { + return Ok(()); + } let age_ms = openshell_core::time::now_ms() .saturating_sub(current_record.created_at_ms) .max(0); @@ -7621,6 +7763,8 @@ mod tests { create_release: Semaphore, create_blocked: AtomicBool, create_error: TestMutex>, + track_compute: AtomicBool, + compute_exists: AtomicBool, stop_started: Notify, stop_finished: Notify, stop_release: Semaphore, @@ -7662,6 +7806,8 @@ mod tests { create_release: Semaphore::new(0), create_blocked: AtomicBool::new(false), create_error: TestMutex::new(None), + track_compute: AtomicBool::new(false), + compute_exists: AtomicBool::new(false), stop_started: Notify::new(), stop_finished: Notify::new(), stop_release: Semaphore::new(0), @@ -7924,6 +8070,9 @@ mod tests { if let Some(error) = create_error { return Err(error); } + if self.track_compute.load(Ordering::SeqCst) { + self.compute_exists.store(true, Ordering::SeqCst); + } Ok(tonic::Response::new(CreateSandboxResponse { runtime_identity: self .runtime_identity @@ -7943,6 +8092,12 @@ mod tests { .expect("stop requests lock poisoned") .push((request.sandbox_id, request.name)); self.stop_calls.fetch_add(1, Ordering::SeqCst); + // Observe backend absence before the barrier, as a STOP may do + // before a concurrently submitted CREATE materializes compute. + let tracked_exists = self + .track_compute + .load(Ordering::SeqCst) + .then(|| self.compute_exists.swap(false, Ordering::SeqCst)); self.stop_started.notify_one(); if self.stop_blocked.load(Ordering::SeqCst) { self.stop_release @@ -7952,11 +8107,15 @@ mod tests { .forget(); } self.stop_finished.notify_one(); - let outcome = self - .stop_outcome - .lock() - .expect("stop outcome lock poisoned") - .clone(); + let outcome = match tracked_exists { + Some(true) => ControlledLifecycleOutcome::Ok, + Some(false) => ControlledLifecycleOutcome::NotFound, + None => self + .stop_outcome + .lock() + .expect("stop outcome lock poisoned") + .clone(), + }; match outcome { ControlledLifecycleOutcome::Ok => Ok(tonic::Response::new(StopSandboxResponse {})), ControlledLifecycleOutcome::NotFound => Err(Status::not_found("sandbox not found")), @@ -8624,12 +8783,13 @@ mod tests { .await .expect("start task must finish") .expect_err("a replaced generation must reject the prior runtime binding"); - assert!( - error - .message() - .contains("changed lifecycle ownership while persisting runtime identity") + assert_eq!(error.code(), Code::Aborted); + assert!(error.message().contains("runtime generation changed")); + assert_eq!( + driver.stop_calls(), + 0, + "stale START cannot stop replacement compute" ); - assert_eq!(driver.stop_calls(), 1); let retained = runtime .store .get_message::(sandbox.object_id()) @@ -11130,8 +11290,8 @@ mod tests { assert_eq!(stored_identity.auth_epoch.get(), 2); assert_eq!( sandbox_resource_version(&stored), - sandbox_resource_version(&before) + 1, - "identity rotation and Starting must be one CAS write" + sandbox_resource_version(&before) + 3, + "identity/Starting transition, operation claim, and settlement are separate CAS writes" ); runtime @@ -11165,8 +11325,8 @@ mod tests { .unwrap(); assert_eq!( sandbox_resource_version(&after_retry), - sandbox_resource_version(&stored), - "idempotent Starting retry must reuse the persisted identity" + sandbox_resource_version(&stored) + 2, + "Starting retry claims and settles another operation without rotating identity" ); } @@ -11195,6 +11355,8 @@ mod tests { .await .expect("detached start worker did not finish the driver call"); + wait_driver_pending(&runtime, sandbox.object_id(), false).await; + driver.release_start(); let starting = tokio::time::timeout( Duration::from_secs(1), @@ -15484,6 +15646,25 @@ mod tests { .unwrap() } + async fn wait_driver_pending(runtime: &ComputeRuntime, id: &str, pending: bool) -> Sandbox { + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let sandbox = runtime + .store + .get_message::(id) + .await + .unwrap() + .unwrap(); + if provisioning_deadline::driver_operation_pending(&sandbox) == pending { + break sandbox; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("driver ownership must reach the expected durable state") + } + fn stage_create_test_upload( runtime: &mut ComputeRuntime, sandbox: &mut Sandbox, @@ -15514,101 +15695,1000 @@ mod tests { slot.upload_path } + async fn assert_late_create_preserves_recovery(create_error: bool, create_identity: &str) { + for recovery_pending in [true, false] { + let driver = ControlledDriver::new(); + driver.block_create(); + driver.set_runtime_identity(create_identity); + if create_error { + *driver.create_error.lock().unwrap() = + Some(Status::failed_precondition("create rejected")); + } + let mut runtime = test_runtime(driver.clone()).await; + enable_runtime_identity_binding(&mut runtime); + let mut other = runtime.clone(); + other.sync_lock = Arc::new(Mutex::new(())); + other.lifecycle_gates = Arc::new(LifecycleGateRegistry::default()); + other.driver_info.gateway_manages_lifecycle = true; + let mut sandbox = sandbox_record( + "sb-result-owner", + "result-owner", + SandboxPhase::Provisioning, + ); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 1800), + ); + let creating = runtime.clone(); + let create = + tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); + driver.create_started.notified().await; + let held = runtime.sync_lock.lock().await; + driver.release_create(); + wait_driver_pending(&other, "sb-result-owner", false).await; + driver.set_runtime_identity("recovery-runtime"); + if recovery_pending { + driver.block_start(); + } + let recovering = other.clone(); + let recovery = + tokio::spawn(async move { recovering.start_persisted_sandboxes().await }); + driver.start_started.notified().await; + if !recovery_pending { + wait_driver_pending(&other, "sb-result-owner", false).await; + // Wait for the binding callback, not just the driver response. + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let row = other + .store + .get_message::("sb-result-owner") + .await + .unwrap() + .unwrap(); + if sandbox_compute_runtime_identity(&row) == "recovery-runtime" { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } + drop(held); + let result = create.await.unwrap(); + let retained = other + .store + .get_message::("sb-result-owner") + .await + .unwrap() + .expect("newer recovery operation must retain its row"); + assert_eq!( + driver.delete_calls(), + 0, + "old CREATE must not compensate against recovery" + ); + if !recovery_pending { + assert_eq!( + sandbox_compute_runtime_identity(&retained), + "recovery-runtime" + ); + } + assert!( + result.is_err(), + "superseded CREATE must not publish success" + ); + if recovery_pending { + driver.release_start(); + } + recovery.await.unwrap().unwrap(); + } + } + #[tokio::test] - async fn initial_create_preparation_expiry_releases_cleanup_gate() { + async fn late_create_error_preserves_newer_recovery_operation() { + assert_late_create_preserves_recovery(true, "").await; + } + + #[tokio::test] + async fn late_create_success_preserves_newer_recovery_binding() { + assert_late_create_preserves_recovery(false, "create-runtime").await; + } + + #[tokio::test] + async fn late_create_compensation_preserves_newer_recovery_operation() { + assert_late_create_preserves_recovery(false, "").await; + } + + #[tokio::test] + async fn driver_settlement_preserves_active_cleanup_stop_lease() { let driver = ControlledDriver::new(); driver.block_create(); - let mut runtime = test_runtime(driver.clone()).await; + driver.block_stop(); + let runtime = test_runtime(driver.clone()).await; + let mut other = runtime.clone(); + other.sync_lock = Arc::new(Mutex::new(())); + other.lifecycle_gates = Arc::new(LifecycleGateRegistry::default()); let now = openshell_core::time::now_ms(); - let preparation = provisioning_deadline::new_preparation_record(now, 1); - let mut sandbox = sandbox_record("sb-create-ttl", "create-ttl", SandboxPhase::Provisioning); - sandbox.status.as_mut().unwrap().provisioning = Some(preparation.clone()); - let upload_root = tempfile::tempdir().unwrap(); - let upload = stage_create_test_upload(&mut runtime, &mut sandbox, upload_root.path()); - let creating_runtime = runtime.clone(); - let mut create = - tokio::spawn( - async move { creating_runtime.create_sandbox(sandbox, None, false).await }, - ); - tokio::time::timeout(Duration::from_secs(1), driver.create_started.notified()) + let mut sandbox = sandbox_record("sb-stop-lease", "stop-lease", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record(now, 1)); + let creating = runtime.clone(); + let create = + tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); + driver.create_started.notified().await; + other + .reconcile_provisioning_deadlines(now + 1000) .await .unwrap(); - - runtime - .reconcile_provisioning_deadlines(now + 1_000) + driver.stop_started.notified().await; + let before = other + .store + .get_message::("sb-stop-lease") .await + .unwrap() .unwrap(); + let lease = before + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap() + .cleanup_retry_time; + assert!(lease.is_some()); + driver.release_create(); assert_eq!( - driver.stop_calls(), - 0, - "create still owns the lifecycle gate" + create.await.unwrap().unwrap_err().code(), + Code::DeadlineExceeded ); - let result = if let Ok(result) = - tokio::time::timeout(Duration::from_secs(3), &mut create).await - { - result.unwrap() - } else { - create.abort(); - let _ = create.await; - panic!("expired initial create kept the lifecycle gate while the driver was blocked"); - }; - assert_eq!(result.unwrap_err().code(), Code::DeadlineExceeded); - tokio::time::timeout(Duration::from_secs(1), async { - while upload.exists() { - tokio::task::yield_now().await; - } - }) - .await - .expect("unaccepted upload must be cleaned up"); - // No create permit was released. Cleanup must acquire the lifecycle - // gate after the deadline waiter cancels the blocked operation. + let settled = wait_driver_pending(&runtime, "sb-stop-lease", false).await; + assert_eq!( + settled + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap() + .cleanup_retry_time, + lease, + "settlement cannot erase an active STOP lease" + ); + let gate = runtime.lifecycle_gates.lock_for("sb-stop-lease").await; runtime - .reconcile_provisioning_deadlines(now + 1_001) + .reclaim_provisioning_timeout(&settled, &gate) .await .unwrap(); - let retained = tokio::time::timeout(Duration::from_secs(1), async { - loop { - let sandbox = runtime - .store - .get_message::("sb-create-ttl") - .await - .unwrap() - .expect("retained timeout record"); - if sandbox - .status - .as_ref() - .unwrap() - .provisioning - .as_ref() - .unwrap() - .cleanup_completed_time - .is_some() - { - break sandbox; - } - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - assert_eq!(driver.stop_calls(), 1); - assert_eq!(driver.delete_calls(), 0); - assert_eq!(retained.phase(), i32::from(SandboxPhase::Error)); - let status = retained.status.as_ref().unwrap(); - let record = status.provisioning.as_ref().unwrap(); - assert_eq!(record.attempt_id, preparation.attempt_id); assert_eq!( - record.preparation_deadline, - preparation.preparation_deadline + driver.stop_calls(), + 1, + "second replica must honor active STOP lease" ); + let current = runtime + .store + .get_message::("sb-stop-lease") + .await + .unwrap() + .unwrap(); assert!( - status - .conditions - .iter() - .any(|c| c.reason == "ImagePreparationTimedOut") - ); - } - + current + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap() + .cleanup_completed_time + .is_none() + ); + drop(gate); + driver.release_stop(); + let completed = other.lifecycle_gates.lock_for("sb-stop-lease").await; + drop(completed); + } + + #[tokio::test] + async fn automatic_restart_waits_for_pending_driver_and_preserves_schedule() { + let driver = ControlledDriver::new(); + driver.block_create(); + driver.set_runtime_identity("create-runtime"); + let mut runtime = test_runtime(driver.clone()).await; + enable_runtime_identity_binding(&mut runtime); + let mut other = runtime.clone(); + other.sync_lock = Arc::new(Mutex::new(())); + other.lifecycle_gates = Arc::new(LifecycleGateRegistry::default()); + let mut sandbox = + sandbox_record("sb-restart-pending", "restart-pending", SandboxPhase::Ready); + sandbox + .spec + .get_or_insert_with(SandboxSpec::default) + .restart_policy = SandboxRestartPolicy::OnFailure as i32; + let status = sandbox.status.as_mut().unwrap(); + status.main_process_instance_id = "instance-1".into(); + status.provisioning = Some(provisioning_deadline::new_preparation_record( + openshell_core::time::now_ms(), + 1800, + )); + let creating = runtime.clone(); + let create = + tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); + driver.create_started.notified().await; + other + .main_process_exited("sb-restart-pending", "instance-1", 9) + .await + .unwrap(); + other + .finalize_main_process_exit("sb-restart-pending", "instance-1") + .await + .unwrap(); + let scheduled = other + .store + .get_message::("sb-restart-pending") + .await + .unwrap() + .unwrap(); + other + .restart_sandbox_runtime("sb-restart-pending") + .await + .unwrap(); + assert_eq!( + driver.stop_calls(), + 0, + "automatic restart cannot STOP an owned driver operation" + ); + assert_eq!(driver.start_calls(), 0); + let deferred = other + .store + .get_message::("sb-restart-pending") + .await + .unwrap() + .unwrap(); + assert_eq!( + deferred, scheduled, + "pending restart keeps its durable schedule" + ); + driver.release_create(); + create.await.unwrap().unwrap(); + other + .restart_sandbox_runtime("sb-restart-pending") + .await + .unwrap(); + assert_eq!(driver.stop_calls(), 1); + assert_eq!(driver.start_calls(), 1); + } + + #[tokio::test] + async fn actual_start_error_restores_authoritative_stopped_state() { + let driver = ControlledDriver::new(); + driver.set_start_outcome(ControlledLifecycleOutcome::Error("rejected start")); + let sandbox = sandbox_record("sb-rejected-start", "rejected-start", SandboxPhase::Stopped); + let mut stopped = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name()); + stopped.status = Some(make_driver_status(make_driver_condition( + "ContainerStopped", + "Stopped", + ))); + stopped + .status + .as_mut() + .unwrap() + .conditions + .push(DriverCondition { + r#type: "Suspended".into(), + status: "True".into(), + ..Default::default() + }); + driver.set_get_outcome(ControlledGetOutcome::Sandbox(Box::new(stopped))); + let runtime = test_runtime(driver).await; + runtime.store.put_message(&sandbox).await.unwrap(); + runtime + .start_sandbox("default", "rejected-start") + .await + .unwrap_err(); + let restored = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(restored.phase(), i32::from(SandboxPhase::Stopped)); + } + + #[tokio::test] + async fn automatic_restart_retains_owned_start_after_caller_cancellation() { + let driver = ControlledDriver::new(); + driver.block_start(); + let runtime = test_runtime(driver.clone()).await; + let mut other = runtime.clone(); + other.sync_lock = Arc::new(Mutex::new(())); + other.lifecycle_gates = Arc::new(LifecycleGateRegistry::default()); + let mut sandbox = sandbox_record("sb-auto-owned", "auto-owned", SandboxPhase::Ready); + sandbox + .spec + .get_or_insert_with(SandboxSpec::default) + .restart_policy = SandboxRestartPolicy::OnFailure as i32; + let status = sandbox.status.as_mut().unwrap(); + status.main_process_instance_id = "instance-1".into(); + status.provisioning = Some(provisioning_deadline::new_record( + openshell_core::time::now_ms(), + )); + runtime.store.put_message(&sandbox).await.unwrap(); + runtime + .main_process_exited(sandbox.object_id(), "instance-1", 9) + .await + .unwrap(); + runtime + .finalize_main_process_exit(sandbox.object_id(), "instance-1") + .await + .unwrap(); + let restarting = runtime.clone(); + let restart = + tokio::spawn(async move { restarting.restart_sandbox_runtime("sb-auto-owned").await }); + driver.start_started.notified().await; + let owned = wait_driver_pending(&other, sandbox.object_id(), true).await; + restart.abort(); + assert!(restart.await.unwrap_err().is_cancelled()); + let error = other + .await_provisioning_operation(&owned, async { Ok(()) }) + .await + .unwrap_err(); + assert_eq!(Status::from(error).code(), Code::FailedPrecondition); + other + .restart_sandbox_runtime(sandbox.object_id()) + .await + .unwrap(); + assert_eq!(driver.stop_calls(), 1); + assert_eq!(driver.start_calls(), 1); + driver.release_start(); + let settled = wait_driver_pending(&other, sandbox.object_id(), false).await; + assert!(provisioning_operation::same_operation(&owned, &settled)); + } + + #[tokio::test] + async fn start_settlement_cannot_restore_over_timeout() { + for failure in [false, true] { + let driver = ControlledDriver::new(); + driver.block_start(); + if failure { + driver.set_start_outcome(ControlledLifecycleOutcome::Error("late rejection")); + } + let runtime = test_runtime(driver.clone()).await; + let sandbox = + sandbox_record("sb-start-timeout", "start-timeout", SandboxPhase::Stopped); + runtime.store.put_message(&sandbox).await.unwrap(); + let starting = runtime.clone(); + let start = + tokio::spawn( + async move { starting.start_sandbox("default", "start-timeout").await }, + ); + driver.start_started.notified().await; + let pending = wait_driver_pending(&runtime, sandbox.object_id(), true).await; + let record = pending + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap(); + let deadline = + openshell_core::time::timestamp_to_millis(record.deadline.as_ref().unwrap()) + .unwrap(); + runtime + .reconcile_provisioning_deadlines(deadline) + .await + .unwrap(); + driver.release_start(); + assert_eq!( + start.await.unwrap().unwrap_err().code(), + Code::DeadlineExceeded + ); + let retained = wait_driver_pending(&runtime, sandbox.object_id(), false).await; + assert!(provisioning_deadline::timed_out(&retained)); + assert_eq!(retained.phase(), i32::from(SandboxPhase::Error)); + assert!(provisioning_operation::same_operation(&pending, &retained)); + } + } + + #[tokio::test] + async fn stale_start_compensation_and_rollback_cannot_reuse_operation_identity() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let mut previous = sandbox_record("sb-start-owner", "start-owner", SandboxPhase::Stopped); + let mut record = + provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 1800); + record.driver_operation_id = "old-operation".into(); + previous.status.as_mut().unwrap().provisioning = Some(record); + runtime.store.put_message(&previous).await.unwrap(); + let current = runtime + .store + .get_message::(previous.object_id()) + .await + .unwrap() + .unwrap(); + let ((), old_settled) = runtime + .await_provisioning_operation(¤t, async { Ok(()) }) + .await + .unwrap(); + let ((), settled) = runtime + .await_provisioning_operation(&old_settled, async { Ok(()) }) + .await + .unwrap(); + let gate = runtime.lifecycle_gates.lock_for(previous.object_id()).await; + let error = runtime + .compensate_successful_start( + &gate, + &old_settled, + &previous, + Status::internal("missing binding"), + ) + .await; + assert_eq!(error.code(), Code::Aborted); + assert_eq!( + driver.stop_calls(), + 0, + "stale compensation cannot reach backend" + ); + assert!( + runtime + .restore_lifecycle_snapshot(&settled, &previous) + .await + ); + let restored = runtime + .store + .get_message::(previous.object_id()) + .await + .unwrap() + .unwrap(); + assert!(provisioning_operation::same_operation(&restored, &settled)); + assert!(!provisioning_operation::same_operation( + &restored, &previous + )); + assert!( + !runtime + .restore_lifecycle_snapshot(&old_settled, &previous) + .await + ); + } + + #[tokio::test] + async fn failed_create_compensation_cannot_delete_without_store_ownership() { + for generation_changed in [false, true] { + let driver = ControlledDriver::new(); + let directory = tempfile::tempdir().unwrap(); + let database_url = + format!("sqlite://{}", directory.path().join("gateway.db").display()); + let mut runtime = test_runtime(driver.clone()).await; + runtime.store = Arc::new(Store::connect(&database_url).await.unwrap()); + let pool = sqlx::SqlitePool::connect(&database_url).await.unwrap(); + let mut sandbox = sandbox_record( + "sb-cleanup-owner", + "cleanup-owner", + SandboxPhase::Provisioning, + ); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_record(openshell_core::time::now_ms()), + ); + runtime.store.put_message(&sandbox).await.unwrap(); + let owned = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + let retained = if generation_changed { + runtime + .store + .update_message_cas::( + sandbox.object_id(), + sandbox_resource_version(&owned), + |current| { + current.metadata.as_mut().unwrap().annotations.insert( + crate::auth::sandbox_session::RUNTIME_GENERATION_ANNOTATION.into(), + "replacement-generation".into(), + ); + }, + ) + .await + .unwrap() + } else { + sqlx::query("CREATE TRIGGER reject_cleanup_claim BEFORE UPDATE ON objects BEGIN SELECT RAISE(ABORT, 'transient store failure'); END").execute(&pool).await.unwrap(); + owned.clone() + }; + let gate = runtime.lifecycle_gates.lock_for(sandbox.object_id()).await; + let global = runtime.lock_global_for_lifecycle(&gate).await; + let error = runtime + .compensate_failed_create(&owned, gate, global, Status::internal("missing binding")) + .await; + if generation_changed { + assert_eq!(error.code(), Code::Aborted); + } else { + assert!( + error + .message() + .contains("could not claim the sandbox record") + ); + } + assert_eq!( + driver.delete_calls(), + 0, + "unknown or changed ownership forbids backend cleanup" + ); + assert_eq!( + runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(), + retained + ); + } + } + + #[tokio::test] + async fn automatic_restart_rejects_generation_change_during_stop() { + let driver = ControlledDriver::new(); + driver.block_stop(); + let runtime = test_runtime(driver.clone()).await; + let mut sandbox = + sandbox_record("sb-auto-generation", "auto-generation", SandboxPhase::Ready); + sandbox + .spec + .get_or_insert_with(SandboxSpec::default) + .restart_policy = SandboxRestartPolicy::OnFailure as i32; + let status = sandbox.status.as_mut().unwrap(); + status.main_process_instance_id = "instance-1".into(); + status.provisioning = Some(provisioning_deadline::new_record( + openshell_core::time::now_ms(), + )); + runtime.store.put_message(&sandbox).await.unwrap(); + runtime + .main_process_exited(sandbox.object_id(), "instance-1", 9) + .await + .unwrap(); + runtime + .finalize_main_process_exit(sandbox.object_id(), "instance-1") + .await + .unwrap(); + let restarting = runtime.clone(); + let restart = tokio::spawn(async move { + restarting + .restart_sandbox_runtime("sb-auto-generation") + .await + }); + driver.stop_started.notified().await; + let claimed = wait_driver_pending(&runtime, sandbox.object_id(), true).await; + runtime + .store + .update_message_cas::( + sandbox.object_id(), + sandbox_resource_version(&claimed), + |current| { + current.metadata.as_mut().unwrap().annotations.insert( + crate::auth::sandbox_session::RUNTIME_GENERATION_ANNOTATION.into(), + "replacement-generation".into(), + ); + }, + ) + .await + .unwrap(); + driver.release_stop(); + assert!(restart.await.unwrap().is_err()); + let settled = wait_driver_pending(&runtime, sandbox.object_id(), false).await; + assert_eq!( + driver.start_calls(), + 0, + "watchdog cannot authorize a replacement generation" + ); + assert!(provisioning_operation::same_operation(&claimed, &settled)); + assert_eq!( + sandbox_runtime_generation(&settled).unwrap().as_str(), + "replacement-generation" + ); + } + + #[tokio::test] + async fn transient_create_monitor_failure_retains_record_after_driver_response() { + let driver = ControlledDriver::new(); + driver.block_create(); + *driver.create_error.lock().unwrap() = + Some(Status::failed_precondition("ordinary rejection")); + let directory = tempfile::tempdir().unwrap(); + let database_url = format!("sqlite://{}", directory.path().join("gateway.db").display()); + let mut runtime = test_runtime(driver.clone()).await; + runtime.store = Arc::new(Store::connect(&database_url).await.unwrap()); + let pool = sqlx::SqlitePool::connect(&database_url).await.unwrap(); + let now = openshell_core::time::now_ms(); + let mut sandbox = sandbox_record("sb-monitor", "monitor", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record(now, 60)); + let creating = runtime.clone(); + let create = + tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); + driver.create_started.notified().await; + let held = runtime.sync_lock.lock().await; + sqlx::query("ALTER TABLE objects RENAME TO temporarily_hidden_objects") + .execute(&pool) + .await + .unwrap(); + // The one-second monitor must observe a real transient SELECT error. + // The held lock prevents failed-create handling until the table returns. + tokio::time::sleep(Duration::from_millis(1500)).await; + sqlx::query("ALTER TABLE temporarily_hidden_objects RENAME TO objects") + .execute(&pool) + .await + .unwrap(); + driver.release_create(); + wait_driver_pending(&runtime, "sb-monitor", false).await; + drop(held); + let result = tokio::time::timeout(Duration::from_secs(3), create) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert!( + result.message().starts_with("monitor provisioning:"), + "{result}" + ); + let retained = runtime + .store + .get_message::("sb-monitor") + .await + .unwrap() + .expect("monitor interruption must not delete the record"); + assert!(!provisioning_deadline::driver_operation_pending(&retained)); + assert_eq!(retained.phase(), i32::from(SandboxPhase::Provisioning)); + runtime + .reconcile_provisioning_deadlines(now + 60_000) + .await + .unwrap(); + driver.stop_finished.notified().await; + assert_eq!(driver.delete_calls(), 0); + } + + #[tokio::test] + async fn canceled_create_retains_upload_and_cannot_be_redispatched() { + let driver = ControlledDriver::new(); + driver.block_create(); + let mut runtime = test_runtime(driver.clone()).await; + let now = openshell_core::time::now_ms(); + let mut sandbox = sandbox_record("sb-owned", "owned", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record(now, 60)); + let root = tempfile::tempdir().unwrap(); + let upload = stage_create_test_upload(&mut runtime, &mut sandbox, root.path()); + let creating = runtime.clone(); + let create = + tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); + driver.create_started.notified().await; + create.abort(); + let _ = create.await; + let owned = wait_driver_pending(&runtime, "sb-owned", true).await; + assert!(upload.exists()); + let polled = Arc::new(AtomicBool::new(false)); + let observed = polled.clone(); + let error = runtime + .await_provisioning_operation(&owned, async move { + observed.store(true, Ordering::SeqCst); + Ok(()) + }) + .await + .unwrap_err(); + assert_eq!(Status::from(error).code(), Code::FailedPrecondition); + assert!( + !polled.load(Ordering::SeqCst), + "second owner must not dispatch" + ); + let mut restarted = runtime.clone(); + restarted.lifecycle_gates = Arc::new(LifecycleGateRegistry::default()); + restarted.sync_lock = Arc::new(Mutex::new(())); + restarted.driver_info.gateway_manages_lifecycle = true; + restarted + .start_persisted_sandboxes_with_authentication( + |_| async { panic!("pending recovery must not rotate authentication") }, + |_| async { Ok(()) }, + |_| panic!("pending recovery must not revoke authentication"), + ) + .await + .unwrap(); + assert_eq!(driver.start_calls(), 0); + let record = restarted + .store + .get(Sandbox::object_type(), "sb-owned") + .await + .unwrap() + .unwrap(); + restarted + .prune_missing_sandbox(record, openshell_core::time::now_ms(), 0) + .await + .unwrap(); + assert!( + restarted + .store + .get_message::("sb-owned") + .await + .unwrap() + .is_some() + ); + assert_eq!(driver.delete_calls(), 0); + assert_eq!( + restarted + .delete_sandbox("default", "owned") + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + driver.release_create(); + wait_driver_pending(&runtime, "sb-owned", false).await; + assert!(upload.exists(), "successful driver owns the archive"); + } + + #[tokio::test] + async fn actual_create_rejection_clears_pending_and_preserves_error_cleanup() { + let driver = ControlledDriver::new(); + *driver.create_error.lock().unwrap() = + Some(Status::failed_precondition("volume no longer exists")); + let mut runtime = test_runtime(driver).await; + let mut sandbox = sandbox_record("sb-rejected", "rejected", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 60), + ); + let root = tempfile::tempdir().unwrap(); + let upload = stage_create_test_upload(&mut runtime, &mut sandbox, root.path()); + let error = runtime + .create_sandbox(sandbox, None, false) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + assert!( + runtime + .store + .get_message::("sb-rejected") + .await + .unwrap() + .is_none() + ); + tokio::time::timeout(Duration::from_secs(1), async { + while upload.exists() { + tokio::task::yield_now().await; + } + }) + .await + .expect("actual rejection cleans the upload"); + } + + #[tokio::test] + async fn concurrent_provisioning_claims_dispatch_only_one_driver_operation() { + let runtime = test_runtime(ControlledDriver::new()).await; + let sandbox = seed_provisioning_attempt(&runtime).await; + let dispatched = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(Semaphore::new(0)); + let spawn = || { + let runtime = runtime.clone(); + let sandbox = sandbox.clone(); + let dispatched = dispatched.clone(); + let release = release.clone(); + tokio::spawn(async move { + runtime + .await_provisioning_operation(&sandbox, async move { + dispatched.fetch_add(1, Ordering::SeqCst); + release.acquire().await.unwrap().forget(); + Ok(()) + }) + .await + }) + }; + let mut first = spawn(); + let mut second = spawn(); + let (loser, winner) = tokio::time::timeout(Duration::from_secs(3), async { + tokio::select! { + result = &mut first => (result, second), + result = &mut second => (result, first), + } + }) + .await + .unwrap(); + assert_eq!( + Status::from(loser.unwrap().unwrap_err()).code(), + Code::FailedPrecondition + ); + assert_eq!(dispatched.load(Ordering::SeqCst), 1); + release.add_permits(1); + assert!(winner.await.unwrap().is_ok()); + wait_driver_pending(&runtime, "sb-ttl", false).await; + } + + #[tokio::test] + async fn driver_completion_retries_transient_store_failure() { + let driver = ControlledDriver::new(); + driver.block_create(); + let directory = tempfile::tempdir().unwrap(); + let database_url = format!("sqlite://{}", directory.path().join("gateway.db").display()); + let mut runtime = test_runtime(driver.clone()).await; + runtime.store = Arc::new(Store::connect(&database_url).await.unwrap()); + let pool = sqlx::SqlitePool::connect(&database_url).await.unwrap(); + let mut sandbox = sandbox_record("sb-settlement", "settlement", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = Some( + provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 60), + ); + let creating = runtime.clone(); + let create = + tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); + driver.create_started.notified().await; + sqlx::query("CREATE TRIGGER reject_settlement BEFORE UPDATE ON objects BEGIN SELECT RAISE(ABORT, 'transient write failure'); END").execute(&pool).await.unwrap(); + sqlx::query("ALTER TABLE objects RENAME TO temporarily_hidden_objects") + .execute(&pool) + .await + .unwrap(); + driver.release_create(); + driver.create_finished.notified().await; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !create.is_finished(), + "owner retains the result after a settlement read failure" + ); + sqlx::query("ALTER TABLE temporarily_hidden_objects RENAME TO objects") + .execute(&pool) + .await + .unwrap(); + // The retry now reaches the injected write failure. Its result must + // remain owned across both failed reads and failed CAS persistence. + tokio::time::sleep(Duration::from_millis(1100)).await; + let pending = runtime + .store + .get_message::("sb-settlement") + .await + .unwrap() + .unwrap(); + assert!(provisioning_deadline::driver_operation_pending(&pending)); + assert!(!create.is_finished()); + sqlx::query("DROP TRIGGER reject_settlement") + .execute(&pool) + .await + .unwrap(); + assert!( + tokio::time::timeout(Duration::from_secs(3), create) + .await + .unwrap() + .unwrap() + .is_ok() + ); + let settled = runtime + .store + .get_message::("sb-settlement") + .await + .unwrap() + .unwrap(); + assert!(!provisioning_deadline::driver_operation_pending(&settled)); + } + + #[tokio::test] + async fn initial_create_preparation_expiry_releases_cleanup_gate() { + let driver = ControlledDriver::new(); + driver.block_create(); + let mut runtime = test_runtime(driver.clone()).await; + let now = openshell_core::time::now_ms(); + let preparation = provisioning_deadline::new_preparation_record(now, 1); + let mut sandbox = sandbox_record("sb-create-ttl", "create-ttl", SandboxPhase::Provisioning); + sandbox.status.as_mut().unwrap().provisioning = Some(preparation.clone()); + let upload_root = tempfile::tempdir().unwrap(); + let upload = stage_create_test_upload(&mut runtime, &mut sandbox, upload_root.path()); + let creating_runtime = runtime.clone(); + let mut create = + tokio::spawn( + async move { creating_runtime.create_sandbox(sandbox, None, false).await }, + ); + tokio::time::timeout(Duration::from_secs(1), driver.create_started.notified()) + .await + .unwrap(); + + runtime + .reconcile_provisioning_deadlines(now + 1_000) + .await + .unwrap(); + assert_eq!( + driver.stop_calls(), + 0, + "create still owns the lifecycle gate" + ); + let result = if let Ok(result) = + tokio::time::timeout(Duration::from_secs(3), &mut create).await + { + result.unwrap() + } else { + create.abort(); + let _ = create.await; + panic!("expired initial create kept the lifecycle gate while the driver was blocked"); + }; + assert_eq!(result.unwrap_err().code(), Code::DeadlineExceeded); + assert!(upload.exists(), "owned create still needs the upload"); + runtime + .reconcile_provisioning_deadlines(now + 1_001) + .await + .unwrap(); + driver.stop_finished.notified().await; + let cleanup_finished = runtime.lifecycle_gates.lock_for("sb-create-ttl").await; + drop(cleanup_finished); + let pending = wait_driver_pending(&runtime, "sb-create-ttl", true).await; + assert!( + pending + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap() + .cleanup_completed_time + .is_none() + ); + driver.release_create(); + let settled = wait_driver_pending(&runtime, "sb-create-ttl", false).await; + // The earlier STOP has returned and released its gate. Advance only + // its retry backoff; settlement must preserve any still-active lease. + runtime + .store + .update_message_cas::( + "sb-create-ttl", + sandbox_resource_version(&settled), + |sandbox| { + sandbox + .status + .as_mut() + .unwrap() + .provisioning + .as_mut() + .unwrap() + .cleanup_retry_time = None; + }, + ) + .await + .unwrap(); + runtime + .reconcile_provisioning_deadlines(now + 1_002) + .await + .unwrap(); + let retained = tokio::time::timeout(Duration::from_secs(1), async { + loop { + let sandbox = runtime + .store + .get_message::("sb-create-ttl") + .await + .unwrap() + .expect("retained timeout record"); + if sandbox + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap() + .cleanup_completed_time + .is_some() + { + break sandbox; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(driver.stop_calls(), 2); + assert_eq!(driver.delete_calls(), 0); + assert_eq!(retained.phase(), i32::from(SandboxPhase::Error)); + let status = retained.status.as_ref().unwrap(); + let record = status.provisioning.as_ref().unwrap(); + assert_eq!(record.attempt_id, preparation.attempt_id); + assert_eq!( + record.preparation_deadline, + preparation.preparation_deadline + ); + assert!( + status + .conditions + .iter() + .any(|c| c.reason == "ImagePreparationTimedOut") + ); + } + #[tokio::test] async fn failed_create_cleanup_preserves_timeout_after_cas_conflict() { let driver = ControlledDriver::new(); @@ -15653,10 +16733,11 @@ mod tests { } #[tokio::test] - async fn late_initial_create_preserves_retry_from_another_replica() { + async fn late_initial_create_requires_cleanup_after_owned_response() { let driver = ControlledDriver::new(); driver.block_create(); - *driver.create_error.lock().unwrap() = Some(Status::internal("late create failure")); + driver.block_stop(); + driver.track_compute.store(true, Ordering::SeqCst); let runtime = test_runtime(driver.clone()).await; let mut other = runtime.clone(); other.sync_lock = Arc::new(Mutex::new(())); @@ -15670,63 +16751,152 @@ mod tests { ); sandbox.status.as_mut().unwrap().provisioning = Some(provisioning_deadline::new_preparation_record(now, 1)); - let creating_runtime = runtime.clone(); + let original_attempt = sandbox_provisioning_attempt_id(&sandbox) + .unwrap() + .to_owned(); + let creating = runtime.clone(); let create = - tokio::spawn( - async move { creating_runtime.create_sandbox(sandbox, None, false).await }, - ); + tokio::spawn(async move { creating.create_sandbox(sandbox, None, false).await }); driver.create_started.notified().await; other .reconcile_provisioning_deadlines(now + 1_000) .await .unwrap(); - tokio::time::timeout(Duration::from_secs(2), async { - loop { - let current = other - .store - .get_message::("sb-create-retry") - .await - .unwrap() - .unwrap(); - if current - .status - .as_ref() - .unwrap() - .provisioning - .as_ref() - .unwrap() - .cleanup_completed_time - .is_some() - { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - // A's driver has completed, but only A's local lock prevents it from - // processing the result. B uses the same store with its own locks. - let held = runtime.sync_lock.lock().await; + driver.stop_started.notified().await; + assert_eq!( + tokio::time::timeout(Duration::from_secs(3), create) + .await + .unwrap() + .unwrap() + .unwrap_err() + .code(), + Code::DeadlineExceeded + ); + assert_eq!( + runtime + .start_sandbox("default", "create-retry") + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + assert_eq!( + runtime + .stop_sandbox("default", "create-retry") + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + assert_eq!( + runtime + .delete_sandbox("default", "create-retry") + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + runtime.apply_deleted("sb-create-retry").await.unwrap(); + assert!( + runtime + .store + .get_message::("sb-create-retry") + .await + .unwrap() + .is_some() + ); + + // STOP has observed absence. Keep its final reread blocked while the + // original CREATE succeeds and durably clears pending ownership. + let held = other.sync_lock.lock().await; + driver.release_stop(); + driver.stop_finished.notified().await; driver.release_create(); - driver.create_finished.notified().await; + let settled = wait_driver_pending(&runtime, "sb-create-retry", false).await; + assert!(driver.compute_exists.load(Ordering::SeqCst)); + assert_eq!( + sandbox_provisioning_attempt_id(&settled), + Some(original_attempt.as_str()) + ); + drop(held); + let gate = other.lifecycle_gates.lock_for("sb-create-retry").await; + let retained = other + .store + .get_message::("sb-create-retry") + .await + .unwrap() + .unwrap(); + let record = retained + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap(); + assert!( + record.cleanup_completed_time.is_none(), + "pre-settlement STOP cannot complete cleanup" + ); + assert_eq!( + runtime + .start_sandbox("default", "create-retry") + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + // Advance only the cleanup retry timer; preserve the original deadline. let retry = other - .start_sandbox("default", "create-retry") + .store + .update_message_cas::( + "sb-create-retry", + sandbox_resource_version(&retained), + |sandbox| { + sandbox + .status + .as_mut() + .unwrap() + .provisioning + .as_mut() + .unwrap() + .cleanup_retry_time = None; + }, + ) .await .unwrap(); - drop(held); - let result = tokio::time::timeout(Duration::from_secs(2), create) + driver.stop_blocked.store(false, Ordering::SeqCst); + other + .reclaim_provisioning_timeout(&retry, &gate) .await - .unwrap() .unwrap(); - assert_eq!(result.unwrap_err().code(), Code::Aborted); - let retained = runtime + drop(gate); + let reclaimed = other .store .get_message::("sb-create-retry") .await .unwrap() - .expect("new attempt must survive"); - assert_eq!(retained, retry); + .unwrap(); + assert!( + reclaimed + .status + .as_ref() + .unwrap() + .provisioning + .as_ref() + .unwrap() + .cleanup_completed_time + .is_some() + ); + assert!(!driver.compute_exists.load(Ordering::SeqCst)); + assert_eq!(driver.stop_calls(), 2); + let retry = other + .start_sandbox("default", "create-retry") + .await + .unwrap(); + assert_ne!( + sandbox_provisioning_attempt_id(&retry), + Some(original_attempt.as_str()) + ); assert_eq!(driver.delete_calls(), 0); } @@ -15978,6 +17148,9 @@ mod tests { let _ = recovery.await; panic!("expired recovery kept the lifecycle gate while the driver was blocked"); } + assert!(provisioning_deadline::driver_operation_pending(&expired)); + driver.release_start(); + wait_driver_pending(&runtime, "sb-recovery-ttl", false).await; runtime .reconcile_provisioning_deadlines(now + 1_800_001) .await @@ -16400,7 +17573,7 @@ mod tests { .await .expect("expired startup releases its lifecycle gate") .unwrap_err(); - assert_eq!(interrupted.code(), Code::DeadlineExceeded); + assert_eq!(Status::from(interrupted).code(), Code::DeadlineExceeded); runtime .mark_sandbox_error(&sandbox, "StartFailed", "late startup failure") .await; diff --git a/crates/openshell-server/src/compute/provisioning_deadline.rs b/crates/openshell-server/src/compute/provisioning_deadline.rs index 56cb421e94..1724e3c984 100644 --- a/crates/openshell-server/src/compute/provisioning_deadline.rs +++ b/crates/openshell-server/src/compute/provisioning_deadline.rs @@ -260,6 +260,15 @@ pub fn timed_out(sandbox: &openshell_core::proto::Sandbox) -> bool { .is_some_and(|record| record.timeout_time.is_some()) } +/// A submitted operation still owns the possibility of a later backend commit. +pub(super) fn driver_operation_pending(sandbox: &openshell_core::proto::Sandbox) -> bool { + sandbox + .status + .as_ref() + .and_then(|status| status.provisioning.as_ref()) + .is_some_and(|record| record.driver_operation_pending) +} + /// Adopt an existing untimed attempt without granting it a new preparation phase. /// New create/start operations use `new_preparation_record` instead. pub fn new_record(now_ms: i64) -> SandboxProvisioning { @@ -647,6 +656,10 @@ impl super::ComputeRuntime { if record.timeout_time.is_none() || record.cleanup_completed_time.is_some() { return Ok(()); } + // A stop can observe NotFound before an in-flight create materializes + // compute. Even if that create finishes before the final read below, + // only a later stop issued after settlement can complete cleanup. + let pending_before_stop = record.driver_operation_pending; let now_ms = openshell_core::time::now_ms(); if record .cleanup_retry_time @@ -700,8 +713,9 @@ impl super::ComputeRuntime { ), ) .await; - let reclaimed = matches!(&result, Ok(Ok(_))) - || matches!(&result, Ok(Err(error)) if error.code() == tonic::Code::NotFound); + let reclaimed = !pending_before_stop + && (matches!(&result, Ok(Ok(_))) + || matches!(&result, Ok(Err(error)) if error.code() == tonic::Code::NotFound)); let _global_guard = self.lock_global_for_lifecycle(lifecycle_guard).await; let Some(current) = self .store @@ -721,6 +735,7 @@ impl super::ComputeRuntime { { return Ok(()); } + let reclaimed = reclaimed && !driver_operation_pending(¤t); let completed_at_ms = openshell_core::time::now_ms(); let updated = self .store @@ -768,6 +783,8 @@ mod tests { // configuration at epoch 0, and an admission deadline at 300 seconds. let bytes = [0x0a, 1, b'a', 0x12, 1, b'c', 0x1a, 0, 0x2a, 3, 8, 0xac, 2]; let mut record = SandboxProvisioning::decode(bytes.as_slice()).unwrap(); + assert!(!record.driver_operation_pending); + assert!(record.driver_operation_id.is_empty()); let before = record.clone(); record_admission_start(&mut record, 299_999).unwrap(); assert_eq!(record, before); @@ -798,6 +815,8 @@ mod tests { fn preparation_config_changes_and_restart_preserve_the_absolute_ceiling() { use prost::Message; let mut record = new_preparation_record(0, 1800); + record.driver_operation_pending = true; + record.driver_operation_id = "operation-1".into(); let mut timer = ProvisioningDeadline::from_record(&record).unwrap(); let attempt = record.attempt_id.clone(); assert!(timer.configuration_changed(&attempt, change("first", 600_000))); @@ -807,6 +826,8 @@ mod tests { timer.write_record(&mut record); let bytes = record.encode_to_vec(); let restored = SandboxProvisioning::decode(bytes.as_slice()).unwrap(); + assert!(restored.driver_operation_pending); + assert_eq!(restored.driver_operation_id, "operation-1"); let mut timer = ProvisioningDeadline::from_record(&restored).unwrap(); assert_eq!(timer.deadline_at_ms(), Some(1_800_000)); assert!(!timer.start_admission("previous-attempt", 1_799_999)); diff --git a/crates/openshell-server/src/compute/provisioning_operation.rs b/crates/openshell-server/src/compute/provisioning_operation.rs new file mode 100644 index 0000000000..457d219b6c --- /dev/null +++ b/crates/openshell-server/src/compute/provisioning_operation.rs @@ -0,0 +1,309 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Retain ownership of submitted driver work until its outcome is known. +//! +//! Local cancellation and backend absence cannot prove that an external create +//! stopped. A durable claim prevents another replica from declaring cleanup +//! complete, retrying, or deleting the only record of an unresolved request. + +use std::future::Future; +use std::time::Duration; + +use openshell_core::{ObjectId, proto::Sandbox}; +use tonic::Status; + +use super::{ + ComputeRuntime, provisioning_deadline, sandbox_provisioning_attempt_id, + sandbox_resource_version, sandbox_runtime_generation, +}; +use crate::persistence::PersistenceError; + +/// An attempt may contain several recovery operations on the same generation. +/// Keep the last operation ID after settlement so a delayed callback cannot +/// adopt a newer operation merely because it has already finished. +pub(super) fn same_operation(left: &Sandbox, right: &Sandbox) -> bool { + sandbox_provisioning_attempt_id(left) == sandbox_provisioning_attempt_id(right) + && operation_id(left) == operation_id(right) +} + +fn operation_id(sandbox: &Sandbox) -> Option<&str> { + sandbox + .status + .as_ref()? + .provisioning + .as_ref() + .map(|record| record.driver_operation_id.as_str()) +} + +/// Called only inside a CAS that has checked the prior operation is settled. +/// Legacy rows without an attempt retain their existing lifecycle contract. +pub(super) fn claim_record(sandbox: &mut Sandbox) { + if let Some(record) = sandbox + .status + .as_mut() + .and_then(|status| status.provisioning.as_mut()) + { + record.driver_operation_pending = true; + record.driver_operation_id = uuid::Uuid::new_v4().to_string(); + } +} + +pub(super) fn ensure_current_result(current: &Sandbox, owned: &Sandbox) -> Result<(), Status> { + if !same_operation(current, owned) + || sandbox_runtime_generation(current) != sandbox_runtime_generation(owned) + { + return Err(Status::aborted( + "sandbox driver operation changed before result handling", + )); + } + if provisioning_deadline::timed_out(current) { + return Err(Status::deadline_exceeded("provisioning deadline expired")); + } + ensure_operation_settled(current) +} + +pub(super) fn ensure_operation_settled(sandbox: &Sandbox) -> Result<(), Status> { + if provisioning_deadline::driver_operation_pending(sandbox) { + return Err(Status::failed_precondition( + "previous driver operation is still pending; retry and deletion are blocked", + )); + } + Ok(()) +} + +/// Keep monitor interruption separate from an actual driver response. Only +/// the latter may enter the caller's existing failure recovery path. +#[derive(Debug, thiserror::Error)] +pub(super) enum ProvisioningOperationError { + #[error("{status}")] + Driver { + status: Status, + settled: Box, + }, + #[error("{0}")] + Monitor(Status), + #[error("{0}")] + Unsettled(Status), +} + +impl From for Status { + fn from(error: ProvisioningOperationError) -> Self { + match error { + ProvisioningOperationError::Driver { status, .. } + | ProvisioningOperationError::Monitor(status) + | ProvisioningOperationError::Unsettled(status) => status, + } + } +} + +impl ComputeRuntime { + /// Claim before polling the operation, then retain its task even when the + /// caller or deadline monitor leaves. Pending describes the owned future, + /// not backend cancellation: transport-error ambiguity remains governed by + /// the compute driver's existing error contract. + pub(super) async fn await_provisioning_operation( + &self, + starting: &Sandbox, + operation: impl Future> + Send + 'static, + ) -> Result<(T, Box), ProvisioningOperationError> { + let claimed = self + .claim_provisioning_operation(starting) + .await + .map_err(ProvisioningOperationError::Monitor)?; + self.await_claimed_provisioning_operation(&claimed, operation) + .await + } + + /// The caller has atomically claimed both its lifecycle transition and the + /// operation. The detached worker owns I/O, not the caller's local gate. + pub(super) async fn await_claimed_provisioning_operation( + &self, + starting: &Sandbox, + operation: impl Future> + Send + 'static, + ) -> Result<(T, Box), ProvisioningOperationError> { + let tracked = sandbox_provisioning_attempt_id(starting).is_some(); + let runtime = self.clone(); + let owned_attempt = starting.clone(); + // Dropping a JoinHandle detaches its task. Do not abort it on a monitor + // error or deadline: the remote side may still commit the request. + let mut worker = tokio::spawn(async move { + let result = operation.await; + let settled = if tracked { + let settled = runtime + .settle_provisioning_operation(&owned_attempt) + .await + .map_err(ProvisioningOperationError::Monitor)?; + // Settlement records a known response even if another writer + // rotated identity. That response cannot bind or compensate + // against the replacement generation in its returned snapshot. + if sandbox_runtime_generation(&settled) + != sandbox_runtime_generation(&owned_attempt) + { + return Err(ProvisioningOperationError::Monitor(Status::aborted( + "sandbox runtime generation changed during driver operation", + ))); + } + settled + } else { + owned_attempt + }; + match result { + Ok(response) => Ok((response, Box::new(settled))), + Err(status) => Err(ProvisioningOperationError::Driver { + status, + settled: Box::new(settled), + }), + } + }); + loop { + tokio::select! { + result = &mut worker => return result.map_err(|error| { + ProvisioningOperationError::Unsettled(Status::internal(format!( + "driver operation owner terminated; outcome remains unknown: {error}" + ))) + })?, + () = tokio::time::sleep(Duration::from_secs(1)) => { + let current = self.store.get_message::(starting.object_id()) + .await.map_err(|error| ProvisioningOperationError::Monitor( + Status::internal(format!("monitor provisioning: {error}")) + ))? + .ok_or_else(|| ProvisioningOperationError::Monitor( + Status::not_found("sandbox removed during startup") + ))?; + if !same_operation(¤t, starting) + || sandbox_runtime_generation(¤t) != sandbox_runtime_generation(starting) { + return Err(ProvisioningOperationError::Monitor(Status::aborted( + "sandbox provisioning attempt changed while waiting for compute" + ))); + } + if provisioning_deadline::timed_out(¤t) { + return Err(ProvisioningOperationError::Monitor(Status::deadline_exceeded( + "provisioning deadline expired; driver settlement may still be pending" + ))); + } + } + } + } + } + + async fn claim_provisioning_operation(&self, starting: &Sandbox) -> Result { + let Some(attempt_id) = sandbox_provisioning_attempt_id(starting) else { + // Legacy rows without an attempt do not gain a synthetic deadline. + return Ok(starting.clone()); + }; + if attempt_id.is_empty() { + return Err(Status::failed_precondition("provisioning attempt is empty")); + } + for _ in 0..super::START_PHASE_CAS_RETRY_LIMIT { + let current = self + .store + .get_message::(starting.object_id()) + .await + .map_err(|error| Status::internal(error.to_string()))? + .ok_or_else(|| Status::not_found("sandbox removed before driver dispatch"))?; + if provisioning_deadline::timed_out(¤t) { + return Err(Status::deadline_exceeded( + "provisioning deadline already expired", + )); + } + if provisioning_deadline::driver_operation_pending(¤t) { + return Err(Status::failed_precondition( + "previous driver operation may still complete; retry and deletion are blocked", + )); + } + if !same_operation(¤t, starting) + || sandbox_runtime_generation(¤t) != sandbox_runtime_generation(starting) + { + return Err(Status::aborted( + "provisioning operation changed before driver dispatch", + )); + } + if current.phase() != starting.phase() { + return Err(Status::aborted( + "sandbox phase changed before driver dispatch", + )); + } + match self + .store + .update_message_cas::( + starting.object_id(), + sandbox_resource_version(¤t), + claim_record, + ) + .await + { + Ok(updated) => { + self.sandbox_index.update_from_sandbox(&updated); + self.sandbox_watch_bus.notify(starting.object_id()); + return Ok(updated); + } + Err(PersistenceError::Conflict { .. }) => {} + Err(error) => return Err(Status::internal(error.to_string())), + } + } + Err(Status::aborted( + "sandbox kept changing before driver dispatch", + )) + } + + async fn settle_provisioning_operation(&self, starting: &Sandbox) -> Result { + // Keep the observed response in this task across transient persistence + // failures. A process crash still leaves the durable claim intact; + // neither lease expiry nor backend absence can safely clear it. + loop { + let result = self.try_settle_provisioning_operation(starting).await; + match result { + Ok(settled) => return Ok(settled), + Err(error) if error.code() == tonic::Code::Aborted => return Err(error), + Err(error) => { + tracing::warn!(sandbox_id = starting.object_id(), %error, + "Retaining driver response until operation ownership can be persisted"); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + } + + async fn try_settle_provisioning_operation( + &self, + starting: &Sandbox, + ) -> Result { + let current = self + .store + .get_message::(starting.object_id()) + .await + .map_err(|error| Status::internal(error.to_string()))? + .ok_or_else(|| Status::aborted("sandbox removed before driver settlement"))?; + if !same_operation(¤t, starting) { + return Err(Status::aborted( + "provisioning attempt changed before driver settlement", + )); + } + let updated = self + .store + .update_message_cas::( + starting.object_id(), + sandbox_resource_version(¤t), + |sandbox| { + if let Some(record) = sandbox + .status + .as_mut() + .and_then(|status| status.provisioning.as_mut()) + { + record.driver_operation_pending = false; + // Any stop before this response may have seen absence before + // the late create committed. Require another cleanup pass. + record.cleanup_completed_time = None; + // The retry timestamp may be another replica's active + // STOP lease. Preserve it through driver settlement. + } + }, + ) + .await + .map_err(|error| Status::internal(error.to_string()))?; + self.sandbox_index.update_from_sandbox(&updated); + self.sandbox_watch_bus.notify(starting.object_id()); + Ok(updated) + } +} diff --git a/crates/openshell-server/src/compute/rootfs_tar.rs b/crates/openshell-server/src/compute/rootfs_tar.rs index 2747d57374..544cdcafec 100644 --- a/crates/openshell-server/src/compute/rootfs_tar.rs +++ b/crates/openshell-server/src/compute/rootfs_tar.rs @@ -32,6 +32,9 @@ const MAX_SLOTS_PER_CALLER: usize = 4; /// let one caller starve everyone else. const MAX_TOTAL_SLOTS: usize = 64; const STAGING_DIR_PREFIX: &str = "req-"; +// A driver request can outlive its gateway. Age alone never proves that such +// a request stopped reading its input. +const PENDING_DRIVER_MARKER: &str = ".driver-request-pending"; const MAX_STAGED_FILE_NAME_LEN: usize = 128; /// `driver_config.` key the CLI sets to redeem a staging slot. @@ -86,6 +89,35 @@ impl StagedRootfsTar { pub fn disarm(&mut self) { self.dir = None; } + + /// Protect this input from the orphan sweep before dispatch. The armed + /// guard still removes it if dispatch is rejected before the driver runs. + pub async fn prepare_dispatch(&self) -> Result<(), Status> { + let directory = self + .dir + .as_ref() + .ok_or_else(|| Status::internal("rootfs upload ownership was already transferred"))?; + tokio::fs::write(directory.join(PENDING_DRIVER_MARKER), b"pending\n") + .await + .map_err(|error| Status::internal(format!("protect rootfs upload: {error}"))) + } + + /// Restore ordinary error cleanup after the owned driver future returns. + /// Successful responses leave cleanup with the driver; an interrupted + /// owner leaves the marker and archive for explicit reconciliation. + pub async fn finish_driver_operation(&mut self, accepted: bool) { + let Some(directory) = self.path.parent() else { + return; + }; + if let Err(error) = tokio::fs::remove_file(directory.join(PENDING_DRIVER_MARKER)).await + && error.kind() != std::io::ErrorKind::NotFound + { + warn!(%error, "Rootfs upload remains protected after driver response"); + } + if !accepted { + self.dir = Some(directory.to_path_buf()); + } + } } impl Drop for StagedRootfsTar { @@ -303,6 +335,12 @@ impl RootfsTarStagingRegistry { if !name.starts_with(STAGING_DIR_PREFIX) { continue; } + // Preserve uncertain ownership across gateway restarts. A missing + // response or old mtime is not permission to remove a live input. + match std::fs::symlink_metadata(entry.path().join(PENDING_DRIVER_MARKER)) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) | Err(_) => continue, + } let stale = entry .metadata() .and_then(|meta| meta.modified()) @@ -690,4 +728,31 @@ mod tests { assert!(!fresh.exists()); assert!(unrelated.exists(), "unrelated entries must be left alone"); } + + #[tokio::test] + async fn orphan_sweep_preserves_dispatched_upload_after_owner_loss() { + let root = temp_root(); + let registry = RootfsTarStagingRegistry::new(Some(root.path().to_path_buf()), 1024); + let slot = registry.begin("default", "test", "rootfs.tar", 7).unwrap(); + std::fs::write(&slot.upload_path, b"archive").unwrap(); + let mut staged = registry.consume(&slot.token).unwrap(); + staged.prepare_dispatch().await.unwrap(); + staged.disarm(); + let restarted = RootfsTarStagingRegistry::with_ttl( + Some(root.path().to_path_buf()), + 1024, + Duration::ZERO, + ); + restarted.sweep_orphans(); + assert!( + slot.upload_path.exists(), + "age cannot settle an owned driver request" + ); + staged.finish_driver_operation(true).await; + restarted.sweep_orphans(); + assert!( + !slot.upload_path.exists(), + "settled uploads follow normal sweep rules" + ); + } } diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index f92d6b1e5d..fb99042d63 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -131,10 +131,13 @@ mod tests { // no stored attempt gains another phase or time budget on upgrade. // Service authorization also extends both schemas additively. Legacy // payloads retain the safe Strip default. + // Driver-operation ownership adds pending and a retained operation ID to + // SandboxProvisioning in both closures. Old rows decode false and empty; + // decoding or deadline updates cannot claim an existing attempt. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "581125d215f2a1967eb73826c411c1e72a53fb3a30a20e85c51d96bbb518e078"; + "18206c52e68fdb0af60f8bb8dfaf47d9bc8021222cb49cacffab6352d3ad5549"; const DURABLE_SCHEMA_SHA256: &str = - "c1e49c80c52a5c7458952b333e7ca9da2dd24478b41756b710ba29a5217b67d7"; + "76487ab369fc3a4b03a179bb5e7ea6be8d20e380ad5563075dff8ee50e539406"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = "761dea31a521b0650840fe2a823ad6e36a265ed323ba4506889781d630df0ee3"; // A persisted Sandbox without endpoint status retains its lifecycle fields; diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index d7ea7fecfb..6078598775 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -945,6 +945,10 @@ The first authenticated configuration report from the current supervisor ends pr Preparation expiry sets the sandbox to `Error` with reason `ImagePreparationTimedOut`; admission repair expiry uses `ProvisioningTimedOut`. The gateway stops its workload and supervisor compute, retaining the sandbox record, diagnostic, and restartable storage. Cleanup can remain pending if the backend is unavailable; the gateway retries it. Inspect `provisioning` in JSON output for the active `phase`, `deadline`, original `preparation_deadline`, `admission_start_time`, and cleanup timestamps. TUI NOTES identifies preparation timeout and distinguishes pending cleanup from reclaimed compute. +The gateway keeps each submitted create or start operation running after its caller disconnects, its deadline expires, or monitoring fails. This includes recovery and the stop/start sequence for automatic restart. While `driver_operation_pending` is true, explicit start, stop, and deletion are blocked; automatic restart keeps its schedule and waits. Timeout cleanup can try to stop partial compute, but it cannot declare completion until the original driver call returns and a subsequent stop succeeds. This prevents a stop that observed missing compute from allowing a retry just before the original create finishes. Uploaded rootfs archives remain available to the owned operation. + +An actual driver response clears the pending flag, including ordinary rejection errors. The gateway retains `driver_operation_id` so a delayed response handler cannot change newer recovery work on the same provisioning attempt. Both fields appear in the `provisioning` JSON object. Backend behavior after a transport error still follows the driver's existing contract. If the owning gateway crashes before recording the response, the flag and any protected upload can remain indefinitely; restart and age alone cannot establish that the operation finished. There is currently no API to resolve that lost ownership, so automatic cleanup completion, retry, and deletion remain blocked for that record. + For `ImagePreparationTimedOut`, inspect image preparation and supervisor startup diagnostics and check whether the configured preparation budget is sufficient. For `ProvisioningTimedOut`, repair the rejected configuration. In either case, wait for cleanup to complete, then explicitly restart: ```shell diff --git a/proto/openshell.proto b/proto/openshell.proto index 356d0baacc..3d9cedad67 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -3789,6 +3789,16 @@ message SandboxProvisioning { // Starts the admission repair window. Absent while preparing, including // after a preparation timeout. Duplicate reports never change this value. google.protobuf.Timestamp admission_start_time = 13; + // An owned driver create/start future is still pending. Claimed before + // dispatch and cleared on its actual success or error response. Monitor + // interruption, owner loss, and StopSandbox/NotFound do not clear it. + // While set, cleanup cannot complete and retry/deletion remain blocked. + // This does not strengthen a driver's transport-error settlement contract. + bool driver_operation_pending = 14; + // Unique for each claimed driver operation, including recovery and automatic + // restart. Retained after settlement to reject delayed callbacks from an + // earlier operation on the same attempt and runtime generation. + string driver_operation_id = 15; } // Create-time request to expose one loopback HTTP service in a sandbox. diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 16d62d565b..9de9c81318 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -17657,8 +17657,18 @@ type SandboxProvisioning struct { // Starts the admission repair window. Absent while preparing, including // after a preparation timeout. Duplicate reports never change this value. AdmissionStartTime *timestamppb.Timestamp `protobuf:"bytes,13,opt,name=admission_start_time,json=admissionStartTime,proto3" json:"admission_start_time,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // An owned driver create/start future is still pending. Claimed before + // dispatch and cleared on its actual success or error response. Monitor + // interruption, owner loss, and StopSandbox/NotFound do not clear it. + // While set, cleanup cannot complete and retry/deletion remain blocked. + // This does not strengthen a driver's transport-error settlement contract. + DriverOperationPending bool `protobuf:"varint,14,opt,name=driver_operation_pending,json=driverOperationPending,proto3" json:"driver_operation_pending,omitempty"` + // Unique for each claimed driver operation, including recovery and automatic + // restart. Retained after settlement to reject delayed callbacks from an + // earlier operation on the same attempt and runtime generation. + DriverOperationId string `protobuf:"bytes,15,opt,name=driver_operation_id,json=driverOperationId,proto3" json:"driver_operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SandboxProvisioning) Reset() { @@ -17782,6 +17792,20 @@ func (x *SandboxProvisioning) GetAdmissionStartTime() *timestamppb.Timestamp { return nil } +func (x *SandboxProvisioning) GetDriverOperationPending() bool { + if x != nil { + return x.DriverOperationPending + } + return false +} + +func (x *SandboxProvisioning) GetDriverOperationId() string { + if x != nil { + return x.DriverOperationId + } + return "" +} + // Create-time request to expose one loopback HTTP service in a sandbox. type SandboxServiceExposure struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -19163,7 +19187,7 @@ const file_openshell_proto_rawDesc = "" + "\x04path\x18\x04 \x01(\tR\x04path\x12=\n" + "\vlast_result\x18\x05 \x01(\x0e2\x1c.openshell.v1.EndpointResultR\n" + "lastResult\x12H\n" + - "\x12last_reported_time\x18j \x01(\v2\x1a.google.protobuf.TimestampR\x10lastReportedTimeJ\x04\b\x06\x10\aR\x10last_reported_at\"\xeb\x06\n" + + "\x12last_reported_time\x18j \x01(\v2\x1a.google.protobuf.TimestampR\x10lastReportedTimeJ\x04\b\x06\x10\aR\x10last_reported_at\"\xd5\a\n" + "\x13SandboxProvisioning\x12\x1d\n" + "\n" + "attempt_id\x18\x01 \x01(\tR\tattemptId\x126\n" + @@ -19179,7 +19203,9 @@ const file_openshell_proto_rawDesc = "" + " \x01(\tR\x12attachmentChangeId\x12P\n" + "\x16attachment_change_time\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\x14attachmentChangeTime\x12M\n" + "\x14preparation_deadline\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\x13preparationDeadline\x12L\n" + - "\x14admission_start_time\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\x12admissionStartTime\"\xaa\x01\n" + + "\x14admission_start_time\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\x12admissionStartTime\x128\n" + + "\x18driver_operation_pending\x18\x0e \x01(\bR\x16driverOperationPending\x12.\n" + + "\x13driver_operation_id\x18\x0f \x01(\tR\x11driverOperationId\"\xaa\x01\n" + "\x16SandboxServiceExposure\x12\x18\n" + "\aservice\x18\x01 \x01(\tR\aservice\x12\x1f\n" + "\vtarget_port\x18\x02 \x01(\rR\n" + From 97b404cea72a7656b59751c3e1e7543a387e3093 Mon Sep 17 00:00:00 2001 From: Shiju Date: Sat, 3 Oct 2026 13:29:33 +0530 Subject: [PATCH 7/7] fix(openshell): preserve tracing and accept ready create responses Carry the request span into the detached provisioning worker so compute driver calls remain attached to their parent trace after task handoff. Update the compensation regression to require no backend DELETE when the durable cleanup claim fails, matching the operation ownership requirement. Accept the gateway's current Ready snapshot when a sandbox becomes ready before CREATE returns. Do not require the client to observe an earlier provisioning phase. Cover the Ready-only watch and command attachment. Signed-off-by: Shiju --- crates/openshell-cli/src/run.rs | 20 ++--- .../sandbox_create_lifecycle_integration.rs | 78 ++++++++++++++++++- crates/openshell-server/src/compute/mod.rs | 14 ++-- .../src/compute/provisioning_operation.rs | 8 +- 4 files changed, 94 insertions(+), 26 deletions(-) diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 353af3c4e9..dcb5e2f880 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -800,11 +800,8 @@ pub async fn sandbox_create( // Non-interactive mode: track start time for timestamps. let provision_start = Instant::now(); - // Don't use stop_on_terminal on the server — the Kubernetes CRD may - // briefly report a stale Ready status before the controller reconciles - // a newly created sandbox. Instead we handle termination client-side: - // we wait until we have observed at least one non-Ready phase followed - // by Ready (a genuine Provisioning → Ready transition). + // Handle terminal states here so a provisional container exit can wait + // for the supervisor's canonical-process result before cleanup. let sandbox_name = sandbox.object_name().to_string(); let sandbox_workspace = sandbox.object_workspace().to_string(); let mut stream = client @@ -832,8 +829,6 @@ pub async fn sandbox_create( let mut last_sandbox = sandbox.clone(); let mut last_error_reason = String::new(); let mut last_condition_message = ready_false_condition_message(sandbox.status.as_ref()); - // Track whether we have seen a non-Ready phase during the watch. - let mut saw_non_ready = SandboxPhase::try_from(sandbox.phase()) != Ok(SandboxPhase::Ready); let provision_timeout = Duration::from_secs( std::env::var("OPENSHELL_PROVISION_TIMEOUT") .ok() @@ -911,10 +906,6 @@ pub async fn sandbox_create( last_condition_message = Some(message); } - if phase != SandboxPhase::Ready { - saw_non_ready = true; - } - let main_process_result = has_main_process_result(&s); if matches!( phase, @@ -949,9 +940,10 @@ pub async fn sandbox_create( break; } - // Only accept Ready as terminal after we've observed a - // non-Ready phase, proving the controller has reconciled. - if saw_non_ready && phase == SandboxPhase::Ready { + // The gateway owns readiness. Its initial watch snapshot may + // already be Ready if provisioning finished before CREATE + // returned; requiring an earlier phase would miss that state. + if phase == SandboxPhase::Ready { if let Some(d) = display.as_interactive_mut() { d.clear(); } diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index e21f747112..6723a25bc7 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -70,6 +70,7 @@ struct SandboxState { vm_error_with_observed_exit: Arc, vm_slow_progress_before_ready: Arc, vm_log_churn_before_ready: Arc, + ready_before_create_returns: Arc, terminal_before_relay: Arc, terminal_after_provisional_container_exit: Arc, provisional_container_exit_without_result: Arc, @@ -194,7 +195,17 @@ impl OpenShell for TestOpenShell { }), ..Sandbox::default() }; - sandbox.set_phase(SandboxPhase::Provisioning as i32); + sandbox.set_phase( + if self + .state + .ready_before_create_returns + .load(Ordering::SeqCst) + { + SandboxPhase::Ready as i32 + } else { + SandboxPhase::Provisioning as i32 + }, + ); Ok(Response::new(SandboxResponse { sandbox: Some(sandbox), service_urls, @@ -677,6 +688,10 @@ impl OpenShell for TestOpenShell { .vm_slow_progress_before_ready .load(Ordering::SeqCst); let vm_log_churn_before_ready = self.state.vm_log_churn_before_ready.load(Ordering::SeqCst); + let ready_before_create_returns = self + .state + .ready_before_create_returns + .load(Ordering::SeqCst); let terminal_before_relay = self.state.terminal_before_relay.load(Ordering::SeqCst); let terminal_after_provisional_container_exit = self .state @@ -723,6 +738,18 @@ impl OpenShell for TestOpenShell { } let mut ready = provisioning.clone(); ready.set_phase(SandboxPhase::Ready as i32); + if ready_before_create_returns { + // A watch starts with the current snapshot. Keep it open so + // stream closure cannot hide a client that ignores Ready. + let _ = tx + .send(Ok(SandboxStreamEvent { + payload: Some(sandbox_stream_event::Payload::Sandbox(ready)), + cursor: String::new(), + })) + .await; + tx.closed().await; + return; + } let mut completed = provisioning.clone(); completed.status = Some(SandboxStatus { exit_code: Some(0), @@ -2372,6 +2399,55 @@ async fn sandbox_create_preserves_vm_error_when_exit_code_is_observed() { assert!(rendered.contains("ProcessExited: VM process exited with status 0")); } +#[tokio::test] +async fn sandbox_create_accepts_ready_before_create_returns() { + let server = run_server().await; + server + .openshell + .state + .ready_before_create_returns + .store(true, Ordering::SeqCst); + let fake_ssh_dir = tempfile::tempdir().unwrap(); + let xdg_dir = tempfile::tempdir().unwrap(); + let _env = test_env_with( + &fake_ssh_dir, + &xdg_dir, + &[("OPENSHELL_PROVISION_TIMEOUT", "1".to_string())], + ); + let tls = test_tls(&server); + install_fake_ssh(&fake_ssh_dir); + + let exit_code = tokio::time::timeout( + Duration::from_secs(10), + run::sandbox_create( + &server.endpoint, + "openshell", + run::SandboxCreateConfig { + name: Some("already-ready"), + command: &["echo".into(), "OK".into()], + ..test_config() + }, + "default", + &tls, + ), + ) + .await + .expect("creation must finish while the watch remains open") + .expect("an already-Ready sandbox must not wait for a new provisioning transition"); + + assert_eq!(exit_code, 0); + assert_eq!(create_requests(&server).await.len(), 1); + assert_eq!( + server + .openshell + .state + .ssh_session_requests + .load(Ordering::SeqCst), + 1, + "the initial Ready snapshot must allow the command to attach" + ); +} + #[tokio::test] async fn sandbox_create_keeps_waiting_while_vm_progress_arrives() { let server = run_server().await; diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 0270157f88..52a15c6300 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -8435,7 +8435,7 @@ mod tests { } #[tokio::test] - async fn create_compensation_deletes_backend_when_delete_transition_cannot_be_stored() { + async fn create_compensation_retains_backend_when_delete_transition_cannot_be_stored() { let directory = tempfile::tempdir().expect("temporary database directory"); let database_url = format!("sqlite://{}", directory.path().join("gateway.db").display()); let store = Arc::new(Store::connect(&database_url).await.expect("connect store")); @@ -8478,14 +8478,10 @@ mod tests { .message() .contains("could not claim the sandbox record") ); - assert_eq!(driver.delete_calls(), 1); - assert_eq!( - driver.delete_requests(), - vec![( - sandbox.object_id().to_string(), - sandbox.object_name().to_string() - )] - ); + // A rejected durable claim leaves cleanup ownership unknown. An + // unclaimed DELETE could destroy newer compute with the same ID. + assert_eq!(driver.delete_calls(), 0); + assert!(driver.delete_requests().is_empty()); let retained = runtime .store .get_message::(sandbox.object_id()) diff --git a/crates/openshell-server/src/compute/provisioning_operation.rs b/crates/openshell-server/src/compute/provisioning_operation.rs index 457d219b6c..764c136cc4 100644 --- a/crates/openshell-server/src/compute/provisioning_operation.rs +++ b/crates/openshell-server/src/compute/provisioning_operation.rs @@ -12,6 +12,7 @@ use std::time::Duration; use openshell_core::{ObjectId, proto::Sandbox}; use tonic::Status; +use tracing::Instrument as _; use super::{ ComputeRuntime, provisioning_deadline, sandbox_provisioning_attempt_id, @@ -127,7 +128,7 @@ impl ComputeRuntime { let owned_attempt = starting.clone(); // Dropping a JoinHandle detaches its task. Do not abort it on a monitor // error or deadline: the remote side may still commit the request. - let mut worker = tokio::spawn(async move { + let worker = async move { let result = operation.await; let settled = if tracked { let settled = runtime @@ -155,7 +156,10 @@ impl ComputeRuntime { settled: Box::new(settled), }), } - }); + }; + // Retain the request span so detaching ownership preserves the driver + // call's parent trace, including when the caller stops waiting. + let mut worker = tokio::spawn(worker.in_current_span()); loop { tokio::select! { result = &mut worker => return result.map_err(|error| {