From a84a32f4b62d1fe9a7ded6a9592bfff160b06982 Mon Sep 17 00:00:00 2001 From: tornquist Date: Tue, 22 Sep 2026 21:53:16 +0000 Subject: [PATCH 01/46] Add owner deletion admission control plane Signed-off-by: tornquist Co-authored-by: Codex --- ARCHITECTURE.md | 12 + crates/buzz-db/src/lib.rs | 2 +- crates/buzz-db/src/runtime/migration.rs | 45 +- .../tests/thread_window_postgres_tests.rs | 2 +- crates/buzz-db/src/store/community.rs | 119 ++++-- crates/buzz-db/src/store/deletion.rs | 403 +++++++++++++++++- crates/buzz-db/src/store/relay_members.rs | 43 +- crates/buzz-relay/src/api/operator.rs | 274 +++++++++++- crates/buzz-relay/src/router.rs | 6 +- ...050_owner_community_deletion_admission.sql | 66 +++ schema/schema.sql | 27 +- 11 files changed, 943 insertions(+), 56 deletions(-) create mode 100644 migrations/0050_owner_community_deletion_admission.sql diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ae82c131ec7..7d7da313d7b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -14,6 +14,18 @@ EVENT, REQ, REST, media, git, search, workflow, or pub/sub handling. Unknown hosts fail closed, and NIP-98/API-token stamps must agree with the host-derived community rather than overriding it. +Deployment-root community management uses operator-signed NIP-98 HTTP requests. +`POST /operator/communities/delete` accepts only an exact normalized, archived +community whose asserted pubkey is still its owner. The caller supplies the +request UUID as the stable correlation/idempotency identity; the durable row +records owner intent, mediating operator, and acknowledgement version. Admission +returns `202` at the `submitted` stage and performs no inventory, approval, +quiescing, object-store access, or deletion execution synchronously. While that +non-aborted request exists, unarchive and ownership transfer conflict and owner +management lists suppress the archived row. Replaying the same UUID converges +to its current stage; a different UUID conflicts with the existing one-active- +request invariant until that request is aborted. + Buzz is a Rust monorepo, licensed Apache 2.0 under Block, Inc. --- diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index e70c5dd85b1..2f5d3e8aacc 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -72,7 +72,7 @@ pub use allowlist::AllowlistEntry; pub use api_token::{ApiTokenRecord, TokenSummary}; pub use community::{ ArchivedCommunityRecord, CommunityRecord, CreateCommunityWithOwnerResult, - CreatedCommunityRecord, EnsuredCommunityRecord, OwnedCommunityRecord, + CreatedCommunityRecord, EnsuredCommunityRecord, OwnedCommunityRecord, UnarchiveCommunityResult, UnarchivedCommunityRecord, }; pub use error::{DbError, Result}; diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 20496ac1cfc..a6727c4ac72 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -703,12 +703,17 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 49); + assert_eq!(migrations.len(), 50); assert_eq!(migrations[48].version, 49); + assert_eq!(migrations[49].version, 50); assert!(migrations[48] .sql .as_str() .contains("idx_thread_metadata_window")); + assert!(migrations[49] + .sql + .as_str() + .contains("community_deletion_owner_provenance")); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -1765,6 +1770,12 @@ mod postgres_tests { .expect("embedded migration 0029") .sql .as_ref(); + let migration_0050: &str = MIGRATOR + .iter() + .find(|migration| migration.version == 50) + .expect("embedded migration 0050") + .sql + .as_ref(); let workspace_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .and_then(std::path::Path::parent) @@ -1773,6 +1784,7 @@ mod postgres_tests { .expect("read schema/schema.sql"); let migration = surface(migration_0029); + let owner_admission_migration = surface(migration_0050); let schema = surface(&schema_sql); assert_eq!( @@ -1801,13 +1813,42 @@ mod postgres_tests { .functions .get(function) .unwrap_or_else(|| panic!("schema.sql is missing deletion function {function}")); - if function != "community_write_fence_excluded_table" { + if function != "community_write_fence_excluded_table" + && function != "prevent_community_deletion_request_retargeting" + { assert_eq!( in_schema, definition, "schema.sql definition of {function}() drifted from migration 0029" ); } } + assert_eq!( + schema + .functions + .get("prevent_community_deletion_request_retargeting") + .expect("schema.sql deletion retargeting guard"), + owner_admission_migration + .functions + .get("prevent_community_deletion_request_retargeting") + .expect("0050 deletion retargeting guard"), + "schema.sql must carry the latest immutable owner-provenance guard" + ); + let request_table = schema + .tables + .get("community_deletion_requests") + .expect("schema.sql deletion request table"); + for owner_provenance_fragment in [ + "request_origin text not null default 'operator'", + "owner_pubkey text", + "mediating_operator_pubkey text", + "acknowledgement_version integer", + "constraint community_deletion_owner_provenance check", + ] { + assert!( + request_table.contains(owner_provenance_fragment), + "schema.sql deletion requests are missing {owner_provenance_fragment}" + ); + } for (trigger, definition) in &migration.triggers { let in_schema = schema .triggers diff --git a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs index 6062528d550..5ca0185af56 100644 --- a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs @@ -311,7 +311,7 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() production_result.is_ok(), "production migrator must preserve ingestion progress: {production_result:?}" ); - assert_eq!(version, 49); + assert_eq!(version, 50); assert_eq!(final_oid, oid, "prebuild must not be replaced"); assert_eq!(count, 4, "all writer witnesses must persist"); } diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index bcd38f4e5ce..461348a5cd3 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -81,6 +81,17 @@ pub struct UnarchivedCommunityRecord { pub host: String, } +/// Result of an owner-authorized unarchive attempt. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum UnarchiveCommunityResult { + /// The community is active, with archive state cleared idempotently. + Unarchived(UnarchivedCommunityRecord), + /// Durable deletion intent exists and wins over restoration. + DeletionPending, + /// The host is absent, unavailable, or not owned by the asserted pubkey. + NotFound, +} + impl Db { /// Returns the community mapped to a normalized request host, if one exists. /// @@ -209,6 +220,10 @@ impl Db { JOIN relay_members rm ON rm.community_id = c.id WHERE rm.pubkey = $1 AND rm.role = 'owner' + AND NOT EXISTS ( + SELECT 1 FROM community_deletion_requests request + WHERE request.community_id = c.id AND request.stage <> 'aborted' + ) ORDER BY c.created_at ASC, c.host ASC "#, ) @@ -531,40 +546,69 @@ impl Db { } /// Idempotently restores a community when the asserted pubkey is its current owner. + /// + /// Locks the community row so owner-deletion admission and restoration have + /// one serial order. A non-aborted deletion request returns + /// [`UnarchiveCommunityResult::DeletionPending`] without clearing archive state. #[datastore_span(name = "unarchive_community_owned_by", system = "postgresql")] pub async fn unarchive_community_owned_by( &self, normalized_host: &str, owner_pubkey: &str, - ) -> Result> { - let mut connection = crate::observability::acquire_writer( + ) -> Result { + let connection = crate::observability::acquire_writer( &self.pool, crate::observability::WriterOperation::Authorization, ) .await?; - let row = sqlx::query( - r#"UPDATE communities c - SET archived_at = NULL - FROM relay_members rm - WHERE lower(c.host) = lower($1) - AND rm.community_id = c.id - AND lower(rm.pubkey) = lower($2) - AND rm.role = 'owner' - AND c.deletion_state = 'active' - AND c.deleted_at IS NULL - RETURNING c.id, c.host"#, + let mut tx = sqlx::Transaction::begin(connection, None).await?; + let target = sqlx::query( + "SELECT id, host FROM communities \ + WHERE lower(host) = lower($1) AND deletion_state = 'active' \ + AND deleted_at IS NULL FOR UPDATE", ) .bind(normalized_host) + .fetch_optional(&mut *tx) + .await?; + let Some(target) = target else { + tx.rollback().await?; + return Ok(UnarchiveCommunityResult::NotFound); + }; + let community_id: Uuid = target.try_get("id")?; + let is_owner: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM relay_members \ + WHERE community_id = $1 AND lower(pubkey) = lower($2) AND role = 'owner')", + ) + .bind(community_id) .bind(owner_pubkey) - .fetch_optional(&mut *connection) + .fetch_one(&mut *tx) .await?; - row.map(|row| { - Ok(UnarchivedCommunityRecord { - id: CommunityId::from_uuid(row.try_get("id")?), - host: row.try_get("host")?, - }) - }) - .transpose() + if !is_owner { + tx.rollback().await?; + return Ok(UnarchiveCommunityResult::NotFound); + } + let deletion_pending: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests \ + WHERE community_id = $1 AND stage <> 'aborted')", + ) + .bind(community_id) + .fetch_one(&mut *tx) + .await?; + if deletion_pending { + tx.rollback().await?; + return Ok(UnarchiveCommunityResult::DeletionPending); + } + sqlx::query("UPDATE communities SET archived_at = NULL WHERE id = $1") + .bind(community_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(UnarchiveCommunityResult::Unarchived( + UnarchivedCommunityRecord { + id: CommunityId::from_uuid(community_id), + host: target.try_get("host")?, + }, + )) } /// Returns the community that owns a channel, if the channel exists. @@ -900,22 +944,26 @@ mod postgres_tests { .is_none(), "archived communities must fail admission" ); - assert!(db - .unarchive_community_owned_by(&host, &outsider) - .await - .expect("wrong-owner unarchive") - .is_none()); - assert!(db - .unarchive_community_owned_by("missing.example", &owner) - .await - .expect("unknown-host unarchive") - .is_none()); + assert_eq!( + db.unarchive_community_owned_by(&host, &outsider) + .await + .expect("wrong-owner unarchive"), + UnarchiveCommunityResult::NotFound + ); + assert_eq!( + db.unarchive_community_owned_by("missing.example", &owner) + .await + .expect("unknown-host unarchive"), + UnarchiveCommunityResult::NotFound + ); let restored = db .unarchive_community_owned_by(&host.to_ascii_uppercase(), &owner) .await - .expect("unarchive community") - .expect("owned community"); + .expect("unarchive community"); + let UnarchiveCommunityResult::Unarchived(restored) = restored else { + panic!("expected owned community") + }; assert_eq!(restored.id, created.id); assert_eq!(restored.host, host); assert_eq!( @@ -938,9 +986,8 @@ mod postgres_tests { let retry = db .unarchive_community_owned_by(&host, &owner) .await - .expect("idempotent retry") - .expect("owned community"); - assert_eq!(retry, restored); + .expect("idempotent retry"); + assert_eq!(retry, UnarchiveCommunityResult::Unarchived(restored)); } #[tokio::test] diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 0e184e00d88..090aa973223 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -27,6 +27,8 @@ pub const POSTGRES_STORE_NAME: &str = "postgres"; pub const OBJECT_STORE_NAME: &str = "object_store"; /// Durable name of the Redis/cache manifest component. pub const REDIS_STORE_NAME: &str = "redis"; +/// Owner acknowledgement contract accepted by the first self-serve deletion API. +pub const OWNER_DELETION_ACKNOWLEDGEMENT_VERSION: i32 = 1; /// Deployment-global advisory-lock key serializing schema migration with /// destructive deletion. @@ -220,7 +222,7 @@ impl FromStr for DeletionStage { } /// Durable community deletion request. -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, PartialEq, Serialize)] pub struct DeletionRequest { /// Request identifier. pub id: Uuid, @@ -233,8 +235,16 @@ pub struct DeletionRequest { pub stage: DeletionStage, /// Stage at which the current consecutive retry streak started. pub retry_stage: Option, - /// Operator identity that submitted the request. + /// Legacy display identity that submitted the request. pub requested_by: String, + /// Whether the request originated from an operator or authenticated owner intent. + pub request_origin: DeletionRequestOrigin, + /// Current owner identity authenticated at owner-request admission. + pub owner_pubkey: Option, + /// Deployment operator that mediated the authenticated owner intent. + pub mediating_operator_pubkey: Option, + /// Owner-facing destructive-action acknowledgement contract version. + pub acknowledgement_version: Option, /// Optional request reason. pub reason: Option, /// Frozen catalog manifest. @@ -283,6 +293,47 @@ pub struct DeletionRequest { pub completed_at: Option>, } +/// Durable provenance class for a community deletion request. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DeletionRequestOrigin { + /// Request was submitted directly by a deployment operator. + Operator, + /// Request records authenticated owner intent mediated by an operator. + Owner, +} + +impl FromStr for DeletionRequestOrigin { + type Err = DbError; + + fn from_str(value: &str) -> std::result::Result { + match value { + "operator" => Ok(Self::Operator), + "owner" => Ok(Self::Owner), + other => Err(DbError::InvalidData(format!( + "unknown community deletion request origin: {other}" + ))), + } + } +} + +/// Result of atomically admitting authenticated owner deletion intent. +#[derive(Debug, Clone, PartialEq)] +pub enum OwnerDeletionAdmission { + /// A new request was created or the stable request UUID converged to its row. + Accepted(Box), + /// The exact host is absent or the asserted owner is no longer current. + NotFoundOrNotOwner, + /// The community exists and is current-owner controlled, but is not archived. + NotArchived, + /// The community is already quiescing, fenced, or deleted. + LifecycleConflict, + /// The request UUID targets different intent, or another active request exists. + RequestConflict, + /// The owner acknowledgement contract is not supported. + UnsupportedAcknowledgementVersion, +} + /// Frozen PostgreSQL catalog inventory. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct SchemaManifest { @@ -742,6 +793,131 @@ impl DeletionStore { } } + /// Atomically admit an archived current owner's deletion intent. + /// + /// `request_id` is both the durable request id and the caller's stable + /// correlation/idempotency identity. Replays return the existing request at + /// its current stage. This operation only persists intent; it never + /// inventories, approves, quiesces, or executes deletion. + pub async fn admit_owner_request( + &self, + normalized_community_host: &str, + owner_pubkey: &str, + mediating_operator_pubkey: &str, + acknowledgement_version: i32, + request_id: Uuid, + ) -> Result { + if acknowledgement_version != OWNER_DELETION_ACKNOWLEDGEMENT_VERSION { + return Ok(OwnerDeletionAdmission::UnsupportedAcknowledgementVersion); + } + let owner_pubkey = owner_pubkey.to_ascii_lowercase(); + let mediating_operator_pubkey = mediating_operator_pubkey.to_ascii_lowercase(); + let mut tx = self.pool.begin().await?; + + sqlx::query( + "SELECT pg_advisory_xact_lock(hashtextextended('buzz-owner-deletion-intent:' || $1::text, 0))", + ) + .bind(request_id) + .execute(&mut *tx) + .await?; + + if let Some(row) = sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1") + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + { + let existing = row_to_request(row)?; + let converges = existing.community_host == normalized_community_host + && existing.request_origin == DeletionRequestOrigin::Owner + && existing.owner_pubkey.as_deref() == Some(owner_pubkey.as_str()) + && existing.acknowledgement_version == Some(acknowledgement_version); + tx.rollback().await?; + return Ok(if converges { + OwnerDeletionAdmission::Accepted(Box::new(existing)) + } else { + OwnerDeletionAdmission::RequestConflict + }); + } + + let target = sqlx::query( + "SELECT id, host, archived_at, deletion_state, deleted_at \ + FROM communities WHERE host = $1 FOR UPDATE", + ) + .bind(normalized_community_host) + .fetch_optional(&mut *tx) + .await?; + let Some(target) = target else { + tx.rollback().await?; + return Ok(OwnerDeletionAdmission::NotFoundOrNotOwner); + }; + let community_id: Uuid = target.try_get("id")?; + let canonical_host: String = target.try_get("host")?; + let deletion_state: String = target.try_get("deletion_state")?; + let deleted_at: Option> = target.try_get("deleted_at")?; + if deletion_state != "active" || deleted_at.is_some() { + tx.rollback().await?; + return Ok(OwnerDeletionAdmission::LifecycleConflict); + } + + let owner_exists = sqlx::query_scalar::<_, String>( + "SELECT pubkey FROM relay_members \ + WHERE community_id = $1 AND pubkey = $2 AND role = 'owner' FOR UPDATE", + ) + .bind(community_id) + .bind(&owner_pubkey) + .fetch_optional(&mut *tx) + .await? + .is_some(); + if !owner_exists { + tx.rollback().await?; + return Ok(OwnerDeletionAdmission::NotFoundOrNotOwner); + } + if target + .try_get::>, _>("archived_at")? + .is_none() + { + tx.rollback().await?; + return Ok(OwnerDeletionAdmission::NotArchived); + } + let active_request_exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests \ + WHERE community_id = $1 AND stage <> 'aborted')", + ) + .bind(community_id) + .fetch_one(&mut *tx) + .await?; + if active_request_exists { + tx.rollback().await?; + return Ok(OwnerDeletionAdmission::RequestConflict); + } + + let row = sqlx::query( + r#" + INSERT INTO community_deletion_requests ( + id, community_id, community_host, requested_by, request_origin, + owner_pubkey, mediating_operator_pubkey, acknowledgement_version + ) VALUES ($1, $2, $3, $4, 'owner', $4, $5, $6) + ON CONFLICT (community_id) WHERE stage <> 'aborted' DO NOTHING + RETURNING * + "#, + ) + .bind(request_id) + .bind(community_id) + .bind(canonical_host) + .bind(owner_pubkey) + .bind(mediating_operator_pubkey) + .bind(acknowledgement_version) + .fetch_optional(&mut *tx) + .await?; + let Some(row) = row else { + tx.rollback().await?; + return Ok(OwnerDeletionAdmission::RequestConflict); + }; + let request = row_to_request(row)?; + tx.commit().await?; + Ok(OwnerDeletionAdmission::Accepted(Box::new(request))) + } + /// List requests newest first with a hard bound. pub async fn list(&self, limit: i64) -> Result> { let rows = sqlx::query( @@ -3112,6 +3288,10 @@ fn row_to_request(row: sqlx::postgres::PgRow) -> Result { .map(|stage| stage.parse()) .transpose()?, requested_by: row.try_get("requested_by")?, + request_origin: row.try_get::("request_origin")?.parse()?, + owner_pubkey: row.try_get("owner_pubkey")?, + mediating_operator_pubkey: row.try_get("mediating_operator_pubkey")?, + acknowledgement_version: row.try_get("acknowledgement_version")?, reason: row.try_get("reason")?, schema_manifest: row.try_get("schema_manifest")?, storage_manifest: row.try_get("storage_manifest")?, @@ -3416,7 +3596,10 @@ mod tests { #[cfg(test)] mod postgres_tests { use super::*; - use crate::{CreateCommunityWithOwnerResult, Db, DbConfig}; + use crate::{ + relay_members::TransferResult, CreateCommunityWithOwnerResult, Db, DbConfig, + UnarchiveCommunityResult, + }; async fn store() -> (Db, DeletionStore) { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") @@ -3485,6 +3668,220 @@ mod postgres_tests { (request, inventory) } + async fn archived_owned_community(db: &Db) -> (String, String, CommunityId) { + let host = format!("owner-delete-{}.example", Uuid::new_v4().simple()); + let owner = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let CreateCommunityWithOwnerResult::Created(created) = db + .create_community_with_owner(&host, &owner) + .await + .expect("create owned community") + else { + panic!("expected a fresh community") + }; + db.archive_community_owned_by(&host, &owner, "protected.example") + .await + .expect("archive community") + .expect("owned community"); + (host, owner, created.id) + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_admission_requires_current_owner_and_archived_active_target() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + + let admitted = store + .admit_owner_request(&host, &owner, operator, 1, request_id) + .await + .expect("admit archived owner request"); + let OwnerDeletionAdmission::Accepted(request) = admitted else { + panic!("expected accepted owner request") + }; + assert_eq!(request.id, request_id); + assert_eq!(request.community_id, community); + assert_eq!(request.community_host, host); + assert_eq!(request.stage, DeletionStage::Submitted); + assert_eq!(request.request_origin, DeletionRequestOrigin::Owner); + assert_eq!(request.owner_pubkey.as_deref(), Some(owner.as_str())); + assert_eq!(request.mediating_operator_pubkey.as_deref(), Some(operator)); + assert_eq!(request.acknowledgement_version, Some(1)); + assert_eq!( + sqlx::query_scalar::<_, String>("SELECT deletion_state FROM communities WHERE id = $1") + .bind(community.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("community lifecycle"), + "active", + "admission must not prematurely quiesce the community" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_admission_rejects_non_archived_non_owner_and_stale_owner() { + let (db, store) = store().await; + let host = format!("owner-delete-active-{}.example", Uuid::new_v4().simple()); + let owner = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let replacement = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let outsider = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let CreateCommunityWithOwnerResult::Created(created) = db + .create_community_with_owner(&host, &owner) + .await + .expect("create community") + else { + panic!("expected fresh community") + }; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + + assert_eq!( + store + .admit_owner_request(&host, &owner, operator, 1, Uuid::new_v4()) + .await + .expect("non-archived admission result"), + OwnerDeletionAdmission::NotArchived + ); + db.archive_community_owned_by(&host, &owner, "protected.example") + .await + .expect("archive") + .expect("owned community"); + assert_eq!( + store + .admit_owner_request(&host, &outsider, operator, 1, Uuid::new_v4()) + .await + .expect("non-owner admission result"), + OwnerDeletionAdmission::NotFoundOrNotOwner + ); + assert!(matches!( + db.transfer_ownership(created.id, &replacement, &owner) + .await + .expect("rotate owner"), + TransferResult::Transferred { .. } + )); + assert_eq!( + store + .admit_owner_request(&host, &owner, operator, 1, Uuid::new_v4()) + .await + .expect("stale-owner admission result"), + OwnerDeletionAdmission::NotFoundOrNotOwner + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_admission_replay_converges_after_stage_advancement_and_rejects_retargeting() { + let (db, store) = store().await; + let (host, owner, _) = archived_owned_community(&db).await; + let (other_host, other_owner, _) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + let OwnerDeletionAdmission::Accepted(first) = store + .admit_owner_request(&host, &owner, operator, 1, request_id) + .await + .expect("first admission") + else { + panic!("expected accepted request") + }; + let inventory = FrozenInventory { + schema: store + .inventory_schema(first.community_id) + .await + .expect("schema inventory"), + storage: empty_storage_manifest(first.community_id), + }; + store + .freeze_inventory(first.id, &inventory) + .await + .expect("advance request"); + + let OwnerDeletionAdmission::Accepted(replayed) = store + .admit_owner_request(&host, &owner, operator, 1, request_id) + .await + .expect("replay admission") + else { + panic!("expected converged replay") + }; + assert_eq!(replayed.id, first.id); + assert_eq!(replayed.stage, DeletionStage::Inventoried); + assert_eq!( + store + .admit_owner_request(&other_host, &other_owner, operator, 1, request_id) + .await + .expect("retargeting result"), + OwnerDeletionAdmission::RequestConflict + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn concurrent_owner_admission_duplicates_return_one_stable_request() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + let (first, second) = tokio::join!( + store.admit_owner_request(&host, &owner, operator, 1, request_id), + store.admit_owner_request(&host, &owner, operator, 1, request_id), + ); + let accepted_id = |result: Result| { + let OwnerDeletionAdmission::Accepted(request) = result.expect("admission") else { + panic!("expected accepted request") + }; + request.id + }; + assert_eq!(accepted_id(first), request_id); + assert_eq!(accepted_id(second), request_id); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM community_deletion_requests WHERE community_id = $1 AND stage <> 'aborted'" + ) + .bind(community.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("active request count"), + 1 + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn accepted_owner_request_blocks_unarchive_and_transfer_and_suppresses_owner_list_row() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let new_owner = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let OwnerDeletionAdmission::Accepted(_) = store + .admit_owner_request(&host, &owner, operator, 1, Uuid::new_v4()) + .await + .expect("admit owner request") + else { + panic!("expected accepted request") + }; + + assert_eq!( + db.unarchive_community_owned_by(&host, &owner) + .await + .expect("unarchive result"), + UnarchiveCommunityResult::DeletionPending + ); + assert_eq!( + db.transfer_ownership(community, &new_owner, &owner) + .await + .expect("transfer result"), + TransferResult::DeletionPending + ); + assert!( + db.list_communities_owned_by(&owner) + .await + .expect("owner list") + .iter() + .all(|row| row.id != community), + "accepted deletion requests must not remain actionable archived rows" + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn approval_boundary_blocks_claim_until_exact_inventory_is_approved() { diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index ecde3924fef..a54adc139ee 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -444,6 +444,8 @@ pub enum TransferResult { /// concurrent transfer or owner rotation has already changed ownership. /// The caller must NOT retry blindly — re-read ownership and re-evaluate. OwnerConflict, + /// Durable deletion intent exists and wins over ownership mutation. + DeletionPending, /// The transferee already owns the maximum number of communities. /// Enforced atomically inside the transfer transaction so concurrent /// transfers to the same recipient cannot both pass the limit. @@ -500,13 +502,16 @@ pub fn owner_count_advisory_lock_key(pubkey_hex: &str) -> i64 { /// so that concurrent transfers to the same recipient serialize. The same /// lock key is also used by `Db::create_community_with_owner` to prevent /// transfer-vs-create races. -/// 2. Locks the current owner row `FOR UPDATE` and verifies +/// 2. Locks the community row and rejects any non-aborted deletion request. +/// Owner-deletion admission takes the same row lock, so whichever operation +/// commits first makes the other re-evaluate and conflict. +/// 3. Locks the current owner row `FOR UPDATE` and verifies /// `expected_owner_pubkey` matches. This prevents a stale-owner race where /// a delayed/retried request overwrites a completed transfer. -/// 3. Enforces the [`MAX_COMMUNITIES_PER_OWNER`] limit on the transferee by +/// 4. Enforces the [`MAX_COMMUNITIES_PER_OWNER`] limit on the transferee by /// counting owned communities inside the same transaction. -/// 4. Upserts `new_owner_pubkey` as `owner` (insert or promote). -/// 5. Demotes every other owner in this community to `member` — **not** +/// 5. Upserts `new_owner_pubkey` as `owner` (insert or promote). +/// 6. Demotes every other owner in this community to `member` — **not** /// `admin`, per product decision: the former owner retains no management /// capabilities. /// @@ -533,7 +538,29 @@ pub async fn transfer_ownership( ) .await?; - // 2. Lock the current owner row FOR UPDATE and verify the expected owner. + let community_exists = + sqlx::query_scalar::<_, Uuid>("SELECT id FROM communities WHERE id = $1 FOR UPDATE") + .bind(community.as_uuid()) + .fetch_optional(&mut *tx) + .await? + .is_some(); + if !community_exists { + tx.rollback().await?; + return Ok(TransferResult::NoOwner); + } + let deletion_pending: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests \ + WHERE community_id = $1 AND stage <> 'aborted')", + ) + .bind(community.as_uuid()) + .fetch_one(&mut *tx) + .await?; + if deletion_pending { + tx.rollback().await?; + return Ok(TransferResult::DeletionPending); + } + + // 3. Lock the current owner row FOR UPDATE and verify the expected owner. // FOR UPDATE prevents the stale-owner race: a concurrent transfer that // already changed the owner will block on this lock until our txn // completes (or vice versa), and the expected_owner check will fail. @@ -570,7 +597,7 @@ pub async fn transfer_ownership( existing_owners.iter().find(|p| **p != pubkey).cloned() }; - // 3. Enforce the transferee's community ownership limit inside the same + // 4. Enforce the transferee's community ownership limit inside the same // transaction that holds the advisory lock. This is the authoritative // check — kgoose's preflight count is advisory only. let owned_count: i64 = sqlx::query_scalar( @@ -585,7 +612,7 @@ pub async fn transfer_ownership( return Ok(TransferResult::LimitReached); } - // 4. Upsert the new owner. + // 5. Upsert the new owner. sqlx::query( "INSERT INTO relay_members (community_id, pubkey, role, added_by) \ VALUES ($1, $2, 'owner', NULL) \ @@ -596,7 +623,7 @@ pub async fn transfer_ownership( .execute(&mut *tx) .await?; - // 5. Demote all other owners to member (not admin). + // 6. Demote all other owners to member (not admin). sqlx::query( "UPDATE relay_members SET role = 'member', updated_at = now() \ WHERE community_id = $1 AND role = 'owner' AND pubkey <> $2", diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index 2c49ca6a5c3..c672f837227 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -203,6 +203,15 @@ pub struct ArchiveCommunityRequest { owner_pubkey: String, } +/// Authenticated owner intent mediated by a trusted deployment operator. +#[derive(Debug, Deserialize)] +pub struct DeleteCommunityRequest { + host: String, + owner_pubkey: String, + request_id: Uuid, + acknowledgement_version: i32, +} + /// Idempotently archive a community owned by the asserted end-user identity. pub async fn archive_community( State(state): State>, @@ -287,12 +296,23 @@ pub async fn unarchive_community( "invalid owner_pubkey: expected 64-char hex pubkey", ) })?; - let record = state + let result = state .db .unarchive_community_owned_by(&normalized_host, &owner) .await - .map_err(|e| internal_error(&format!("unarchive community: {e}")))? - .ok_or_else(|| api_error(StatusCode::NOT_FOUND, "community not found"))?; + .map_err(|e| internal_error(&format!("unarchive community: {e}")))?; + let record = match result { + buzz_db::UnarchiveCommunityResult::Unarchived(record) => record, + buzz_db::UnarchiveCommunityResult::DeletionPending => { + return Err(api_error( + StatusCode::CONFLICT, + "community deletion is pending", + )); + } + buzz_db::UnarchiveCommunityResult::NotFound => { + return Err(api_error(StatusCode::NOT_FOUND, "community not found")); + } + }; tracing::info!(community = %record.id, host = %record.host, "community unarchived"); Ok(Json(serde_json::json!({ "community_id": record.id.to_string(), @@ -302,6 +322,106 @@ pub async fn unarchive_community( }))) } +/// Persist authenticated owner deletion intent without executing deletion work. +/// +/// `POST /operator/communities/delete`, NIP-98 signed by a pubkey in +/// `RELAY_OPERATOR_PUBKEYS`, body: +/// +/// ```json +/// { +/// "host": "archived.communities.example", +/// "owner_pubkey": "<64-char hex>", +/// "request_id": "", +/// "acknowledgement_version": 1 +/// } +/// ``` +/// +/// The request UUID is the correlation/idempotency key. Acceptance is a fast +/// PostgreSQL-only transaction and returns `202`; inventory, approval, +/// quiescing, object-store access, and executor work remain asynchronous. +pub async fn delete_community( + State(state): State>, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Result<(StatusCode, Json), (StatusCode, Json)> { + const PATH: &str = "/operator/communities/delete"; + let operator = + authorize_operator_request(&state, &headers, "POST", PATH, None, Some(&body)).await?; + let request: DeleteCommunityRequest = serde_json::from_slice(&body).map_err(|e| { + api_error( + StatusCode::BAD_REQUEST, + &format!("invalid delete-community JSON: {e}"), + ) + })?; + let normalized_host = normalize_candidate_host(&request.host) + .map_err(|msg| api_error(StatusCode::BAD_REQUEST, &msg))?; + let deployment_host = buzz_core::tenant::relay_url_authority(&state.config.relay_url); + if normalized_host == deployment_host { + return Err(api_error( + StatusCode::CONFLICT, + "the deployment community cannot be deleted", + )); + } + let owner = validate_pubkey_hex(&request.owner_pubkey).ok_or_else(|| { + api_error( + StatusCode::BAD_REQUEST, + "invalid owner_pubkey: expected 64-char hex pubkey", + ) + })?; + let operator = operator.to_hex(); + let admission = state + .db + .deletion_store() + .admit_owner_request( + &normalized_host, + &owner, + &operator, + request.acknowledgement_version, + request.request_id, + ) + .await + .map_err(|error| internal_error(&format!("admit owner deletion request: {error}")))?; + let accepted = match admission { + buzz_db::deletion::OwnerDeletionAdmission::Accepted(request) => request, + buzz_db::deletion::OwnerDeletionAdmission::NotFoundOrNotOwner => { + return Err(api_error(StatusCode::NOT_FOUND, "community not found")); + } + buzz_db::deletion::OwnerDeletionAdmission::NotArchived => { + return Err(api_error( + StatusCode::CONFLICT, + "community must be archived before deletion", + )); + } + buzz_db::deletion::OwnerDeletionAdmission::LifecycleConflict => { + return Err(api_error( + StatusCode::CONFLICT, + "community deletion lifecycle is already active", + )); + } + buzz_db::deletion::OwnerDeletionAdmission::RequestConflict => { + return Err(api_error( + StatusCode::CONFLICT, + "deletion request conflicts with existing intent", + )); + } + buzz_db::deletion::OwnerDeletionAdmission::UnsupportedAcknowledgementVersion => { + return Err(api_error( + StatusCode::BAD_REQUEST, + "unsupported acknowledgement_version", + )); + } + }; + Ok(( + StatusCode::ACCEPTED, + Json(serde_json::json!({ + "request_id": accepted.id, + "community_id": accepted.community_id.to_string(), + "host": accepted.community_host, + "status": accepted.stage.to_string(), + })), + )) +} + /// List communities where a pubkey currently holds the `owner` role. pub async fn list_owned_communities( State(state): State>, @@ -424,6 +544,12 @@ pub async fn transfer_community( "owner_conflict: the current owner no longer matches expected_owner_pubkey", )); } + buzz_db::relay_members::TransferResult::DeletionPending => { + return Err(api_error( + StatusCode::CONFLICT, + "community deletion is pending", + )); + } buzz_db::relay_members::TransferResult::LimitReached => { return Err(api_error( StatusCode::CONFLICT, @@ -669,6 +795,29 @@ mod postgres_tests { signed_operator_request(state, operator, "POST", "/operator/communities", Some(body)).await } + async fn archive_for_owner_deletion(state: &AppState, host: &str, owner: &Keys) { + state + .db + .archive_community_owned_by( + host, + &owner.public_key().to_hex(), + &buzz_core::tenant::relay_url_authority(&state.config.relay_url), + ) + .await + .expect("archive community") + .expect("owned community"); + } + + fn owner_delete_body(host: &str, owner: &Keys, request_id: Uuid) -> String { + serde_json::json!({ + "host": host, + "owner_pubkey": owner.public_key().to_hex(), + "request_id": request_id, + "acknowledgement_version": 1, + }) + .to_string() + } + fn is_member_tag(tag: &Tag, pubkey: &str, role: &str) -> bool { let values = tag.as_slice(); values.first().is_some_and(|value| value == "member") @@ -738,6 +887,125 @@ mod postgres_tests { assert_eq!(response.status(), StatusCode::FORBIDDEN); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_delete_endpoint_accepts_archived_owner_intent_without_running_inventory() { + let operator = Keys::generate(); + let owner = Keys::generate(); + let Some(state) = operator_test_state(std::slice::from_ref(&operator)).await else { + return; + }; + let host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(Arc::clone(&state), &operator, &host, &owner) + .await + .status(), + StatusCode::OK + ); + archive_for_owner_deletion(&state, &host, &owner).await; + let request_id = Uuid::new_v4(); + let response = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(owner_delete_body(&host, &owner, request_id)), + ) + .await; + + assert_eq!(response.status(), StatusCode::ACCEPTED); + let json = read_json(response).await; + assert_eq!(json["request_id"], request_id.to_string()); + assert_eq!(json["status"], "submitted"); + let request = state + .db + .deletion_store() + .get(request_id) + .await + .expect("persisted deletion request"); + assert_eq!(request.stage, buzz_db::deletion::DeletionStage::Submitted); + assert!(request.inventory_manifest.is_none()); + assert!(request.inventory_digest.is_none()); + assert_eq!( + request.mediating_operator_pubkey.as_deref(), + Some(operator.public_key().to_hex().as_str()) + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_delete_endpoint_rejects_protected_host_and_malformed_or_mismatched_target() { + let operator = Keys::generate(); + let owner = Keys::generate(); + let Some(state) = operator_test_state(std::slice::from_ref(&operator)).await else { + return; + }; + let protected_host = buzz_core::tenant::relay_url_authority(&state.config.relay_url); + let protected = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(owner_delete_body(&protected_host, &owner, Uuid::new_v4())), + ) + .await; + assert_eq!(protected.status(), StatusCode::CONFLICT); + + let malformed = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some( + serde_json::json!({ + "host": "https://not-an-authority.example/path", + "owner_pubkey": "not-a-pubkey", + "request_id": "not-a-uuid", + "acknowledgement_version": 1, + }) + .to_string(), + ), + ) + .await; + assert_eq!(malformed.status(), StatusCode::BAD_REQUEST); + + let host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(Arc::clone(&state), &operator, &host, &owner) + .await + .status(), + StatusCode::OK + ); + archive_for_owner_deletion(&state, &host, &owner).await; + let request_id = Uuid::new_v4(); + let first = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(owner_delete_body(&host, &owner, request_id)), + ) + .await; + assert_eq!(first.status(), StatusCode::ACCEPTED); + let other_host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(Arc::clone(&state), &operator, &other_host, &owner) + .await + .status(), + StatusCode::OK + ); + archive_for_owner_deletion(&state, &other_host, &owner).await; + let mismatched = signed_operator_request( + state, + &operator, + "POST", + "/operator/communities/delete", + Some(owner_delete_body(&other_host, &owner, request_id)), + ) + .await; + assert_eq!(mismatched.status(), StatusCode::CONFLICT); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn post_operator_body_requires_payload_tag() { diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index 61aedf70be0..5c9bf04db6a 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -96,6 +96,10 @@ pub fn build_router(state: Arc) -> Router { "/operator/communities/unarchive", post(api::operator::unarchive_community), ) + .route( + "/operator/communities/delete", + post(api::operator::delete_community), + ) .route( "/operator/communities/availability", get(api::operator::community_availability), @@ -710,7 +714,7 @@ mod tests { async fn readiness_state(evaluator: Arc) -> Arc { let mut config = crate::config::Config::from_env().expect("default config loads"); config.require_relay_membership = false; - config.database_url = "postgres://buzz:buzz_dev@127.0.0.1:1/buzz".to_string(); + config.database_url = "postgres://buzz:buzz_dev@127.0.0.1:1/buzz".to_string(); // sadscan:disable np.postgres.1 -- local test-only credentials config.redis_url = "redis://127.0.0.1:1".to_string(); let pool = sqlx::PgPool::connect_lazy(&config.database_url).expect("lazy pg pool"); let db = buzz_db::Db::from_pool(pool.clone()); diff --git a/migrations/0050_owner_community_deletion_admission.sql b/migrations/0050_owner_community_deletion_admission.sql new file mode 100644 index 00000000000..3fc78a9d92c --- /dev/null +++ b/migrations/0050_owner_community_deletion_admission.sql @@ -0,0 +1,66 @@ +-- Structured provenance for owner-origin whole-community deletion requests. +-- +-- The existing request UUID is the stable correlation/idempotency identity. +-- Owner admission supplies that UUID instead of creating a second identity +-- column, while these bounded columns distinguish owner intent from the +-- deployment operator that mediated it. +SET LOCAL lock_timeout = '5s'; + +ALTER TABLE community_deletion_requests + ADD COLUMN request_origin TEXT NOT NULL DEFAULT 'operator' + CHECK (request_origin IN ('operator', 'owner')), + ADD COLUMN owner_pubkey TEXT, + ADD COLUMN mediating_operator_pubkey TEXT, + ADD COLUMN acknowledgement_version INTEGER, + ADD CONSTRAINT community_deletion_owner_provenance CHECK ( + (request_origin = 'operator' + AND owner_pubkey IS NULL + AND mediating_operator_pubkey IS NULL + AND acknowledgement_version IS NULL) + OR + (request_origin = 'owner' + AND owner_pubkey ~ '^[0-9a-f]{64}$' + AND mediating_operator_pubkey ~ '^[0-9a-f]{64}$' + AND acknowledgement_version BETWEEN 1 AND 32767 + AND requested_by = owner_pubkey) + ); + +CREATE OR REPLACE FUNCTION prevent_community_deletion_request_retargeting() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.community_id IS DISTINCT FROM OLD.community_id + OR NEW.community_host IS DISTINCT FROM OLD.community_host + THEN + RAISE EXCEPTION 'community deletion target identity is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF NEW.request_origin IS DISTINCT FROM OLD.request_origin + OR NEW.owner_pubkey IS DISTINCT FROM OLD.owner_pubkey + OR NEW.mediating_operator_pubkey IS DISTINCT FROM OLD.mediating_operator_pubkey + OR NEW.acknowledgement_version IS DISTINCT FROM OLD.acknowledgement_version + THEN + RAISE EXCEPTION 'community deletion request provenance is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF OLD.inventory_frozen_at IS NOT NULL AND ( + NEW.schema_manifest IS DISTINCT FROM OLD.schema_manifest + OR NEW.storage_manifest IS DISTINCT FROM OLD.storage_manifest + OR NEW.inventory_manifest IS DISTINCT FROM OLD.inventory_manifest + OR NEW.inventory_digest IS DISTINCT FROM OLD.inventory_digest + OR NEW.inventory_frozen_at IS DISTINCT FROM OLD.inventory_frozen_at + ) THEN + RAISE EXCEPTION 'frozen community deletion inventory is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF OLD.destructive_storage_frozen_at IS NOT NULL AND ( + NEW.destructive_storage_manifest IS DISTINCT FROM OLD.destructive_storage_manifest + OR NEW.destructive_storage_frozen_at IS DISTINCT FROM OLD.destructive_storage_frozen_at + ) THEN + RAISE EXCEPTION 'frozen destructive storage manifest is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN NEW; +END; +$$; diff --git a/schema/schema.sql b/schema/schema.sql index 1a0a849f7ac..d18e2eec513 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1232,6 +1232,11 @@ CREATE TABLE community_deletion_requests ( 'logically_verified', 'retention_pending', 'aborted' )), requested_by TEXT NOT NULL, + request_origin TEXT NOT NULL DEFAULT 'operator' + CHECK (request_origin IN ('operator', 'owner')), + owner_pubkey TEXT, + mediating_operator_pubkey TEXT, + acknowledgement_version INTEGER, reason TEXT, schema_manifest JSONB, storage_manifest JSONB, @@ -1268,6 +1273,18 @@ CREATE TABLE community_deletion_requests ( CHECK ((aborted_at IS NULL) = (aborted_by IS NULL)), CHECK ((aborted_at IS NULL) = (abort_reason IS NULL)), CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)), + CONSTRAINT community_deletion_owner_provenance CHECK ( + (request_origin = 'operator' + AND owner_pubkey IS NULL + AND mediating_operator_pubkey IS NULL + AND acknowledgement_version IS NULL) + OR + (request_origin = 'owner' + AND owner_pubkey ~ '^[0-9a-f]{64}$' + AND mediating_operator_pubkey ~ '^[0-9a-f]{64}$' + AND acknowledgement_version BETWEEN 1 AND 32767 + AND requested_by = owner_pubkey) + ), UNIQUE (id, community_id, inventory_digest) ); CREATE UNIQUE INDEX community_deletion_requests_active_community @@ -1293,7 +1310,7 @@ CREATE TABLE community_deletion_approvals ( ON DELETE RESTRICT ); -CREATE FUNCTION prevent_community_deletion_request_retargeting() +CREATE OR REPLACE FUNCTION prevent_community_deletion_request_retargeting() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1304,6 +1321,14 @@ BEGIN RAISE EXCEPTION 'community deletion target identity is immutable' USING ERRCODE = 'integrity_constraint_violation'; END IF; + IF NEW.request_origin IS DISTINCT FROM OLD.request_origin + OR NEW.owner_pubkey IS DISTINCT FROM OLD.owner_pubkey + OR NEW.mediating_operator_pubkey IS DISTINCT FROM OLD.mediating_operator_pubkey + OR NEW.acknowledgement_version IS DISTINCT FROM OLD.acknowledgement_version + THEN + RAISE EXCEPTION 'community deletion request provenance is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; IF OLD.inventory_frozen_at IS NOT NULL AND ( NEW.schema_manifest IS DISTINCT FROM OLD.schema_manifest OR NEW.storage_manifest IS DISTINCT FROM OLD.storage_manifest From acf561219f4223bc6090ce5c82555ee294d11ea4 Mon Sep 17 00:00:00 2001 From: tornquist Date: Wed, 23 Sep 2026 00:05:22 +0000 Subject: [PATCH 02/46] Fence archived community owner rotation Co-authored-by: Codex Signed-off-by: tornquist --- ARCHITECTURE.md | 6 + crates/buzz-db/src/store/deletion.rs | 377 +++++++++++++++++- crates/buzz-db/src/store/relay_members.rs | 227 +++++++++-- crates/buzz-relay/src/api/operator.rs | 74 +++- .../src/handlers/community_provisioning.rs | 32 +- 5 files changed, 653 insertions(+), 63 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7d7da313d7b..ede53a4b838 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -26,6 +26,12 @@ management lists suppress the archived row. Replaying the same UUID converges to its current stage; a different UUID conflicts with the existing one-active- request invariant until that request is aborted. +Ownership is mutable only while a community is active. Archiving freezes the +current owner. Normal transfer and deployment-root legacy convergence take the +same community-row lock as owner-deletion admission, then reject archived, +quiescing, deleted, or deletion-pending rotation without changing membership. +Initial owner bootstrap for a newly created community remains supported. + Buzz is a Rust monorepo, licensed Apache 2.0 under Block, Inc. --- diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 090aa973223..01e9a09c8b1 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -3597,9 +3597,11 @@ mod tests { mod postgres_tests { use super::*; use crate::{ - relay_members::TransferResult, CreateCommunityWithOwnerResult, Db, DbConfig, - UnarchiveCommunityResult, + relay_members::{ProvisionOwnerResult, TransferResult}, + CreateCommunityWithOwnerResult, Db, DbConfig, UnarchiveCommunityResult, }; + use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; + use std::str::FromStr; async fn store() -> (Db, DeletionStore) { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") @@ -3685,6 +3687,188 @@ mod postgres_tests { (host, owner, created.id) } + struct AdmissionInsertGate { + connection: sqlx::pool::PoolConnection, + trigger_name: String, + function_name: String, + first_key: i32, + second_key: i32, + } + + async fn install_admission_insert_gate(db: &Db, request_id: Uuid) -> AdmissionInsertGate { + let suffix = Uuid::new_v4().simple().to_string(); + let trigger_name = format!("owner_admission_gate_{suffix}"); + let function_name = format!("owner_admission_gate_fn_{suffix}"); + let first_key = (request_id.as_u128() as u32 & 0x7fff_ffff) as i32; + let second_key = ((request_id.as_u128() >> 32) as u32 & 0x7fff_ffff) as i32; + let mut connection = db.pool.acquire().await.expect("acquire gate connection"); + sqlx::query("SELECT pg_advisory_lock(712345, 193847)") + .execute(&mut *connection) + .await + .expect("serialize admission gate fixtures"); + sqlx::query("SELECT pg_advisory_lock($1, $2)") + .bind(first_key) + .bind(second_key) + .execute(&mut *connection) + .await + .expect("hold admission gate"); + sqlx::query(AssertSqlSafe(format!( + "CREATE FUNCTION {function_name}() RETURNS trigger LANGUAGE plpgsql AS $$ \ + BEGIN \ + IF NEW.id = '{request_id}'::uuid THEN \ + PERFORM pg_advisory_xact_lock({first_key}, {second_key}); \ + END IF; \ + RETURN NEW; \ + END $$" + ))) + .execute(&db.pool) + .await + .expect("install admission gate function"); + sqlx::query(AssertSqlSafe(format!( + "CREATE TRIGGER {trigger_name} BEFORE INSERT ON community_deletion_requests \ + FOR EACH ROW EXECUTE FUNCTION {function_name}()" + ))) + .execute(&db.pool) + .await + .expect("install admission gate trigger"); + AdmissionInsertGate { + connection, + trigger_name, + function_name, + first_key, + second_key, + } + } + + async fn wait_for_admission_gate(db: &Db, gate: &AdmissionInsertGate) { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let waiting: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pg_locks \ + WHERE locktype = 'advisory' AND classid = $1::oid AND objid = $2::oid \ + AND objsubid = 2 AND NOT granted)", + ) + .bind(gate.first_key) + .bind(gate.second_key) + .fetch_one(&db.pool) + .await + .expect("inspect admission gate"); + if waiting { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("owner admission reached gated insert"); + } + + async fn release_admission_insert_gate(gate: &mut AdmissionInsertGate) { + sqlx::query("SELECT pg_advisory_unlock($1, $2)") + .bind(gate.first_key) + .bind(gate.second_key) + .execute(&mut *gate.connection) + .await + .expect("release admission gate"); + } + + async fn remove_admission_insert_gate(db: &Db, mut gate: AdmissionInsertGate) { + sqlx::query(AssertSqlSafe(format!( + "DROP TRIGGER {} ON community_deletion_requests", + gate.trigger_name + ))) + .execute(&db.pool) + .await + .expect("remove admission gate trigger"); + sqlx::query(AssertSqlSafe(format!( + "DROP FUNCTION {}()", + gate.function_name + ))) + .execute(&db.pool) + .await + .expect("remove admission gate function"); + sqlx::query("SELECT pg_advisory_unlock(712345, 193847)") + .execute(&mut *gate.connection) + .await + .expect("release admission fixture serialization"); + } + + async fn contender_db(application_name: &str) -> Db { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + let options = PgConnectOptions::from_str(&database_url) + .expect("parse test database URL") + .application_name(application_name); + let pool = PgPoolOptions::new() + .max_connections(1) + .connect_with(options) + .await + .expect("connect contender DB"); + Db::from_pool(pool) + } + + async fn wait_for_contender_lock(db: &Db, application_name: &str) { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let waiting: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity \ + WHERE datname = current_database() AND application_name = $1 \ + AND wait_event_type = 'Lock')", + ) + .bind(application_name) + .fetch_one(&db.pool) + .await + .expect("inspect contender lock"); + if waiting { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("contender reached community row lock"); + } + + async fn membership_roles(db: &Db, community: CommunityId) -> Vec<(String, String)> { + sqlx::query_as( + "SELECT pubkey, role FROM relay_members WHERE community_id = $1 ORDER BY pubkey", + ) + .bind(community.as_uuid()) + .fetch_all(&db.pool) + .await + .expect("read membership roles") + } + + async fn assert_owner_admission_is_only_committed_mutation( + db: &Db, + community: CommunityId, + request_id: Uuid, + ) { + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM community_deletion_requests \ + WHERE id = $1 AND community_id = $2 AND stage = 'submitted'", + ) + .bind(request_id) + .bind(community.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("count admitted request"), + 1 + ); + assert!( + sqlx::query_scalar::<_, bool>( + "SELECT archived_at IS NOT NULL FROM communities WHERE id = $1", + ) + .bind(community.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("read archive state"), + "the losing mutation must not clear archive state" + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn owner_admission_requires_current_owner_and_archived_active_target() { @@ -3754,19 +3938,196 @@ mod postgres_tests { .expect("non-owner admission result"), OwnerDeletionAdmission::NotFoundOrNotOwner ); - assert!(matches!( + assert_eq!( db.transfer_ownership(created.id, &replacement, &owner) .await - .expect("rotate owner"), - TransferResult::Transferred { .. } + .expect("reject archived owner rotation"), + TransferResult::LifecycleConflict + ); + assert!(matches!( + store + .admit_owner_request(&host, &owner, operator, 1, Uuid::new_v4()) + .await + .expect("current-owner admission result"), + OwnerDeletionAdmission::Accepted(_) )); - assert_eq!( + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn accepted_owner_deletion_blocks_legacy_owner_convergence_without_membership_change() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let replacement = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let before = membership_roles(&db, community).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + assert!(matches!( store .admit_owner_request(&host, &owner, operator, 1, Uuid::new_v4()) .await - .expect("stale-owner admission result"), - OwnerDeletionAdmission::NotFoundOrNotOwner + .expect("admit owner request"), + OwnerDeletionAdmission::Accepted(_) + )); + + assert_eq!( + db.provision_owner(community, &replacement) + .await + .expect("legacy convergence result"), + ProvisionOwnerResult::DeletionPending ); + assert_eq!(membership_roles(&db, community).await, before); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn concurrent_owner_admission_serializes_before_normal_transfer() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let replacement = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let before = membership_roles(&db, community).await; + let request_id = Uuid::new_v4(); + let mut gate = install_admission_insert_gate(&db, request_id).await; + let admission = tokio::spawn({ + let store = store.clone(); + let host = host.clone(); + let owner = owner.clone(); + async move { + store + .admit_owner_request( + &host, + &owner, + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + 1, + request_id, + ) + .await + } + }); + wait_for_admission_gate(&db, &gate).await; + + let application_name = format!("owner-transfer-{}", Uuid::new_v4().simple()); + let contender = contender_db(&application_name).await; + let transfer = tokio::spawn({ + let owner = owner.clone(); + let replacement = replacement.clone(); + async move { + contender + .transfer_ownership(community, &replacement, &owner) + .await + } + }); + wait_for_contender_lock(&db, &application_name).await; + release_admission_insert_gate(&mut gate).await; + + let admission_result = admission.await.expect("join admission").expect("admission"); + let transfer_result = transfer.await.expect("join transfer").expect("transfer"); + remove_admission_insert_gate(&db, gate).await; + assert!(matches!( + admission_result, + OwnerDeletionAdmission::Accepted(_) + )); + assert_eq!(transfer_result, TransferResult::DeletionPending); + assert_eq!(membership_roles(&db, community).await, before); + assert_owner_admission_is_only_committed_mutation(&db, community, request_id).await; + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn concurrent_owner_admission_serializes_before_unarchive() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let before = membership_roles(&db, community).await; + let request_id = Uuid::new_v4(); + let mut gate = install_admission_insert_gate(&db, request_id).await; + let admission = tokio::spawn({ + let store = store.clone(); + let host = host.clone(); + let owner = owner.clone(); + async move { + store + .admit_owner_request( + &host, + &owner, + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + 1, + request_id, + ) + .await + } + }); + wait_for_admission_gate(&db, &gate).await; + + let application_name = format!("owner-unarchive-{}", Uuid::new_v4().simple()); + let contender = contender_db(&application_name).await; + let unarchive = tokio::spawn({ + let host = host.clone(); + let owner = owner.clone(); + async move { contender.unarchive_community_owned_by(&host, &owner).await } + }); + wait_for_contender_lock(&db, &application_name).await; + release_admission_insert_gate(&mut gate).await; + + let admission_result = admission.await.expect("join admission").expect("admission"); + let unarchive_result = unarchive.await.expect("join unarchive").expect("unarchive"); + remove_admission_insert_gate(&db, gate).await; + assert!(matches!( + admission_result, + OwnerDeletionAdmission::Accepted(_) + )); + assert_eq!(unarchive_result, UnarchiveCommunityResult::DeletionPending); + assert_eq!(membership_roles(&db, community).await, before); + assert_owner_admission_is_only_committed_mutation(&db, community, request_id).await; + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn concurrent_owner_admission_serializes_before_legacy_owner_rotation() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let replacement = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let before = membership_roles(&db, community).await; + let request_id = Uuid::new_v4(); + let mut gate = install_admission_insert_gate(&db, request_id).await; + let admission = tokio::spawn({ + let store = store.clone(); + let host = host.clone(); + let owner = owner.clone(); + async move { + store + .admit_owner_request( + &host, + &owner, + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + 1, + request_id, + ) + .await + } + }); + wait_for_admission_gate(&db, &gate).await; + + let application_name = format!("owner-legacy-{}", Uuid::new_v4().simple()); + let contender = contender_db(&application_name).await; + let provision = tokio::spawn({ + let replacement = replacement.clone(); + async move { contender.provision_owner(community, &replacement).await } + }); + wait_for_contender_lock(&db, &application_name).await; + release_admission_insert_gate(&mut gate).await; + + let admission_result = admission.await.expect("join admission").expect("admission"); + let provision_result = provision + .await + .expect("join legacy convergence") + .expect("legacy convergence"); + remove_admission_insert_gate(&db, gate).await; + assert!(matches!( + admission_result, + OwnerDeletionAdmission::Accepted(_) + )); + assert_eq!(provision_result, ProvisionOwnerResult::DeletionPending); + assert_eq!(membership_roles(&db, community).await, before); + assert_owner_admission_is_only_committed_mutation(&db, community, request_id).await; } #[tokio::test] diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index a54adc139ee..3bd09c625fe 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -366,8 +366,10 @@ pub async fn update_relay_member_role( /// Ensures the configured owner pubkey holds the `"owner"` role *in /// `community`*, and demotes any other owners in that community to `"admin"`. /// This handles owner rotation: if `RELAY_OWNER_PUBKEY` changes, the old owner -/// is automatically demoted. Scoped to one community — an owner of community A -/// is never bootstrapped into community B. +/// is automatically demoted only while the community is active and has no +/// durable deletion intent. Scoped to one community — an owner of community A +/// is never bootstrapped into community B. Initial insertion and exact-owner +/// convergence remain allowed because neither rotates existing ownership. /// /// Runs in a single transaction. Safe to call at every startup — idempotent. /// @@ -383,13 +385,92 @@ pub async fn bootstrap_owner( community: CommunityId, owner_pubkey: &str, ) -> Result<()> { - bootstrap_owner_with_operation( + match bootstrap_owner_with_operation( pool, community, owner_pubkey, observability::WriterOperation::Bootstrap, ) - .await + .await? + { + ProvisionOwnerResult::Applied => Ok(()), + ProvisionOwnerResult::LifecycleConflict => Err(DbError::AccessDenied( + "community lifecycle freezes owner rotation".to_string(), + )), + ProvisionOwnerResult::DeletionPending => Err(DbError::AccessDenied( + "community deletion is pending".to_string(), + )), + } +} + +#[derive(Clone, Copy)] +enum OwnerMutationMode { + Transfer, + Converge, +} + +enum OwnerMutationAdmission { + Allowed(Vec), + NotFound, + LifecycleConflict, + DeletionPending, +} + +async fn lock_owner_mutation_admission( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + community: CommunityId, + proposed_owner: &str, + mode: OwnerMutationMode, +) -> Result { + let target = sqlx::query( + "SELECT archived_at, deletion_state, deleted_at FROM communities \ + WHERE id = $1 FOR UPDATE", + ) + .bind(community.as_uuid()) + .fetch_optional(&mut **tx) + .await?; + let Some(target) = target else { + return Ok(OwnerMutationAdmission::NotFound); + }; + let existing_owners: Vec = sqlx::query_scalar( + "SELECT pubkey FROM relay_members \ + WHERE community_id = $1 AND role = 'owner' \ + ORDER BY pubkey FOR UPDATE", + ) + .bind(community.as_uuid()) + .fetch_all(&mut **tx) + .await?; + + let changes_existing_owner = match mode { + OwnerMutationMode::Transfer => true, + OwnerMutationMode::Converge => { + !(existing_owners.is_empty() + || existing_owners.len() == 1 && existing_owners[0] == proposed_owner) + } + }; + if !changes_existing_owner { + return Ok(OwnerMutationAdmission::Allowed(existing_owners)); + } + + let deletion_pending: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests \ + WHERE community_id = $1 AND stage <> 'aborted')", + ) + .bind(community.as_uuid()) + .fetch_one(&mut **tx) + .await?; + if deletion_pending { + return Ok(OwnerMutationAdmission::DeletionPending); + } + + let archived_at: Option> = target.try_get("archived_at")?; + let deletion_state: String = target.try_get("deletion_state")?; + let deleted_at: Option> = target.try_get("deleted_at")?; + if archived_at.is_some() || deletion_state != "active" || deleted_at.is_some() { + return Ok(OwnerMutationAdmission::LifecycleConflict); + } + + Ok(OwnerMutationAdmission::Allowed(existing_owners)) } async fn bootstrap_owner_with_operation( @@ -397,11 +478,25 @@ async fn bootstrap_owner_with_operation( community: CommunityId, owner_pubkey: &str, operation: observability::WriterOperation, -) -> Result<()> { +) -> Result { let pubkey = owner_pubkey.to_ascii_lowercase(); let connection = observability::acquire_writer(pool, operation).await?; let mut tx = sqlx::Transaction::begin(connection, None).await?; + match lock_owner_mutation_admission(&mut tx, community, &pubkey, OwnerMutationMode::Converge) + .await? + { + OwnerMutationAdmission::Allowed(_) => {} + OwnerMutationAdmission::NotFound | OwnerMutationAdmission::LifecycleConflict => { + tx.rollback().await?; + return Ok(ProvisionOwnerResult::LifecycleConflict); + } + OwnerMutationAdmission::DeletionPending => { + tx.rollback().await?; + return Ok(ProvisionOwnerResult::DeletionPending); + } + } + // 1. Upsert the configured owner for this community. sqlx::query( "INSERT INTO relay_members (community_id, pubkey, role, added_by) \ @@ -424,7 +519,7 @@ async fn bootstrap_owner_with_operation( .await?; tx.commit().await?; - Ok(()) + Ok(ProvisionOwnerResult::Applied) } /// The result of a transfer-ownership attempt. @@ -444,6 +539,8 @@ pub enum TransferResult { /// concurrent transfer or owner rotation has already changed ownership. /// The caller must NOT retry blindly — re-read ownership and re-evaluate. OwnerConflict, + /// The community is archived, quiescing, or deleted; ownership is frozen. + LifecycleConflict, /// Durable deletion intent exists and wins over ownership mutation. DeletionPending, /// The transferee already owns the maximum number of communities. @@ -452,6 +549,17 @@ pub enum TransferResult { LimitReached, } +/// Result of converging an owner through deployment-root provisioning. +#[derive(Debug, PartialEq, Eq)] +pub enum ProvisionOwnerResult { + /// The initial owner was inserted or existing ownership converged. + Applied, + /// The community lifecycle freezes ownership mutation. + LifecycleConflict, + /// Durable deletion intent exists and wins over owner convergence. + DeletionPending, +} + /// Default maximum number of communities a single pubkey can own. Enforced at /// the relay layer — the authoritative layer — so that concurrent transfers or /// transfer-vs-create races cannot both pass a preflight count. @@ -502,7 +610,8 @@ pub fn owner_count_advisory_lock_key(pubkey_hex: &str) -> i64 { /// so that concurrent transfers to the same recipient serialize. The same /// lock key is also used by `Db::create_community_with_owner` to prevent /// transfer-vs-create races. -/// 2. Locks the community row and rejects any non-aborted deletion request. +/// 2. Locks the community row and rejects archive/non-active lifecycle or any +/// non-aborted deletion request. /// Owner-deletion admission takes the same row lock, so whichever operation /// commits first makes the other re-evaluate and conflict. /// 3. Locks the current owner row `FOR UPDATE` and verifies @@ -538,40 +647,28 @@ pub async fn transfer_ownership( ) .await?; - let community_exists = - sqlx::query_scalar::<_, Uuid>("SELECT id FROM communities WHERE id = $1 FOR UPDATE") - .bind(community.as_uuid()) - .fetch_optional(&mut *tx) - .await? - .is_some(); - if !community_exists { - tx.rollback().await?; - return Ok(TransferResult::NoOwner); - } - let deletion_pending: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM community_deletion_requests \ - WHERE community_id = $1 AND stage <> 'aborted')", - ) - .bind(community.as_uuid()) - .fetch_one(&mut *tx) - .await?; - if deletion_pending { - tx.rollback().await?; - return Ok(TransferResult::DeletionPending); - } - - // 3. Lock the current owner row FOR UPDATE and verify the expected owner. - // FOR UPDATE prevents the stale-owner race: a concurrent transfer that - // already changed the owner will block on this lock until our txn - // completes (or vice versa), and the expected_owner check will fail. - let existing_owners: Vec = sqlx::query_scalar( - "SELECT pubkey FROM relay_members \ - WHERE community_id = $1 AND role = 'owner' \ - FOR UPDATE", + let existing_owners = match lock_owner_mutation_admission( + &mut tx, + community, + &pubkey, + OwnerMutationMode::Transfer, ) - .bind(community.as_uuid()) - .fetch_all(&mut *tx) - .await?; + .await? + { + OwnerMutationAdmission::Allowed(existing_owners) => existing_owners, + OwnerMutationAdmission::NotFound => { + tx.rollback().await?; + return Ok(TransferResult::NoOwner); + } + OwnerMutationAdmission::LifecycleConflict => { + tx.rollback().await?; + return Ok(TransferResult::LifecycleConflict); + } + OwnerMutationAdmission::DeletionPending => { + tx.rollback().await?; + return Ok(TransferResult::DeletionPending); + } + }; if existing_owners.is_empty() { tx.rollback().await?; @@ -825,7 +922,11 @@ impl Db { /// Ensure an owner during operator-driven community provisioning. #[datastore_span(name = "provision_owner", system = "postgresql")] - pub async fn provision_owner(&self, community: CommunityId, owner_pubkey: &str) -> Result<()> { + pub async fn provision_owner( + &self, + community: CommunityId, + owner_pubkey: &str, + ) -> Result { bootstrap_owner_with_operation( &self.pool, community, @@ -1368,6 +1469,50 @@ mod postgres_tests { assert_role(&pool, community, &owner, "owner").await; } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn transfer_ownership_rejects_archived_community_without_membership_change() { + let pool = setup_pool().await; + let (community, owner) = owned_community(&pool).await; + let replacement = test_pubkey(); + sqlx::query("UPDATE communities SET archived_at = now() WHERE id = $1") + .bind(community.as_uuid()) + .execute(&pool) + .await + .expect("archive community"); + + let result = transfer_ownership(&pool, community, &replacement, &owner) + .await + .expect("transfer archived community"); + + assert_eq!(result, TransferResult::LifecycleConflict); + assert_role(&pool, community, &owner, "owner").await; + assert!( + get_relay_member(&pool, community, &replacement) + .await + .expect("get replacement") + .is_none(), + "archived transfer must not add the replacement owner" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn provision_owner_still_bootstraps_initial_owner() { + let pool = setup_pool().await; + let community = make_test_community(&pool).await; + let owner = test_pubkey(); + let db = Db::from_pool(pool.clone()); + + assert_eq!( + db.provision_owner(community, &owner) + .await + .expect("provision initial owner"), + ProvisionOwnerResult::Applied + ); + assert_role(&pool, community, &owner, "owner").await; + } + /// Transferring a community with no owner row returns `NoOwner`. #[tokio::test] #[ignore = "requires Postgres"] diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index c672f837227..20c80efd3d9 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -182,7 +182,11 @@ pub async fn provision_community( Err(msg) if msg.starts_with("actor not authorized") => { Err(api_error(StatusCode::FORBIDDEN, &msg)) } - Err(msg) if msg == "community already exists" || msg.starts_with("limit_reached:") => { + Err(msg) + if msg == "community already exists" + || msg.starts_with("limit_reached:") + || msg.starts_with("owner_conflict:") => + { Err(api_error(StatusCode::CONFLICT, &msg)) } Err(msg) @@ -544,6 +548,12 @@ pub async fn transfer_community( "owner_conflict: the current owner no longer matches expected_owner_pubkey", )); } + buzz_db::relay_members::TransferResult::LifecycleConflict => { + return Err(api_error( + StatusCode::CONFLICT, + "community must be active to transfer ownership", + )); + } buzz_db::relay_members::TransferResult::DeletionPending => { return Err(api_error( StatusCode::CONFLICT, @@ -1346,6 +1356,68 @@ mod postgres_tests { assert_snapshot_roles(&state, community.id, &[(&owner_hex, "owner")]).await; } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn archived_legacy_owner_convergence_returns_conflict_without_membership_change() { + let operator = Keys::generate(); + let owner = Keys::generate(); + let replacement = Keys::generate(); + let Some(state) = operator_test_state(std::slice::from_ref(&operator)).await else { + return; + }; + let host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(state.clone(), &operator, &host, &owner) + .await + .status(), + StatusCode::OK + ); + let community = state + .db + .lookup_community_by_host(&host) + .await + .expect("lookup community") + .expect("community exists"); + archive_for_owner_deletion(&state, &host, &owner).await; + + let body = serde_json::json!({ + "host": host, + "initial_owner_pubkey": replacement.public_key().to_hex(), + "create_only": false, + }) + .to_string(); + let response = signed_operator_request( + state.clone(), + &operator, + "POST", + "/operator/communities", + Some(body), + ) + .await; + + assert_eq!(response.status(), StatusCode::CONFLICT); + assert_eq!( + read_json(response).await["error"], + "owner_conflict: community must be active to rotate ownership" + ); + assert_eq!( + state + .db + .get_relay_member(community.id, &owner.public_key().to_hex()) + .await + .expect("get owner") + .expect("owner exists") + .role, + "owner" + ); + assert!(state + .db + .get_relay_member(community.id, &replacement.public_key().to_hex()) + .await + .expect("get replacement") + .is_none()); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn fresh_host_at_owner_limit_returns_limit_reached_conflict() { diff --git a/crates/buzz-relay/src/handlers/community_provisioning.rs b/crates/buzz-relay/src/handlers/community_provisioning.rs index 229b5f37161..1b13e6d921a 100644 --- a/crates/buzz-relay/src/handlers/community_provisioning.rs +++ b/crates/buzz-relay/src/handlers/community_provisioning.rs @@ -21,9 +21,9 @@ //! { "host": "acme.communities.buzz.xyz", "initial_owner_pubkey": "" } //! ``` //! -//! `initial_owner_pubkey` is optional. When present for an existing community, -//! it rotates that community owner through the same bootstrap path used by -//! `RELAY_OWNER_PUBKEY`; relay operators are deployment-root authorities. +//! `initial_owner_pubkey` is optional. Existing-community convergence may +//! rotate ownership only while the community is active and has no durable +//! deletion intent. Archive freezes ownership. use std::sync::Arc; @@ -238,14 +238,9 @@ async fn publish_membership_snapshot_if_required( /// signer here for the deployment-level `RELAY_OPERATOR_PUBKEYS` allowlist. /// /// Idempotency and owner semantics: the request is idempotent on the host row -/// (re-sending it never duplicates a community). When `initial_owner_pubkey` is -/// present, the owner is (re)bootstrapped via [`buzz_db::Db::bootstrap_owner`] -/// even if the community already existed — any previous owner is demoted to -/// admin, exactly like rotating `RELAY_OWNER_PUBKEY` for the deployment -/// community. This makes a retry after a partial failure (row created, owner -/// bootstrap crashed) converge, at the cost that an operator-signed request can -/// rotate an existing community's owner. The operator allowlist is therefore -/// documented as deployment-root authority, not create-only authority. +/// (re-sending it never duplicates a community). Initial owner bootstrap still +/// converges after a partial create, but rotating an existing owner requires an +/// active, non-archived community with no non-aborted deletion request. pub async fn provision_community( state: &Arc, operator_pubkey: &nostr::PublicKey, @@ -325,11 +320,22 @@ pub async fn provision_community( .map_err(|e| format!("failed to create community: {e}"))?; if let Some(owner_hex) = &initial_owner { - state + match state .db .provision_owner(record.id, owner_hex) .await - .map_err(|e| format!("community provisioned but owner bootstrap failed: {e}"))?; + .map_err(|e| format!("community provisioned but owner bootstrap failed: {e}"))? + { + buzz_db::relay_members::ProvisionOwnerResult::Applied => {} + buzz_db::relay_members::ProvisionOwnerResult::LifecycleConflict => { + return Err( + "owner_conflict: community must be active to rotate ownership".to_string(), + ); + } + buzz_db::relay_members::ProvisionOwnerResult::DeletionPending => { + return Err("owner_conflict: community deletion is pending".to_string()); + } + } publish_membership_snapshot_if_required(state, record.id, &record.host).await; } From 4432b5700067fb80692cfea19baee6cd362191e7 Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:38 +0000 Subject: [PATCH 03/46] Allow privileged abort at the reversible deletion boundary Owner admission records durable intent with no owner-facing cancellation, so an operator needs a recovery path when preparation cannot continue. Abort already reversed approved and fenced requests; extend it to the submitted and inventoried stages, which have destroyed nothing. Aborting releases the durable request fence over owner listing, unarchive, and owner rotation. It reverses deletion intent only: the community stays archived and the owner restores it explicitly. Stages from drained onward stay closed. Lock ordering and the active-serving-write-lease guard are unchanged. Co-Authored-By: Claude Opus 5 Signed-off-by: tornquist --- crates/buzz-db/src/store/deletion.rs | 140 ++++++++++++++++++++++++++- 1 file changed, 138 insertions(+), 2 deletions(-) diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 01e9a09c8b1..c10ef5f6c1b 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -2186,7 +2186,16 @@ impl DeletionStore { Ok(()) } - /// Terminally abort an approved or fenced request before object deletion begins. + /// Terminally abort a request at the reversible pre-destruction boundary. + /// + /// `submitted`, `inventoried`, `approved`, and `fenced` are reversible: + /// nothing tenant-visible has been destroyed, so abort releases the durable + /// request fence over owner listing, unarchive, and owner rotation. Owner + /// admission deliberately has no owner-facing cancellation, so this + /// privileged path is the only recovery when preparation cannot continue. + /// Abort reverses deletion intent, not the owner's archive decision: the + /// community stays archived and the owner restores it explicitly. + /// Stages from `drained` onward have destroyed tenant state and stay closed. pub async fn abort( &self, request_id: Uuid, @@ -2225,7 +2234,10 @@ impl DeletionStore { } if !matches!( request.stage, - DeletionStage::Approved | DeletionStage::Fenced + DeletionStage::Submitted + | DeletionStage::Inventoried + | DeletionStage::Approved + | DeletionStage::Fenced ) { return Err(DbError::DeletionSafety(format!( "deletion {request_id} at stage {} cannot be aborted", @@ -4243,6 +4255,130 @@ mod postgres_tests { ); } + /// Privileged recovery abort at the reversible pre-approval boundary. + /// + /// Owner admission has no owner-facing cancellation. When preparation + /// cannot safely continue, an operator aborts the request; that must + /// release the durable request fence over listing, unarchive, and owner + /// rotation without silently unarchiving the community behind the owner. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn privileged_abort_of_submitted_owner_request_releases_the_request_fence() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let new_owner = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + let OwnerDeletionAdmission::Accepted(request) = store + .admit_owner_request(&host, &owner, operator, 1, request_id) + .await + .expect("admit owner request") + else { + panic!("expected accepted request") + }; + assert_eq!(request.stage, DeletionStage::Submitted); + assert_eq!( + db.transfer_ownership(community, &new_owner, &owner) + .await + .expect("fenced transfer"), + TransferResult::DeletionPending + ); + + let aborted = store + .abort( + request_id, + "recovery-operator", + "owner preparation cannot continue", + ) + .await + .expect("privileged abort at the reversible submitted boundary"); + assert_eq!(aborted.stage, DeletionStage::Aborted); + assert_eq!(aborted.aborted_by.as_deref(), Some("recovery-operator")); + + // Abort reverses deletion intent, not the owner's archive decision. + let (deletion_state, archived_at): (String, Option>) = + sqlx::query_as("SELECT deletion_state, archived_at FROM communities WHERE id = $1") + .bind(community.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("community lifecycle after abort"); + assert_eq!(deletion_state, "active"); + assert!( + archived_at.is_some(), + "abort must not unarchive the community on the owner's behalf" + ); + + assert!( + db.list_communities_owned_by(&owner) + .await + .expect("owner list after abort") + .iter() + .any(|row| row.id == community), + "aborting the request must restore the owner's actionable archived row" + ); + let UnarchiveCommunityResult::Unarchived(restored) = db + .unarchive_community_owned_by(&host, &owner) + .await + .expect("unarchive after abort") + else { + panic!("aborting the request must restore owner-authorized unarchive") + }; + assert_eq!(restored.id, community); + assert_eq!( + db.transfer_ownership(community, &new_owner, &owner) + .await + .expect("transfer after abort"), + TransferResult::Transferred { + previous_owner: Some(owner.clone()), + } + ); + } + + /// The reversible boundary stops at `inventoried`. Once execution has + /// destroyed anything, abort must stay closed. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn privileged_abort_spans_only_the_reversible_pre_destruction_boundary() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + assert_eq!(request.stage, DeletionStage::Inventoried); + let aborted = store + .abort(request.id, "recovery-operator", "inventory needs recovery") + .await + .expect("privileged abort at the inventoried boundary"); + assert_eq!(aborted.stage, DeletionStage::Aborted); + + for irreversible in [ + DeletionStage::Drained, + DeletionStage::BindingsRemoved, + DeletionStage::PostgresPurged, + DeletionStage::CachePurged, + DeletionStage::LogicallyVerified, + DeletionStage::RetentionPending, + ] { + let (later, _) = inventoried_request(&db, &store).await; + sqlx::query("UPDATE community_deletion_requests SET stage = $2 WHERE id = $1") + .bind(later.id) + .bind(irreversible.to_string()) + .execute(&db.pool) + .await + .expect("advance stage"); + let error = store + .abort(later.id, "recovery-operator", "too late") + .await + .expect_err("abort must stay closed after destruction begins"); + assert!( + error.to_string().contains("cannot be aborted"), + "unexpected error at {irreversible}: {error}" + ); + assert_eq!( + store.get(later.id).await.expect("unchanged request").stage, + irreversible, + "a refused abort must not move the request" + ); + } + } + #[tokio::test] #[ignore = "requires Postgres"] async fn approval_boundary_blocks_claim_until_exact_inventory_is_approved() { From 3edd5948df816051cc32344f96841280c0aa3d9e Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 04/46] Reject owner deletion rows that omit provenance The owner branch of community_deletion_owner_provenance tested only the shape of each provenance column. A bare `col ~ '...'` on a NULL column evaluates to NULL, and a CHECK constraint is satisfied by NULL, so `FALSE OR NULL` admitted owner-origin rows with no owner key, no mediating operator, or no acknowledgement version. Reject a NULL in each column before testing its shape. Spell it `NOT (col IS NULL)`: pgschema drops a named CHECK whose body contains `IS NOT NULL` and still exits 0, which would leave the desired-state bootstrap silently unguarded while the migration path stayed correct. Assert one shared case table from both schema sources -- the migration upgrade path and the pgschema desired-state bootstrap -- so the two cannot drift into different owner-provenance guarantees. Co-Authored-By: Claude Opus 5 Signed-off-by: tornquist --- crates/buzz-db/src/runtime/migration.rs | 16 ++ crates/buzz-db/src/store/deletion.rs | 212 ++++++++++++++++++ ...050_owner_community_deletion_admission.sql | 15 ++ schema/schema.sql | 3 + 4 files changed, 246 insertions(+) diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index a6727c4ac72..7b64547c2f5 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -2498,6 +2498,22 @@ mod postgres_tests { assert_eq!(after, vec![(1, Some(true)), (30_179, None), (30_350, None)]); } + /// Migration-upgrade half of the owner-provenance contract. + /// + /// The desired-state bootstrap half lives in + /// `store::deletion::postgres_tests` and asserts the same shared case + /// table, so `schema/schema.sql` cannot admit owner rows the migration + /// path refuses (or the reverse). + #[tokio::test] + #[ignore = "requires Postgres"] + async fn migrated_schema_enforces_owner_provenance_contract() { + let pool = connect_test_pool().await; + reset_public_schema(&pool).await; + run_migrations(&pool).await.expect("run migrations"); + + crate::store::deletion::owner_provenance_contract::assert_contract(&pool).await; + } + #[tokio::test] #[ignore = "requires Postgres"] async fn run_migrations_applies_consolidated_initial_schema_on_fresh_database() { diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index c10ef5f6c1b..073e9cafd08 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -3605,6 +3605,207 @@ mod tests { } } +/// Shared `community_deletion_owner_provenance` contract cases. +/// +/// The same table is asserted against the migration-upgrade schema +/// (`runtime::migration::postgres_tests`) and the desired-state bootstrap +/// schema (`postgres_tests` below), so the two schema sources cannot drift +/// into different owner-provenance guarantees. +#[cfg(test)] +pub(crate) mod owner_provenance_contract { + use sqlx::{PgPool, Row}; + use uuid::Uuid; + + /// Constraint that must reject every malformed owner-provenance row. + pub(crate) const CONSTRAINT: &str = "community_deletion_owner_provenance"; + + const VALID_OWNER: &str = "1111111111111111111111111111111111111111111111111111111111111111"; + const VALID_OPERATOR: &str = "2222222222222222222222222222222222222222222222222222222222222222"; + + /// One rejected owner-origin row: `(case name, owner, mediator, ack, requested_by)`. + type Case = ( + &'static str, + Option<&'static str>, + Option<&'static str>, + Option, + &'static str, + ); + + /// Every owner-origin row the constraint must refuse. + pub(crate) fn rejected_cases() -> Vec { + vec![ + ( + "missing owner_pubkey", + None, + Some(VALID_OPERATOR), + Some(1), + VALID_OWNER, + ), + ( + "missing mediating_operator_pubkey", + Some(VALID_OWNER), + None, + Some(1), + VALID_OWNER, + ), + ( + "missing acknowledgement_version", + Some(VALID_OWNER), + Some(VALID_OPERATOR), + None, + VALID_OWNER, + ), + ( + "owner_pubkey too short", + Some("abc"), + Some(VALID_OPERATOR), + Some(1), + "abc", + ), + ( + "owner_pubkey uppercase hex", + Some("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"), + Some(VALID_OPERATOR), + Some(1), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ), + ( + "owner_pubkey non-hex", + Some("zzzz111111111111111111111111111111111111111111111111111111111111"), + Some(VALID_OPERATOR), + Some(1), + "zzzz111111111111111111111111111111111111111111111111111111111111", + ), + ( + "mediating_operator_pubkey too short", + Some(VALID_OWNER), + Some("abc"), + Some(1), + VALID_OWNER, + ), + ( + "mediating_operator_pubkey uppercase hex", + Some(VALID_OWNER), + Some("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"), + Some(1), + VALID_OWNER, + ), + ( + "acknowledgement_version zero", + Some(VALID_OWNER), + Some(VALID_OPERATOR), + Some(0), + VALID_OWNER, + ), + ( + "acknowledgement_version negative", + Some(VALID_OWNER), + Some(VALID_OPERATOR), + Some(-1), + VALID_OWNER, + ), + ( + "requested_by is not the owner", + Some(VALID_OWNER), + Some(VALID_OPERATOR), + Some(1), + VALID_OPERATOR, + ), + ] + } + + async fn seed_community(pool: &PgPool) -> Uuid { + let host = format!("owner-provenance-{}.example", Uuid::new_v4().simple()); + sqlx::query_scalar::<_, Uuid>("INSERT INTO communities (host) VALUES ($1) RETURNING id") + .bind(&host) + .fetch_one(pool) + .await + .expect("seed owner-provenance community") + } + + async fn insert_owner_row( + pool: &PgPool, + owner: Option<&str>, + mediator: Option<&str>, + acknowledgement: Option, + requested_by: &str, + ) -> Result<(), sqlx::Error> { + let community_id = seed_community(pool).await; + let host: String = sqlx::query("SELECT host FROM communities WHERE id = $1") + .bind(community_id) + .fetch_one(pool) + .await + .expect("seeded host") + .try_get("host") + .expect("host column"); + sqlx::query( + "INSERT INTO community_deletion_requests \ + (id, community_id, community_host, requested_by, request_origin, \ + owner_pubkey, mediating_operator_pubkey, acknowledgement_version) \ + VALUES ($1, $2, $3, $4, 'owner', $5, $6, $7)", + ) + .bind(Uuid::new_v4()) + .bind(community_id) + .bind(host) + .bind(requested_by) + .bind(owner) + .bind(mediator) + .bind(acknowledgement) + .execute(pool) + .await + .map(|_| ()) + } + + /// Assert the live schema refuses every malformed owner row and still + /// admits a well-formed one. + pub(crate) async fn assert_contract(pool: &PgPool) { + for (name, owner, mediator, acknowledgement, requested_by) in rejected_cases() { + let error = insert_owner_row(pool, owner, mediator, acknowledgement, requested_by) + .await + .expect_err(&format!("owner-provenance case must be rejected: {name}")); + let constraint = error + .as_database_error() + .and_then(sqlx::error::DatabaseError::constraint); + assert_eq!( + constraint, + Some(CONSTRAINT), + "case {name:?} must fail {CONSTRAINT}, got: {error}" + ); + } + + // Falsifiability: the constraint must still admit well-formed intent. + insert_owner_row( + pool, + Some(VALID_OWNER), + Some(VALID_OPERATOR), + Some(1), + VALID_OWNER, + ) + .await + .expect("well-formed owner provenance must be accepted"); + + // The operator branch stays exclusive of owner columns. + let community_id = seed_community(pool).await; + let operator_with_owner_columns = sqlx::query( + "INSERT INTO community_deletion_requests \ + (id, community_id, community_host, requested_by, request_origin, owner_pubkey) \ + VALUES ($1, $2, 'operator.example', 'operator', 'operator', $3)", + ) + .bind(Uuid::new_v4()) + .bind(community_id) + .bind(VALID_OWNER) + .execute(pool) + .await; + assert_eq!( + operator_with_owner_columns + .expect_err("operator-origin rows must not carry owner provenance") + .as_database_error() + .and_then(sqlx::error::DatabaseError::constraint), + Some(CONSTRAINT) + ); + } +} + #[cfg(test)] mod postgres_tests { use super::*; @@ -4255,6 +4456,17 @@ mod postgres_tests { ); } + /// Desired-state bootstrap half of the owner-provenance contract. + /// + /// The migration-upgrade half lives in `runtime::migration::postgres_tests` + /// and asserts the same shared case table. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn desired_state_schema_enforces_owner_provenance_contract() { + let (db, _) = store().await; + owner_provenance_contract::assert_contract(&db.pool).await; + } + /// Privileged recovery abort at the reversible pre-approval boundary. /// /// Owner admission has no owner-facing cancellation. When preparation diff --git a/migrations/0050_owner_community_deletion_admission.sql b/migrations/0050_owner_community_deletion_admission.sql index 3fc78a9d92c..25002b32da0 100644 --- a/migrations/0050_owner_community_deletion_admission.sql +++ b/migrations/0050_owner_community_deletion_admission.sql @@ -4,6 +4,18 @@ -- Owner admission supplies that UUID instead of creating a second identity -- column, while these bounded columns distinguish owner intent from the -- deployment operator that mediated it. +-- +-- The owner branch rejects a NULL in every provenance column before testing +-- its shape. A bare `col ~ '...'` on a NULL column yields NULL, and a CHECK is +-- satisfied by NULL, so `FALSE OR NULL` would silently admit an owner-origin +-- row with no owner key, no mediating operator, or no acknowledgement version. +-- +-- The null rejection is spelled `NOT (col IS NULL)` rather than +-- `col IS NOT NULL`: pgschema drops a named CHECK whose body contains +-- `IS NOT NULL` and still exits 0, which would leave the desired-state +-- bootstrap in `schema/schema.sql` silently unguarded while the migration path +-- stayed correct. `store::deletion::owner_provenance_contract` asserts both +-- schema sources against one case table so that divergence fails loudly. SET LOCAL lock_timeout = '5s'; ALTER TABLE community_deletion_requests @@ -19,6 +31,9 @@ ALTER TABLE community_deletion_requests AND acknowledgement_version IS NULL) OR (request_origin = 'owner' + AND NOT (owner_pubkey IS NULL) + AND NOT (mediating_operator_pubkey IS NULL) + AND NOT (acknowledgement_version IS NULL) AND owner_pubkey ~ '^[0-9a-f]{64}$' AND mediating_operator_pubkey ~ '^[0-9a-f]{64}$' AND acknowledgement_version BETWEEN 1 AND 32767 diff --git a/schema/schema.sql b/schema/schema.sql index d18e2eec513..c91cd7375d6 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1280,6 +1280,9 @@ CREATE TABLE community_deletion_requests ( AND acknowledgement_version IS NULL) OR (request_origin = 'owner' + AND NOT (owner_pubkey IS NULL) + AND NOT (mediating_operator_pubkey IS NULL) + AND NOT (acknowledgement_version IS NULL) AND owner_pubkey ~ '^[0-9a-f]{64}$' AND mediating_operator_pubkey ~ '^[0-9a-f]{64}$' AND acknowledgement_version BETWEEN 1 AND 32767 From 2abeed3f65e0d9401e272c2b750c8e763ef99386 Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 05/46] Reach each owner-delete rejection guard separately MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The malformed-body coverage sent a single request carrying several invalid fields at once, including an unparseable `request_id`. Request deserialization runs before the host and pubkey guards, so that request was refused by serde and the guards it claimed to cover were never executed — dropping the `owner_pubkey` validation entirely left the test passing. Give each guard its own request whose remaining fields are valid, so the case reaches the guard under test, and assert no deletion intent persists for any refusal. Co-Authored-By: Claude Opus 5 Signed-off-by: tornquist --- crates/buzz-relay/src/api/operator.rs | 133 +++++++++++++++++++++++++- 1 file changed, 131 insertions(+), 2 deletions(-) diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index 20c80efd3d9..ce792c2c2c8 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -790,6 +790,40 @@ mod postgres_tests { .expect("response") } + /// Send a delete-community request with a caller-controlled `Authorization` + /// header so signature-binding failures can be exercised directly. + async fn raw_owner_delete( + state: Arc, + auth: Option, + extra_header: Option<(&str, String)>, + body: String, + ) -> axum::response::Response { + let mut request = Request::builder() + .method("POST") + .uri("/operator/communities/delete") + .header(header::HOST, INGRESS_HOST) + .header(header::CONTENT_TYPE, "application/json"); + if let Some(auth) = auth { + request = request.header(header::AUTHORIZATION, auth); + } + if let Some((name, value)) = extra_header { + request = request.header(name, value); + } + build_router(state) + .oneshot(request.body(Body::from(body)).expect("request")) + .await + .expect("response") + } + + /// The endpoint must not have persisted intent under `request_id`. + async fn assert_no_persisted_request(state: &AppState, request_id: Uuid, case: &str) { + let found = state.db.deletion_store().get(request_id).await; + assert!( + matches!(found, Err(buzz_db::DbError::NotFound(_))), + "{case}: rejected request must not persist deletion intent" + ); + } + async fn provision_community( state: Arc, operator: &Keys, @@ -961,7 +995,14 @@ mod postgres_tests { .await; assert_eq!(protected.status(), StatusCode::CONFLICT); - let malformed = signed_operator_request( + // Each malformed case keeps every unrelated field valid so it reaches + // the guard under test instead of tripping an earlier one, and none of + // them may leave durable intent behind. + let valid_owner = owner.public_key().to_hex(); + let reachable_host = format!("community-{}.example", Uuid::new_v4().simple()); + + let bad_host_id = Uuid::new_v4(); + let bad_host = signed_operator_request( Arc::clone(&state), &operator, "POST", @@ -969,7 +1010,95 @@ mod postgres_tests { Some( serde_json::json!({ "host": "https://not-an-authority.example/path", + "owner_pubkey": valid_owner, + "request_id": bad_host_id, + "acknowledgement_version": 1, + }) + .to_string(), + ), + ) + .await; + assert_eq!(bad_host.status(), StatusCode::BAD_REQUEST); + assert_no_persisted_request(&state, bad_host_id, "unnormalizable host").await; + + let bad_pubkey_id = Uuid::new_v4(); + let bad_pubkey = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some( + serde_json::json!({ + "host": reachable_host, "owner_pubkey": "not-a-pubkey", + "request_id": bad_pubkey_id, + "acknowledgement_version": 1, + }) + .to_string(), + ), + ) + .await; + assert_eq!(bad_pubkey.status(), StatusCode::BAD_REQUEST); + let bad_pubkey_error = read_json(bad_pubkey).await; + assert!( + bad_pubkey_error + .get("error") + .and_then(Value::as_str) + .unwrap_or_default() + .contains("owner_pubkey"), + "invalid owner_pubkey must reach the pubkey guard: {bad_pubkey_error:?}" + ); + assert_no_persisted_request(&state, bad_pubkey_id, "invalid owner_pubkey").await; + + let bad_version_id = Uuid::new_v4(); + let bad_version = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some( + serde_json::json!({ + "host": reachable_host, + "owner_pubkey": valid_owner, + "request_id": bad_version_id, + "acknowledgement_version": 2, + }) + .to_string(), + ), + ) + .await; + assert_eq!(bad_version.status(), StatusCode::BAD_REQUEST); + assert_no_persisted_request(&state, bad_version_id, "unsupported acknowledgement").await; + + let unknown_host_id = Uuid::new_v4(); + let unknown_host = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some( + serde_json::json!({ + "host": reachable_host, + "owner_pubkey": valid_owner, + "request_id": unknown_host_id, + "acknowledgement_version": 1, + }) + .to_string(), + ), + ) + .await; + assert_eq!(unknown_host.status(), StatusCode::NOT_FOUND); + assert_no_persisted_request(&state, unknown_host_id, "unprovisioned host").await; + + let bad_uuid = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some( + serde_json::json!({ + "host": reachable_host, + "owner_pubkey": valid_owner, "request_id": "not-a-uuid", "acknowledgement_version": 1, }) @@ -977,7 +1106,7 @@ mod postgres_tests { ), ) .await; - assert_eq!(malformed.status(), StatusCode::BAD_REQUEST); + assert_eq!(bad_uuid.status(), StatusCode::BAD_REQUEST); let host = format!("community-{}.example", Uuid::new_v4().simple()); assert_eq!( From 6d82d2b12a826ae74124defd4b9a50aab78edf0b Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 06/46] Add typed deletion drain operator CronJob Co-authored-by: Codex Signed-off-by: tornquist --- ARCHITECTURE.md | 9 + deploy/charts/buzz/Chart.yaml | 4 +- deploy/charts/buzz/README.md | 22 +- .../charts/buzz/templates/_operator-jobs.tpl | 86 +++++++ deploy/charts/buzz/templates/_validate.tpl | 7 + .../templates/deletion-drain-cronjob.yaml | 4 + .../buzz/tests/deletion_drain_test.yaml | 214 ++++++++++++++++++ deploy/charts/buzz/values.schema.json | 28 +++ deploy/charts/buzz/values.yaml | 23 ++ docs/operator-community-deletion.md | 87 +++++++ 10 files changed, 481 insertions(+), 3 deletions(-) create mode 100644 deploy/charts/buzz/templates/_operator-jobs.tpl create mode 100644 deploy/charts/buzz/templates/deletion-drain-cronjob.yaml create mode 100644 deploy/charts/buzz/tests/deletion_drain_test.yaml create mode 100644 docs/operator-community-deletion.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a3fa2af013a..eaf4a47794f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -749,10 +749,19 @@ Subcommands: | `remove-member` | Remove a pubkey from the relay membership list (`--pubkey`, optional `--role` guard); publishes kind:13534 roster | | `list-members` | List all relay members | | `generate-key` | Generate a new Nostr keypair (for bootstrapping) | +| `deletions` | Submit, inspect, approve, abort, unblock, run, or drain durable whole-community deletion requests | +| `storage-snapshot` | Run one isolated S3 accounting scan and publish its complete Postgres snapshot | | `reconcile-channels` | Emit kind:39000/39002 discovery events for channels missing them (idempotent) | The `buzz-admin` binary is shipped in the relay Docker image (`/usr/local/bin/buzz-admin`) and is the recommended way to manage relay membership in production. Use `./run.sh add-member`, `./run.sh remove-member`, and `./run.sh list-members` in Docker Compose deployments. +Kubernetes deployments may schedule the typed one-shot +`buzz-admin deletions drain` command directly. The pod owns its bounded +Postgres/Redis clients and S3 client; it does not call relay HTTP. Durable +requests, leases, retry timing, and checkpoints in Postgres are the handoff and +execution authority, so Kubernetes uses `Forbid` concurrency and zero Job +retries rather than introducing a second retry system. + --- ### buzz-test-client — Integration Test Harness diff --git a/deploy/charts/buzz/Chart.yaml b/deploy/charts/buzz/Chart.yaml index e26a2815fff..55d9db5488f 100644 --- a/deploy/charts/buzz/Chart.yaml +++ b/deploy/charts/buzz/Chart.yaml @@ -7,7 +7,7 @@ description: | PostgreSQL and Redis. Configurable for single-node evaluation (subcharts on) and HA production (external services, existingSecret). type: application -version: 0.1.9 +version: 0.1.10 appVersion: "0.1.0" home: https://github.com/block/buzz sources: @@ -24,7 +24,7 @@ maintainers: annotations: artifacthub.io/changes: | - kind: added - description: Optional isolated storage-accounting CronJob with durable relay snapshots. + description: Optional typed deletion-drain operator CronJob using durable database leases. artifacthub.io/license: Apache-2.0 # Optional eval-only subcharts. Production deploys disable both and point diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index a6b74e87572..288681b0c98 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -12,7 +12,7 @@ This chart has two operating profiles selected by values: ## Quickstart (eval only) ```sh -helm install buzz oci://ghcr.io/block/buzz/charts/buzz --version 0.1.8 \ +helm install buzz oci://ghcr.io/block/buzz/charts/buzz --version 0.1.10 \ --create-namespace --namespace buzz \ --set quickstart=true \ --set postgresql.enabled=true \ @@ -115,6 +115,26 @@ disables that probe through `relay.extraEnv`, `/_readiness` does not test object storage; configuration is still parsed strictly, but reachability and addressing errors surface on the first storage operation. +## Community deletion operator job + +`operatorJobs.deletionDrain` is a disabled-by-default, typed CronJob for +`/usr/local/bin/buzz-admin deletions drain`. It runs inside the relay image with +bounded Job lifetime/history, `concurrencyPolicy: Forbid`, `backoffLimit: 0`, +and no relay HTTP call. Postgres deletion requests, leases, retries, and +checkpoints remain the execution authority. + +The pod receives only `DATABASE_URL`, `REDIS_URL`, and required S3 +configuration/credential variables. It does not receive the relay private key, +git-hook secret, relay URL, service-account token, service links, or a generic +environment registry. Schedule, +deadline, history, termination grace, resources, service account, pod labels, +and pod annotations are independently configurable under +`operatorJobs.deletionDrain`. + +See [`docs/operator-community-deletion.md`](../../../docs/operator-community-deletion.md) +for enablement, permissions, the staffed first-run procedure, failure recovery, +and the current explicit approval/alerting boundaries. + ### Early-startup telemetry contract `buzz_process_lifecycle` JSON records are the authoritative history for the diff --git a/deploy/charts/buzz/templates/_operator-jobs.tpl b/deploy/charts/buzz/templates/_operator-jobs.tpl new file mode 100644 index 00000000000..aec91d52ff2 --- /dev/null +++ b/deploy/charts/buzz/templates/_operator-jobs.tpl @@ -0,0 +1,86 @@ +{{/* Closed rendering foundation for typed Buzz operator jobs. */}} + +{{- define "buzz.operatorCronJob" -}} +{{- $root := .root -}} +{{- if eq .type "deletionDrain" -}} +{{- $job := $root.Values.operatorJobs.deletionDrain -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "buzz.fullname" $root }}-deletion-drain + labels: + {{- include "buzz.labels" $root | nindent 4 }} + app.kubernetes.io/component: deletion-drain +spec: + schedule: {{ $job.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: {{ $job.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ $job.failedJobsHistoryLimit }} + jobTemplate: + spec: + activeDeadlineSeconds: {{ $job.activeDeadlineSeconds }} + backoffLimit: 0 + template: + metadata: + labels: + {{- include "buzz.selectorLabels" $root | nindent 12 }} + app.kubernetes.io/component: deletion-drain + {{- with $job.podLabels }} + {{- toYaml . | nindent 12 }} + {{- end }} + annotations: + {{- toYaml $job.podAnnotations | nindent 12 }} + spec: + restartPolicy: Never + terminationGracePeriodSeconds: {{ $job.terminationGracePeriodSeconds }} + serviceAccountName: {{ default (include "buzz.serviceAccountName" $root) $job.serviceAccountName }} + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + {{- toYaml $root.Values.relay.securityContext | nindent 12 }} + {{- with $root.Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: deletion-drain + image: {{ include "buzz.image" $root }} + imagePullPolicy: {{ $root.Values.image.pullPolicy }} + securityContext: + {{- omit $root.Values.relay.containerSecurityContext "readOnlyRootFilesystem" | toYaml | nindent 16 }} + readOnlyRootFilesystem: true + command: ["/usr/local/bin/buzz-admin"] + args: ["deletions", "drain"] + env: + - { name: BUZZ_S3_ENDPOINT, value: {{ required "s3.endpoint is required when operatorJobs.deletionDrain.enabled=true" (include "buzz.s3Endpoint" $root) | quote }} } + - { name: BUZZ_S3_BUCKET, value: {{ required "s3.bucket is required when operatorJobs.deletionDrain.enabled=true" $root.Values.s3.bucket | quote }} } + - { name: BUZZ_S3_REGION, value: {{ $root.Values.s3.region | quote }} } + - { name: BUZZ_S3_ADDRESSING_STYLE, value: {{ $root.Values.s3.addressingStyle | quote }} } + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "buzz.envSecretName" $root }} + key: DATABASE_URL + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: {{ include "buzz.envSecretName" $root }} + key: REDIS_URL + - name: BUZZ_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "buzz.envSecretName" $root }} + key: BUZZ_S3_ACCESS_KEY + optional: true + - name: BUZZ_S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "buzz.envSecretName" $root }} + key: BUZZ_S3_SECRET_KEY + optional: true + resources: + {{- toYaml $job.resources | nindent 16 }} +{{- else -}} +{{- fail (printf "unsupported typed operator job %q" .type) -}} +{{- end -}} +{{- end -}} diff --git a/deploy/charts/buzz/templates/_validate.tpl b/deploy/charts/buzz/templates/_validate.tpl index aa7f7ac13cf..8d055f825ed 100644 --- a/deploy/charts/buzz/templates/_validate.tpl +++ b/deploy/charts/buzz/templates/_validate.tpl @@ -18,6 +18,13 @@ surface at template time regardless of which manifest helm renders first. {{- end -}} {{- end -}} +{{/* The deletion executor always uses Redis for tenant-scoped invalidation. */}} +{{- if .Values.operatorJobs.deletionDrain.enabled -}} + {{- if and (not .Values.redis.enabled) (not .Values.externalRedis.url) (not .Values.secrets.existingSecret) -}} + {{- fail "operatorJobs.deletionDrain requires Redis. Enable redis.enabled=true, set externalRedis.url, or provide secrets.existingSecret with key REDIS_URL." -}} + {{- end -}} +{{- end -}} + {{/* Multiple replicas do NOT require ReadWriteMany git storage. Git ref/object state is object-store-backed: every read and write hydrates diff --git a/deploy/charts/buzz/templates/deletion-drain-cronjob.yaml b/deploy/charts/buzz/templates/deletion-drain-cronjob.yaml new file mode 100644 index 00000000000..84acc56590a --- /dev/null +++ b/deploy/charts/buzz/templates/deletion-drain-cronjob.yaml @@ -0,0 +1,4 @@ +{{- include "buzz.validate" . -}} +{{- if .Values.operatorJobs.deletionDrain.enabled }} +{{- include "buzz.operatorCronJob" (dict "root" . "type" "deletionDrain") }} +{{- end }} diff --git a/deploy/charts/buzz/tests/deletion_drain_test.yaml b/deploy/charts/buzz/tests/deletion_drain_test.yaml new file mode 100644 index 00000000000..18e0d1a9ef2 --- /dev/null +++ b/deploy/charts/buzz/tests/deletion_drain_test.yaml @@ -0,0 +1,214 @@ +suite: typed deletion drain operator job +templates: + - templates/deletion-drain-cronjob.yaml +tests: + - it: renders no deletion executor by default + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.accessKey: test + s3.secretKey: test + asserts: + - hasDocuments: + count: 0 + + - it: renders the bounded typed drain command with isolated pod controls + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.region: us-west-2 + s3.addressingStyle: virtual + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain: + enabled: true + schedule: "*/7 * * * *" + activeDeadlineSeconds: 900 + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 4 + terminationGracePeriodSeconds: 45 + serviceAccountName: buzz-deletion-drain + podLabels: + tags.datadoghq.com/service: buzz-deletion-drain + podAnnotations: + sidecar.istio.io/inject: "false" + example.com/operator-job: deletion-drain + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + asserts: + - hasDocuments: + count: 1 + - isAPIVersion: + of: batch/v1 + - isKind: + of: CronJob + - equal: + path: metadata.name + value: RELEASE-NAME-buzz-deletion-drain + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: deletion-drain + - equal: + path: spec.schedule + value: "*/7 * * * *" + - equal: + path: spec.concurrencyPolicy + value: Forbid + - equal: + path: spec.successfulJobsHistoryLimit + value: 2 + - equal: + path: spec.failedJobsHistoryLimit + value: 4 + - equal: + path: spec.jobTemplate.spec.activeDeadlineSeconds + value: 900 + - equal: + path: spec.jobTemplate.spec.backoffLimit + value: 0 + - equal: + path: spec.jobTemplate.spec.template.spec.restartPolicy + value: Never + - equal: + path: spec.jobTemplate.spec.template.spec.terminationGracePeriodSeconds + value: 45 + - equal: + path: spec.jobTemplate.spec.template.spec.serviceAccountName + value: buzz-deletion-drain + - equal: + path: spec.jobTemplate.spec.template.spec.automountServiceAccountToken + value: false + - equal: + path: spec.jobTemplate.spec.template.spec.enableServiceLinks + value: false + - equal: + path: spec.jobTemplate.spec.template.metadata.labels["tags.datadoghq.com/service"] + value: buzz-deletion-drain + - equal: + path: spec.jobTemplate.spec.template.metadata.annotations["example.com/operator-job"] + value: deletion-drain + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].command + value: ["/usr/local/bin/buzz-admin"] + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].args + value: ["deletions", "drain"] + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.requests.cpu + value: 250m + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.memory + value: 1Gi + + - it: exposes only database redis and deletion S3 configuration + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + secrets.existingSecret: buzz-operator-secrets + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.region: us-west-2 + s3.addressingStyle: virtual + operatorJobs.deletionDrain.enabled: true + asserts: + - lengthEqual: + path: spec.jobTemplate.spec.template.spec.containers[0].env + count: 8 + - notExists: + path: spec.jobTemplate.spec.template.spec.containers[0].envFrom + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: DATABASE_URL + valueFrom: + secretKeyRef: + name: buzz-operator-secrets + key: DATABASE_URL + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: REDIS_URL + valueFrom: + secretKeyRef: + name: buzz-operator-secrets + key: REDIS_URL + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_ENDPOINT + value: https://s3.example.com + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_BUCKET + value: buzz-media-example + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_REGION + value: us-west-2 + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_ADDRESSING_STYLE + value: virtual + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: buzz-operator-secrets + key: BUZZ_S3_ACCESS_KEY + optional: true + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: buzz-operator-secrets + key: BUZZ_S3_SECRET_KEY + optional: true + - notContains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_RELAY_PRIVATE_KEY + - notContains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_GIT_HOOK_HMAC_SECRET + - notContains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: RELAY_URL + + - it: rejects enablement without a Redis source + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + s3.endpoint: https://s3.example.com + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain.enabled: true + asserts: + - failedTemplate: + errorPattern: "operatorJobs.deletionDrain requires Redis" diff --git a/deploy/charts/buzz/values.schema.json b/deploy/charts/buzz/values.schema.json index ea5db6398dd..309b2f5c37f 100644 --- a/deploy/charts/buzz/values.schema.json +++ b/deploy/charts/buzz/values.schema.json @@ -234,6 +234,34 @@ "resources": { "type": "object" } } }, + "operatorJobs": { + "type": "object", + "additionalProperties": false, + "properties": { + "deletionDrain": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" }, + "schedule": { "type": "string", "minLength": 1 }, + "activeDeadlineSeconds": { "type": "integer", "minimum": 1, "maximum": 86400 }, + "successfulJobsHistoryLimit": { "type": "integer", "minimum": 0, "maximum": 10 }, + "failedJobsHistoryLimit": { "type": "integer", "minimum": 0, "maximum": 10 }, + "terminationGracePeriodSeconds": { "type": "integer", "minimum": 1, "maximum": 300 }, + "serviceAccountName": { "type": "string" }, + "podLabels": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "podAnnotations": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "resources": { "type": "object" } + } + } + } + }, "minio": { "type": "object", "additionalProperties": false, diff --git a/deploy/charts/buzz/values.yaml b/deploy/charts/buzz/values.yaml index 496c32e979f..f43925346e5 100644 --- a/deploy/charts/buzz/values.yaml +++ b/deploy/charts/buzz/values.yaml @@ -391,6 +391,29 @@ storageAccounting: cpu: "1" memory: 20Gi +# Typed, one-shot operator jobs. The chart intentionally exposes no generic +# command or environment registry: each job has a reviewed executable and +# least-privilege environment contract. +operatorJobs: + deletionDrain: + enabled: false + schedule: "*/5 * * * *" + activeDeadlineSeconds: 3600 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + terminationGracePeriodSeconds: 30 + serviceAccountName: "" # defaults to the main Buzz service account + podLabels: {} + podAnnotations: + sidecar.istio.io/inject: "false" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + # In-cluster MinIO for the quickstart profile only. Production deploys leave # this disabled and use s3.* (or secrets.existingSecret) against managed S3. minio: diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md new file mode 100644 index 00000000000..ec2c20f0004 --- /dev/null +++ b/docs/operator-community-deletion.md @@ -0,0 +1,87 @@ +# Community Deletion Operator Job + +Buzz executes whole-community deletion through the typed, one-shot +`/usr/local/bin/buzz-admin deletions drain` command. The Helm chart can schedule +that command as a Kubernetes CronJob; it does not call relay HTTP and it does +not add another queue or retry service. + +Postgres remains the handoff and source of truth. A run claims only requests +that the deletion store considers runnable, heartbeats the existing lease, and +resumes from durable checkpoints. `concurrencyPolicy: Forbid` prevents scheduled +pod overlap, `backoffLimit: 0` prevents Kubernetes Job retries, and the deletion +store remains authoritative when a pod exits, reaches its deadline, or is +replaced. + +## Enablement + +The CronJob is disabled by default. Production deployments should use an +existing Secret and a dedicated service account when their cluster policy +supports one: + +```yaml +secrets: + existingSecret: buzz-operator-secrets + +operatorJobs: + deletionDrain: + enabled: true + schedule: "*/5 * * * *" + activeDeadlineSeconds: 3600 + terminationGracePeriodSeconds: 30 + serviceAccountName: buzz-deletion-drain + podLabels: + tags.datadoghq.com/service: buzz-deletion-drain + podAnnotations: + sidecar.istio.io/inject: "false" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi +``` + +Set `s3.endpoint`, `s3.bucket`, `s3.region`, and `s3.addressingStyle` in chart +values. The selected Secret must contain `DATABASE_URL` and `REDIS_URL`; it may +contain `BUZZ_S3_ACCESS_KEY` and `BUZZ_S3_SECRET_KEY` when the object store uses +static credentials. The pod receives only those connection values and the four +non-secret S3 settings. It does not receive `BUZZ_RELAY_PRIVATE_KEY`, +`BUZZ_GIT_HOOK_HMAC_SECRET`, `RELAY_URL`, or the full Secret through `envFrom`. +The pod also disables service-account token automounting and Kubernetes service +link environment injection because the executor does not call the Kubernetes +API or discover cluster Services. + +The S3 principal needs the relay's normal object permissions plus bucket-level +`s3:ListBucketVersions` and object-level `s3:DeleteObjectVersion` for every +tenant-owned prefix. This also applies to never-versioned buckets because S3 +reports their objects with the `null` version id. + +## Runbook + +1. Confirm database migrations are current and the deletion request has crossed + the explicit inventory and approval boundary with + `buzz-admin deletions inspect `. +2. Confirm the selected Secret contains the required keys and the S3 principal + has version-list and exact-version delete permissions. +3. Enable the CronJob and inspect its rendered command and environment before + rollout. +4. Start one staffed manual run with + `kubectl create job --from=cronjob/-buzz-deletion-drain `. +5. Follow pod logs and re-run `buzz-admin deletions inspect ` to + verify lease, checkpoint, retry, blocked, and terminal state. +6. If a run fails or times out, fix the recorded dependency or permission + failure. Do not add Kubernetes retries: the next scheduled drain consults the + durable retry/checkpoint state and resumes only when the store allows it. + +The current owner self-serve relay admission records an owner-origin request at +`submitted` and intentionally performs no inventory or approval synchronously. +The deletion engine rejects `submitted` and `inventoried` requests at its +explicit approval boundary. Automating the privileged inventory/approval step +is therefore a separate control-plane slice; enabling this CronJob alone does +not make a newly accepted owner request destructive. + +The chart has no existing PrometheusRule or provider-neutral CronJob alert +integration. Operators must alert on failed/missed Jobs and long-running active +Jobs in their deployment platform. Adding a chart-native alert abstraction is +debt, not part of this job contract. From 1cef1ed116ade2f9812b66ed30fee1b7b6048c99 Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 07/46] Cover operator-bound signing and replay on owner delete `/operator/communities/delete` admits an irreversible request, but the endpoint's own tests only exercised the happy path and relied on generic operator-auth coverage elsewhere. Nothing pinned that this route refuses an unsigned caller, the X-Pubkey dev fallback, a non-operator signer, or a signature bound to a different method, URL, or body. Add direct coverage for each binding failure, and for idempotent replay: an identical resubmission converges on the stored request, while a resubmission that changes the owner, the host, or the acknowledgement version is refused and leaves the stored intent untouched. Every rejection asserts no deletion request was persisted. Co-Authored-By: Claude Opus 5 Signed-off-by: tornquist --- crates/buzz-relay/src/api/operator.rs | 280 ++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index ce792c2c2c8..eea2dd7c1f1 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -1145,6 +1145,286 @@ mod postgres_tests { assert_eq!(mismatched.status(), StatusCode::CONFLICT); } + /// Every signature-binding failure mode on the owner-delete endpoint. + /// + /// The endpoint mediates an irreversible request, so a caller that cannot + /// prove operator authority over this exact method, URL, and body must be + /// refused without leaving durable intent behind. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_delete_endpoint_requires_operator_bound_nip98_signature() { + let operator = Keys::generate(); + let outsider = Keys::generate(); + let owner = Keys::generate(); + let Some(state) = operator_test_state(std::slice::from_ref(&operator)).await else { + return; + }; + let host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(Arc::clone(&state), &operator, &host, &owner) + .await + .status(), + StatusCode::OK + ); + archive_for_owner_deletion(&state, &host, &owner).await; + + let delete_url = format!("http://{INGRESS_HOST}/operator/communities/delete"); + + // 1. No Authorization header at all. + let unsigned_id = Uuid::new_v4(); + let unsigned = raw_owner_delete( + Arc::clone(&state), + None, + None, + owner_delete_body(&host, &owner, unsigned_id), + ) + .await; + assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED); + assert_no_persisted_request(&state, unsigned_id, "unsigned").await; + + // 2. X-Pubkey only: the dev fallback is disabled for operator endpoints. + let x_pubkey_id = Uuid::new_v4(); + let x_pubkey_only = raw_owner_delete( + Arc::clone(&state), + None, + Some(("x-pubkey", operator.public_key().to_hex())), + owner_delete_body(&host, &owner, x_pubkey_id), + ) + .await; + assert_eq!(x_pubkey_only.status(), StatusCode::UNAUTHORIZED); + assert_no_persisted_request(&state, x_pubkey_id, "X-Pubkey only").await; + + // 3. Valid NIP-98 from a key that is not an allowlisted operator. + let outsider_id = Uuid::new_v4(); + let outsider_body = owner_delete_body(&host, &owner, outsider_id); + let outsider_response = raw_owner_delete( + Arc::clone(&state), + Some(nip98_auth_header( + &outsider, + &delete_url, + "POST", + Some(outsider_body.as_bytes()), + )), + None, + outsider_body, + ) + .await; + assert_eq!(outsider_response.status(), StatusCode::FORBIDDEN); + assert_no_persisted_request(&state, outsider_id, "non-operator signer").await; + + // 4. Operator signature that omits the payload tag. + let no_payload_id = Uuid::new_v4(); + let no_payload = raw_owner_delete( + Arc::clone(&state), + Some(nip98_auth_header_without_payload( + &operator, + &delete_url, + "POST", + )), + None, + owner_delete_body(&host, &owner, no_payload_id), + ) + .await; + assert_eq!(no_payload.status(), StatusCode::UNAUTHORIZED); + assert_no_persisted_request(&state, no_payload_id, "missing payload tag").await; + + // 5. Payload tag bound to a different body than the one sent. + let signed_id = Uuid::new_v4(); + let tampered_id = Uuid::new_v4(); + let signed_body = owner_delete_body(&host, &owner, signed_id); + let tampered = raw_owner_delete( + Arc::clone(&state), + Some(nip98_auth_header( + &operator, + &delete_url, + "POST", + Some(signed_body.as_bytes()), + )), + None, + owner_delete_body(&host, &owner, tampered_id), + ) + .await; + assert_eq!(tampered.status(), StatusCode::UNAUTHORIZED); + assert_no_persisted_request(&state, tampered_id, "tampered payload").await; + assert_no_persisted_request(&state, signed_id, "tampered payload (signed id)").await; + + // 6. Signature bound to a different operator URL. + let wrong_url_id = Uuid::new_v4(); + let wrong_url_body = owner_delete_body(&host, &owner, wrong_url_id); + let wrong_url = raw_owner_delete( + Arc::clone(&state), + Some(nip98_auth_header( + &operator, + &format!("http://{INGRESS_HOST}/operator/communities"), + "POST", + Some(wrong_url_body.as_bytes()), + )), + None, + wrong_url_body, + ) + .await; + assert_eq!(wrong_url.status(), StatusCode::UNAUTHORIZED); + assert_no_persisted_request(&state, wrong_url_id, "wrong signed URL").await; + + // 7. Signature bound to a different method. + let wrong_method_id = Uuid::new_v4(); + let wrong_method_body = owner_delete_body(&host, &owner, wrong_method_id); + let wrong_method = raw_owner_delete( + Arc::clone(&state), + Some(nip98_auth_header( + &operator, + &delete_url, + "GET", + Some(wrong_method_body.as_bytes()), + )), + None, + wrong_method_body, + ) + .await; + assert_eq!(wrong_method.status(), StatusCode::UNAUTHORIZED); + assert_no_persisted_request(&state, wrong_method_id, "wrong signed method").await; + + // Falsifiability: the same shape, correctly bound, is accepted. + let accepted_id = Uuid::new_v4(); + let accepted = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(owner_delete_body(&host, &owner, accepted_id)), + ) + .await; + assert_eq!(accepted.status(), StatusCode::ACCEPTED); + } + + /// The request UUID is the correlation identity, so a replay that changes + /// any bound field is a conflict rather than a second interpretation. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_delete_replay_rejects_any_changed_field_without_mutating_intent() { + let operator = Keys::generate(); + let owner = Keys::generate(); + let other_owner = Keys::generate(); + let Some(state) = operator_test_state(std::slice::from_ref(&operator)).await else { + return; + }; + let host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(Arc::clone(&state), &operator, &host, &owner) + .await + .status(), + StatusCode::OK + ); + archive_for_owner_deletion(&state, &host, &owner).await; + + let other_host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(Arc::clone(&state), &operator, &other_host, &owner) + .await + .status(), + StatusCode::OK + ); + archive_for_owner_deletion(&state, &other_host, &owner).await; + + let request_id = Uuid::new_v4(); + assert_eq!( + signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(owner_delete_body(&host, &owner, request_id)), + ) + .await + .status(), + StatusCode::ACCEPTED + ); + let admitted = state + .db + .deletion_store() + .get(request_id) + .await + .expect("admitted request"); + + // Exactly one field differs per case; the rest replay verbatim. + let owner_hex = owner.public_key().to_hex(); + let cases: Vec<(&str, StatusCode, String)> = vec![ + ( + "changed owner_pubkey", + StatusCode::CONFLICT, + serde_json::json!({ + "host": host, + "owner_pubkey": other_owner.public_key().to_hex(), + "request_id": request_id, + "acknowledgement_version": 1, + }) + .to_string(), + ), + ( + "changed host", + StatusCode::CONFLICT, + serde_json::json!({ + "host": other_host, + "owner_pubkey": owner_hex, + "request_id": request_id, + "acknowledgement_version": 1, + }) + .to_string(), + ), + ( + "changed acknowledgement_version", + StatusCode::BAD_REQUEST, + serde_json::json!({ + "host": host, + "owner_pubkey": owner_hex, + "request_id": request_id, + "acknowledgement_version": 2, + }) + .to_string(), + ), + ]; + for (case, expected, body) in cases { + let response = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(body), + ) + .await; + assert_eq!(response.status(), expected, "{case}"); + let current = state + .db + .deletion_store() + .get(request_id) + .await + .expect("request still readable"); + assert_eq!(current.community_id, admitted.community_id, "{case}"); + assert_eq!(current.community_host, admitted.community_host, "{case}"); + assert_eq!(current.owner_pubkey, admitted.owner_pubkey, "{case}"); + assert_eq!( + current.acknowledgement_version, admitted.acknowledgement_version, + "{case}" + ); + assert_eq!(current.stage, admitted.stage, "{case}"); + } + + // An identical replay still converges on the same request. + let converged = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(owner_delete_body(&host, &owner, request_id)), + ) + .await; + assert_eq!(converged.status(), StatusCode::ACCEPTED); + assert_eq!( + read_json(converged).await["request_id"], + request_id.to_string() + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn post_operator_body_requires_payload_tag() { From 99c26f97417b9d5ab14f6ab36ac64d5b6bd565f7 Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 08/46] Automate owner deletion preparation Co-authored-by: Codex Co-Authored-By: Claude Opus 5 Signed-off-by: tornquist --- ARCHITECTURE.md | 16 + crates/buzz-db/src/runtime/migration.rs | 34 +- .../tests/thread_window_postgres_tests.rs | 2 +- crates/buzz-db/src/store/deletion.rs | 635 +++++++++++++++++- crates/buzz-deletion/src/lib.rs | 448 +++++++++++- docs/operator-community-deletion.md | 34 +- .../0051_owner_deletion_auto_approval.sql | 22 + schema/schema.sql | 9 +- 8 files changed, 1163 insertions(+), 37 deletions(-) create mode 100644 migrations/0051_owner_deletion_auto_approval.sql diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index eaf4a47794f..77812d4e650 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -44,6 +44,18 @@ on the row. Owner provenance pins `requested_by` to `owner_pubkey`, so passing the operator's own pubkey does not converge — it conflicts with the existing one-active-request invariant instead. +The privileged one-shot `buzz-admin deletions drain` process gives already- +approved work priority. When none is ready, it may claim only an authenticated +owner-origin `submitted` request under the same durable generation lease used +for execution, inventory it with lease-loss cancellation, and atomically freeze +the inventory plus a digest-bound `owner_automatic` approval. The mediating +operator remains the approval actor; the owner acknowledgement is pre-inventory +intent, not a claim that the owner reviewed the digest. The retained lease then +enters the unchanged approved-request executor. Operator-origin requests never +auto-progress and still require explicit inventory and approval. Owner +admission still has no owner-facing cancellation or grace period; the +privileged abort described above remains the recovery path. + Ownership is mutable only while a community is active. Archiving freezes the current owner. Normal transfer and deployment-root legacy convergence take the same community-row lock as owner-deletion admission, then reject archived, @@ -761,6 +773,10 @@ Postgres/Redis clients and S3 client; it does not call relay HTTP. Durable requests, leases, retry timing, and checkpoints in Postgres are the handoff and execution authority, so Kubernetes uses `Forbid` concurrency and zero Job retries rather than introducing a second retry system. +The same drain first claims runnable approved work and, only when none exists, +may prepare one owner-origin submission. Inventory, automatic approval, and +execution share one generation lease and the existing retry/block/checkpoint +records; there is no preparation worker, command, queue, or retry authority. --- diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 7b64547c2f5..348d24dc0a8 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -703,9 +703,10 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 50); + assert_eq!(migrations.len(), 51); assert_eq!(migrations[48].version, 49); assert_eq!(migrations[49].version, 50); + assert_eq!(migrations[50].version, 51); assert!(migrations[48] .sql .as_str() @@ -714,6 +715,7 @@ mod postgres_tests { .sql .as_str() .contains("community_deletion_owner_provenance")); + assert!(migrations[50].sql.as_str().contains("approval_origin")); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -1657,6 +1659,34 @@ mod postgres_tests { ); } + #[test] + fn owner_deletion_auto_approval_migration_matches_desired_schema() { + let migration = MIGRATOR + .iter() + .find(|migration| migration.version == 51) + .expect("embedded migration 0051") + .sql + .as_ref() + .to_ascii_lowercase(); + let workspace_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(std::path::Path::parent) + .expect("workspace root"); + let schema = std::fs::read_to_string(workspace_root.join("schema/schema.sql")) + .expect("read schema/schema.sql") + .to_ascii_lowercase(); + + for sql in [&migration, &schema] { + assert!(sql.contains("approval_origin text not null default 'operator'")); + assert!(sql.contains("approval_origin in ('operator', 'owner_automatic')")); + assert!(sql.contains("'submitted', 'approved', 'fenced'")); + assert!(sql.contains("community_deletion_requests_owner_preparable")); + assert!(sql.contains("request_origin = 'owner'")); + assert!(sql.contains("stage = 'submitted'")); + } + assert!(migration.contains("set local lock_timeout = '5s'")); + } + /// Structural parity between migration 0029's deletion surface and the /// desired-state bootstrap schema (`schema/schema.sql`). /// @@ -1801,7 +1831,7 @@ mod postgres_tests { .tables .get(table) .unwrap_or_else(|| panic!("schema.sql is missing deletion table {table}")); - if table != "community_deletion_requests" { + if table != "community_deletion_requests" && table != "community_deletion_approvals" { assert_eq!( in_schema, definition, "schema.sql definition of {table} drifted from migration 0029" diff --git a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs index 5ca0185af56..5d51f4348d5 100644 --- a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs @@ -311,7 +311,7 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() production_result.is_ok(), "production migrator must preserve ingestion progress: {production_result:?}" ); - assert_eq!(version, 50); + assert_eq!(version, 51); assert_eq!(final_oid, oid, "prebuild must not be replaced"); assert_eq!(count, 4, "all writer witnesses must persist"); } diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 333b4da5728..167e4503a1e 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -606,12 +606,38 @@ pub struct DeletionApproval { pub inventory_digest: String, /// Approving operator identity. pub approved_by: String, + /// Bounded provenance for the approval decision. + pub approval_origin: DeletionApprovalOrigin, /// Optional approval note. pub note: Option, /// Approval timestamp. pub approved_at: DateTime, } +/// Durable provenance class for an inventory approval. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DeletionApprovalOrigin { + /// A deployment operator explicitly approved the inventory. + Operator, + /// Privileged policy approved authenticated owner intent automatically. + OwnerAutomatic, +} + +impl FromStr for DeletionApprovalOrigin { + type Err = DbError; + + fn from_str(value: &str) -> std::result::Result { + match value { + "operator" => Ok(Self::Operator), + "owner_automatic" => Ok(Self::OwnerAutomatic), + other => Err(DbError::InvalidData(format!( + "unknown community deletion approval origin: {other}" + ))), + } + } +} + /// Monotonic lease token required by every execution mutation. #[derive(Debug, Clone, PartialEq, Eq)] pub struct LeaseToken { @@ -955,7 +981,7 @@ impl DeletionStore { pub async fn inspect(&self, request_id: Uuid) -> Result { let request = self.get(request_id).await?; let approval_row = sqlx::query( - "SELECT inventory_digest, approved_by, note, approved_at \ + "SELECT inventory_digest, approved_by, approval_origin, note, approved_at \ FROM community_deletion_approvals WHERE request_id = $1", ) .bind(request_id) @@ -966,6 +992,7 @@ impl DeletionStore { Ok::(DeletionApproval { inventory_digest: hex::encode(row.try_get::, _>("inventory_digest")?), approved_by: row.try_get("approved_by")?, + approval_origin: row.try_get::("approval_origin")?.parse()?, note: row.try_get("note")?, approved_at: row.try_get("approved_at")?, }) @@ -1183,7 +1210,7 @@ impl DeletionStore { .bind(schema) .bind(storage) .bind(frozen) - .bind(digest) + .bind(&digest) .fetch_optional(&self.pool) .await? .ok_or_else(|| { @@ -1225,8 +1252,8 @@ impl DeletionStore { } sqlx::query( "INSERT INTO community_deletion_approvals \ - (request_id, community_id, inventory_digest, approved_by, note) \ - VALUES ($1, $2, $3, $4, $5)", + (request_id, community_id, inventory_digest, approved_by, approval_origin, note) \ + VALUES ($1, $2, $3, $4, 'operator', $5)", ) .bind(request_id) .bind(community_id) @@ -1272,6 +1299,194 @@ impl DeletionStore { self.claim(None, owner, lease_duration).await } + /// Claim the oldest due authenticated owner submission for preparation. + pub async fn claim_next_owner_submission( + &self, + owner: &str, + lease_duration: Duration, + ) -> Result> { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + let row = sqlx::query( + r#"WITH candidate AS ( + SELECT id FROM community_deletion_requests + WHERE request_origin = 'owner' AND acknowledgement_version = $3 + AND stage = 'submitted' + AND blocked_at IS NULL AND next_attempt_at <= now() + AND (lease_until IS NULL OR lease_until < now()) + ORDER BY created_at, id + FOR UPDATE SKIP LOCKED LIMIT 1 + ) + UPDATE community_deletion_requests request + SET lease_owner = $1, lease_generation = lease_generation + 1, + lease_until = now() + make_interval(secs => $2), + attempts = attempts + 1, updated_at = now() + FROM candidate WHERE request.id = candidate.id + RETURNING request.*"#, + ) + .bind(owner) + .bind(lease_seconds) + .bind(OWNER_DELETION_ACKNOWLEDGEMENT_VERSION) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + let Some(row) = row else { + return Ok(None); + }; + let request = row_to_request(row)?; + let lease = LeaseToken { + request_id: request.id, + owner: owner.to_owned(), + generation: request.lease_generation, + community_id: request.community_id, + fence_generation: request.fence_generation, + }; + Ok(Some(ClaimedDeletion { request, lease })) + } + + /// Renew a live owner-submission preparation lease. + pub async fn heartbeat_owner_submission( + &self, + token: &LeaseToken, + executor_mode: &str, + lease_duration: Duration, + draining: bool, + ) -> Result<()> { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + verify_owner_submission_lease(&mut tx, token).await?; + let affected = sqlx::query( + "UPDATE community_deletion_requests SET lease_until = \ + now() + make_interval(secs => $4), updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(lease_seconds) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(stale_lease_error(token)); + } + upsert_executor_heartbeat(&mut tx, token, executor_mode, draining).await?; + tx.commit().await?; + Ok(()) + } + + /// Atomically freeze inventory and approve authenticated owner intent. + pub async fn complete_owner_preparation( + &self, + token: &LeaseToken, + inventory: &FrozenInventory, + ) -> Result { + validate_storage_manifest(&inventory.storage)?; + let digest = inventory.digest()?; + let schema = serde_json::to_value(&inventory.schema)?; + let storage = serde_json::to_value(&inventory.storage)?; + let frozen = serde_json::to_value(inventory)?; + let mut tx = self.pool.begin().await?; + let request_row = + sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1 FOR UPDATE") + .bind(token.request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| stale_lease_error(token))?; + let request = row_to_request(request_row)?; + let lease_live: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3 \ + AND lease_until >= now())", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .fetch_one(&mut *tx) + .await?; + let lease_matches = lease_live + && request.community_id == token.community_id + && request.request_origin == DeletionRequestOrigin::Owner + && request.acknowledgement_version == Some(OWNER_DELETION_ACKNOWLEDGEMENT_VERSION) + && request.lease_owner.as_deref() == Some(token.owner.as_str()) + && request.lease_generation == token.generation + && request.blocked_reason.is_none(); + if !lease_matches { + return Err(stale_lease_error(token)); + } + if request.stage == DeletionStage::Approved { + let approval: Option<(Vec, String, String)> = sqlx::query_as( + "SELECT inventory_digest, approved_by, approval_origin \ + FROM community_deletion_approvals WHERE request_id = $1", + ) + .bind(token.request_id) + .fetch_optional(&mut *tx) + .await?; + let expected_operator = request.mediating_operator_pubkey.as_deref(); + let digest_hex = hex::encode(&digest); + let converged = request.inventory_digest.as_deref() == Some(digest_hex.as_str()) + && approval + .as_ref() + .is_some_and(|(approved_digest, approved_by, origin)| { + approved_digest.as_slice() == digest + && Some(approved_by.as_str()) == expected_operator + && origin == "owner_automatic" + }); + if converged { + tx.commit().await?; + return Ok(request); + } + return Err(DbError::DeletionSafety(format!( + "deletion {} automatic preparation does not match frozen approval evidence", + token.request_id + ))); + } + if request.stage != DeletionStage::Submitted { + return Err(stale_lease_error(token)); + } + let mediating_operator = request.mediating_operator_pubkey.clone().ok_or_else(|| { + DbError::DeletionSafety(format!( + "owner deletion {} is missing mediating operator provenance", + token.request_id + )) + })?; + sqlx::query( + "UPDATE community_deletion_requests SET stage = 'inventoried', \ + schema_manifest = $2, storage_manifest = $3, inventory_manifest = $4, \ + inventory_digest = $5, inventory_frozen_at = now(), updated_at = now() \ + WHERE id = $1", + ) + .bind(token.request_id) + .bind(schema) + .bind(storage) + .bind(frozen) + .bind(&digest) + .execute(&mut *tx) + .await?; + sqlx::query( + "INSERT INTO community_deletion_approvals \ + (request_id, community_id, inventory_digest, approved_by, approval_origin) \ + VALUES ($1, $2, $3, $4, 'owner_automatic')", + ) + .bind(token.request_id) + .bind(token.community_id.as_uuid()) + .bind(digest) + .bind(mediating_operator) + .execute(&mut *tx) + .await?; + let row = sqlx::query( + "UPDATE community_deletion_requests SET stage = 'approved', \ + retry_count = 0, retry_stage = NULL, next_attempt_at = now(), \ + last_error = NULL, last_error_at = NULL, updated_at = now() \ + WHERE id = $1 AND stage = 'inventoried' RETURNING *", + ) + .bind(token.request_id) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + row_to_request(row) + } + async fn claim( &self, request_id: Option, @@ -1402,19 +1617,7 @@ impl DeletionStore { if affected != 1 { return Err(stale_lease_error(token)); } - sqlx::query( - "INSERT INTO community_deletion_executor_heartbeats \ - (executor_id, mode, request_id, draining) VALUES ($1, $2, $3, $4) \ - ON CONFLICT (executor_id) DO UPDATE SET mode = EXCLUDED.mode, \ - request_id = EXCLUDED.request_id, heartbeat_at = now(), \ - draining = EXCLUDED.draining, stopped_at = NULL", - ) - .bind(&token.owner) - .bind(executor_mode) - .bind(token.request_id) - .bind(draining) - .execute(&mut *tx) - .await?; + upsert_executor_heartbeat(&mut tx, token, executor_mode, draining).await?; tx.commit().await?; Ok(()) } @@ -2198,6 +2401,81 @@ impl DeletionStore { Ok(()) } + /// Persist a retryable owner-inventory failure and release its preparation lease. + pub async fn record_owner_preparation_retry( + &self, + token: &LeaseToken, + unit_key: &str, + error: &str, + retry_after: Duration, + ) -> Result<()> { + let bounded = bound_text(error, 4096); + let retry_seconds = i64::try_from(retry_after.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + verify_owner_submission_lease(&mut tx, token).await?; + let (retry_count, retry_stage): (i32, Option) = sqlx::query_as( + "SELECT retry_count, retry_stage FROM community_deletion_requests \ + WHERE id = $1 FOR UPDATE", + ) + .bind(token.request_id) + .fetch_one(&mut *tx) + .await?; + let consecutive_retries = if retry_stage.as_deref() == Some("submitted") { + retry_count.saturating_add(1) + } else { + 1 + }; + let exhausted = consecutive_retries >= 8; + checkpoint_failed_tx(&mut tx, token, DeletionStage::Submitted, unit_key, &bounded).await?; + sqlx::query( + "UPDATE community_deletion_requests SET retry_count = $5, \ + retry_stage = 'submitted', last_error = $4, last_error_at = now(), \ + next_attempt_at = CASE WHEN $6 THEN next_attempt_at \ + ELSE now() + make_interval(secs => $7) END, \ + blocked_at = CASE WHEN $6 THEN now() ELSE blocked_at END, \ + blocked_reason = CASE WHEN $6 THEN $4 ELSE blocked_reason END, \ + lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(&bounded) + .bind(consecutive_retries) + .bind(exhausted) + .bind(retry_seconds) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Permanently block unsafe owner inventory preparation and release its lease. + pub async fn block_owner_preparation( + &self, + token: &LeaseToken, + unit_key: &str, + error: &str, + ) -> Result<()> { + let bounded = bound_text(error, 4096); + let mut tx = self.pool.begin().await?; + verify_owner_submission_lease(&mut tx, token).await?; + checkpoint_failed_tx(&mut tx, token, DeletionStage::Submitted, unit_key, &bounded).await?; + sqlx::query( + "UPDATE community_deletion_requests SET blocked_at = now(), blocked_reason = $4, \ + last_error = $4, last_error_at = now(), lease_owner = NULL, lease_until = NULL, \ + updated_at = now() WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(&bounded) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + /// Terminally abort a request at the reversible pre-destruction boundary. /// /// `submitted`, `inventoried`, `approved`, and `fenced` are reversible: @@ -3116,6 +3394,55 @@ fn validate_manifest_key_chunks( Ok(()) } +async fn upsert_executor_heartbeat( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + executor_mode: &str, + draining: bool, +) -> Result<()> { + sqlx::query( + "INSERT INTO community_deletion_executor_heartbeats \ + (executor_id, mode, request_id, draining) VALUES ($1, $2, $3, $4) \ + ON CONFLICT (executor_id) DO UPDATE SET mode = EXCLUDED.mode, \ + request_id = EXCLUDED.request_id, heartbeat_at = now(), \ + draining = EXCLUDED.draining, stopped_at = NULL", + ) + .bind(&token.owner) + .bind(executor_mode) + .bind(token.request_id) + .bind(draining) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn verify_owner_submission_lease( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, +) -> Result<()> { + let valid = sqlx::query_scalar::<_, Uuid>( + "SELECT request.id FROM community_deletion_requests request \ + WHERE request.id = $1 AND request.community_id = $4 \ + AND request.request_origin = 'owner' AND request.stage = 'submitted' \ + AND request.acknowledgement_version = $5 \ + AND request.lease_owner = $2 AND request.lease_generation = $3 \ + AND request.lease_until >= now() AND request.blocked_at IS NULL \ + FOR UPDATE", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(token.community_id.as_uuid()) + .bind(OWNER_DELETION_ACKNOWLEDGEMENT_VERSION) + .fetch_optional(&mut **tx) + .await?; + if valid.is_some() { + Ok(()) + } else { + Err(stale_lease_error(token)) + } +} + async fn verify_lease( tx: &mut Transaction<'_, Postgres>, token: &LeaseToken, @@ -4603,6 +4930,270 @@ mod postgres_tests { } } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_preparation_claim_is_owner_only_concurrent_and_reclaimable() { + let (db, store) = store().await; + let (owner_host, owner, _) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + let OwnerDeletionAdmission::Accepted(owner_request) = store + .admit_owner_request(&owner_host, &owner, operator, 1, request_id) + .await + .expect("admit owner request") + else { + panic!("expected accepted owner request") + }; + let operator_host = format!("operator-delete-{}.example", Uuid::new_v4().simple()); + db.ensure_configured_community(&operator_host) + .await + .expect("create operator community"); + let operator_request = store + .submit(&operator_host, "manual-operator", None) + .await + .expect("submit manual request"); + + let (first, second) = tokio::join!( + store.claim_next_owner_submission("preparer-a", DEFAULT_LEASE_DURATION), + store.claim_next_owner_submission("preparer-b", DEFAULT_LEASE_DURATION), + ); + let claims = [first.expect("first claim"), second.expect("second claim")]; + assert_eq!(claims.iter().filter(|claim| claim.is_some()).count(), 1); + let claim = claims.into_iter().flatten().next().expect("one winner"); + assert_eq!(claim.request.id, owner_request.id); + assert_eq!(claim.request.request_origin, DeletionRequestOrigin::Owner); + assert_ne!(claim.request.id, operator_request.id); + + store + .heartbeat_owner_submission(&claim.lease, "drain", DEFAULT_LEASE_DURATION, false) + .await + .expect("heartbeat owner preparation"); + sqlx::query( + "UPDATE community_deletion_requests SET lease_until = now() - interval '1 second' WHERE id = $1", + ) + .bind(owner_request.id) + .execute(&db.pool) + .await + .expect("expire preparation lease"); + let successor = store + .claim_next_owner_submission("preparer-c", DEFAULT_LEASE_DURATION) + .await + .expect("reclaim expired preparation") + .expect("expired owner preparation is reclaimable"); + assert_eq!(successor.request.id, owner_request.id); + assert!(successor.lease.generation > claim.lease.generation); + assert!(store + .heartbeat_owner_submission(&claim.lease, "drain", DEFAULT_LEASE_DURATION, false) + .await + .is_err()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_preparation_atomically_approves_exact_inventory_and_converges() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + store + .admit_owner_request(&host, &owner, operator, 1, request_id) + .await + .expect("admit owner request"); + let claim = store + .claim_next_owner_submission("preparer", DEFAULT_LEASE_DURATION) + .await + .expect("claim owner request") + .expect("owner request is preparable"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("schema inventory"), + storage: empty_storage_manifest(community), + }; + + let approved = store + .complete_owner_preparation(&claim.lease, &inventory) + .await + .expect("complete owner preparation"); + assert_eq!(approved.stage, DeletionStage::Approved); + assert_eq!( + approved.inventory_digest, + Some(hex::encode(inventory.digest().unwrap())) + ); + let replay = store + .complete_owner_preparation(&claim.lease, &inventory) + .await + .expect("ambiguous commit replay converges"); + assert_eq!(replay, approved); + let inspection = store.inspect(request_id).await.expect("inspect approval"); + let approval = inspection.approval.expect("automatic approval evidence"); + assert_eq!( + approval.approval_origin, + DeletionApprovalOrigin::OwnerAutomatic + ); + assert_eq!(approval.approved_by, operator); + assert!(store + .claim_specific(request_id, "other-executor", DEFAULT_LEASE_DURATION) + .await + .expect("existing execution claim remains approval-bound") + .is_none()); + store + .verify_execution_token(&claim.lease, DeletionStage::Approved) + .await + .expect("retained lease is immediately execution eligible"); + + let changed = FrozenInventory { + schema: inventory.schema.clone(), + storage: StorageManifest { + version: inventory.storage.version, + prefixes: inventory + .storage + .prefixes + .iter() + .cloned() + .map(|mut prefix| { + prefix.total_bytes += 1; + prefix + }) + .collect(), + }, + }; + assert!(store + .complete_owner_preparation(&claim.lease, &changed) + .await + .is_err()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn stale_owner_preparation_generation_cannot_freeze_or_approve() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + store + .admit_owner_request(&host, &owner, operator, 1, Uuid::new_v4()) + .await + .expect("admit owner request"); + let stale = store + .claim_next_owner_submission("stale-preparer", DEFAULT_LEASE_DURATION) + .await + .expect("claim owner request") + .expect("owner request is preparable"); + sqlx::query( + "UPDATE community_deletion_requests SET lease_until = now() - interval '1 second' WHERE id = $1", + ) + .bind(stale.request.id) + .execute(&db.pool) + .await + .expect("expire stale lease"); + let successor = store + .claim_next_owner_submission("successor-preparer", DEFAULT_LEASE_DURATION) + .await + .expect("reclaim owner request") + .expect("expired request is reclaimable"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("schema inventory"), + storage: empty_storage_manifest(community), + }; + assert!(is_stale_deletion_lease( + &store + .complete_owner_preparation(&stale.lease, &inventory) + .await + .expect_err("stale generation cannot commit") + )); + let approved = store + .complete_owner_preparation(&successor.lease, &inventory) + .await + .expect("successor commits preparation"); + assert_eq!(approved.stage, DeletionStage::Approved); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_preparation_retry_block_and_privileged_abort_are_recoverable() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + store + .admit_owner_request(&host, &owner, operator, 1, request_id) + .await + .expect("admit owner request"); + let claim = store + .claim_next_owner_submission("preparer", DEFAULT_LEASE_DURATION) + .await + .expect("claim owner request") + .expect("owner request is preparable"); + store + .record_owner_preparation_retry( + &claim.lease, + "inventory", + "temporary object-store failure", + Duration::ZERO, + ) + .await + .expect("record preparation retry"); + let retried = store.get(request_id).await.expect("load retried request"); + assert_eq!(retried.retry_stage, Some(DeletionStage::Submitted)); + assert_eq!(retried.retry_count, 1); + assert!(retried.lease_owner.is_none()); + + let claim = store + .claim_next_owner_submission("preparer-2", DEFAULT_LEASE_DURATION) + .await + .expect("reclaim owner request") + .expect("retried request is due"); + store + .block_owner_preparation(&claim.lease, "inventory", "unsafe storage taxonomy") + .await + .expect("block preparation"); + assert_eq!( + store + .get(request_id) + .await + .expect("load blocked request") + .blocked_reason + .as_deref(), + Some("unsafe storage taxonomy") + ); + let aborted = store + .abort( + request_id, + "recovery-operator", + "cannot safely enumerate storage", + ) + .await + .expect("abort submitted request"); + assert_eq!(aborted.stage, DeletionStage::Aborted); + assert_eq!( + sqlx::query_scalar::<_, String>("SELECT deletion_state FROM communities WHERE id = $1") + .bind(community.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("community lifecycle"), + "active" + ); + assert!(sqlx::query_scalar::<_, Option>>( + "SELECT archived_at FROM communities WHERE id = $1" + ) + .bind(community.as_uuid()) + .fetch_one(&db.pool) + .await + .expect("community archive state") + .is_some()); + assert!(matches!( + store + .admit_owner_request(&host, &owner, operator, 1, Uuid::new_v4()) + .await + .expect("fresh owner request after recovery abort"), + OwnerDeletionAdmission::Accepted(_) + )); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn approval_boundary_blocks_claim_until_exact_inventory_is_approved() { @@ -4651,6 +5242,16 @@ mod postgres_tests { .await .expect("approve"); assert_eq!(approved.stage, DeletionStage::Approved); + assert_eq!( + store + .inspect(request.id) + .await + .expect("inspect manual approval") + .approval + .expect("manual approval evidence") + .approval_origin, + DeletionApprovalOrigin::Operator + ); assert_eq!( approved.inventory_digest, Some(hex::encode(inventory.digest().unwrap())) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 2814c486917..f7768385568 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -290,7 +290,7 @@ pub enum Command { #[arg(long)] executor_id: Option, }, - /// Drain the currently runnable deletion queue, then exit. + /// Drain runnable work, preparing authenticated owner submissions when idle. Drain { /// Executor identity (defaults to hostname/pid). #[arg(long)] @@ -450,7 +450,7 @@ async fn run_with_services(command: Command, services: Services) -> Result .store .submit(&host, &requested_by, reason.as_deref()) .await?; - let inventory = build_inventory(&services, &request).await?; + let inventory = build_inventory(&services, &request, None).await?; let request = services .store .freeze_inventory(request.id, &inventory) @@ -683,12 +683,13 @@ fn validate_storage_ownership(request: &DeletionRequest, manifest: &StorageManif async fn build_inventory( services: &Services, request: &DeletionRequest, + heartbeat_lost: Option<&CancellationToken>, ) -> Result { let schema = services .store .inventory_schema(request.community_id) .await?; - let storage = enumerate_tenant_prefixes(services, request, None, None).await?; + let storage = enumerate_tenant_prefixes(services, request, heartbeat_lost, None).await?; Ok(FrozenInventory { schema, storage }) } @@ -977,7 +978,15 @@ async fn run_loop( .await? } }; - let Some(claim) = claim else { + let claim = if claim.is_none() && mode == LoopMode::Drain && request_id.is_none() { + services + .store + .claim_next_owner_submission(&executor_id, DEFAULT_LEASE_DURATION) + .await? + } else { + claim + }; + let Some(mut claim) = claim else { if mode == LoopMode::Run && !ran { anyhow::bail!( "deletion request is not runnable, is blocked, or is leased by another executor" @@ -986,6 +995,34 @@ async fn run_loop( return Ok(0); }; ran = true; + if claim.request.stage == DeletionStage::Submitted { + let preparation_request = claim.request.clone(); + let preparation_services = &services; + match prepare_owner_claim_with( + &services, + mode, + claim, + &shutdown, + HEARTBEAT_INTERVAL, + |heartbeat_lost| async move { + build_inventory( + preparation_services, + &preparation_request, + Some(&heartbeat_lost), + ) + .await + }, + ) + .await? + { + OwnerPreparationOutcome::Prepared(prepared) => claim = *prepared, + OwnerPreparationOutcome::Stopped => return Ok(0), + OwnerPreparationOutcome::Failed(output) => { + print_json(&output)?; + return Ok(1); + } + } + } let output = execute_claim(&services, mode, claim, &shutdown).await?; print_json(&output)?; let failed = output.last_error.is_some() || output.blocked_reason.is_some(); @@ -995,6 +1032,173 @@ async fn run_loop( } } +enum OwnerPreparationOutcome { + Prepared(Box), + Stopped, + Failed(RunOutput), +} + +async fn record_owner_preparation_failure( + services: &Services, + token: &LeaseToken, + error: &anyhow::Error, +) -> Result<()> { + let message = format!("{error:#}"); + let result = if is_permanent_error(error) { + services + .store + .block_owner_preparation(token, "inventory", &message) + .await + } else { + services + .store + .record_owner_preparation_retry(token, "inventory", &message, RETRY_DELAY) + .await + }; + match result { + Ok(()) => Ok(()), + Err(error) if buzz_db::deletion::is_stale_deletion_lease(&error) => Ok(()), + Err(error) => Err(error.into()), + } +} + +async fn prepare_owner_claim_with( + services: &Services, + mode: LoopMode, + claim: ClaimedDeletion, + shutdown: &CancellationToken, + heartbeat_period: Duration, + build: F, +) -> Result +where + F: FnOnce(CancellationToken) -> Fut, + Fut: Future>, +{ + let token = claim.lease.clone(); + if let Err(error) = services + .store + .heartbeat_owner_submission(&token, mode.as_str(), DEFAULT_LEASE_DURATION, false) + .await + { + if buzz_db::deletion::is_stale_deletion_lease(&error) { + let request = services.store.get(token.request_id).await?; + return Ok(OwnerPreparationOutcome::Failed(run_output(request))); + } + return Err(error.into()); + } + + let heartbeat_services = services.clone(); + let heartbeat_token = token.clone(); + let heartbeat_mode = mode.as_str(); + let heartbeat_shutdown = CancellationToken::new(); + let heartbeat_cancel = heartbeat_shutdown.clone(); + let heartbeat_error = CancellationToken::new(); + let heartbeat_error_signal = heartbeat_error.clone(); + let heartbeat = tokio::spawn(async move { + let mut interval = tokio::time::interval(heartbeat_period); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + tokio::select! { + _ = heartbeat_cancel.cancelled() => return, + _ = interval.tick() => { + if heartbeat_services + .store + .heartbeat_owner_submission( + &heartbeat_token, + heartbeat_mode, + DEFAULT_LEASE_DURATION, + false, + ) + .await + .is_err() + { + heartbeat_error_signal.cancel(); + return; + } + } + } + } + }); + + enum PreparationStage { + Prepared(Box), + Stopped, + Failed(anyhow::Error), + } + let preparation = async { + let inventory = build(heartbeat_error.clone()).await?; + services + .store + .complete_owner_preparation(&token, &inventory) + .await + .map_err(Into::into) + }; + let stage = tokio::select! { + biased; + _ = shutdown.cancelled() => PreparationStage::Stopped, + _ = heartbeat_error.cancelled() => PreparationStage::Failed(DeletionLeaseLost.into()), + result = preparation => match result { + Ok(request) => PreparationStage::Prepared(Box::new(request)), + Err(error) => PreparationStage::Failed(error), + }, + }; + heartbeat_shutdown.cancel(); + let stage = match heartbeat.await { + Ok(()) => stage, + Err(error) => PreparationStage::Failed(anyhow::anyhow!( + "owner deletion preparation heartbeat task failed: {error}" + )), + }; + + match stage { + PreparationStage::Prepared(request) => Ok(OwnerPreparationOutcome::Prepared(Box::new( + ClaimedDeletion { + request: *request, + lease: token, + }, + ))), + PreparationStage::Stopped => { + let _ = services + .store + .heartbeat_owner_submission(&token, mode.as_str(), DEFAULT_LEASE_DURATION, true) + .await; + services + .store + .stop_executor(Some(&token), &token.owner) + .await?; + Ok(OwnerPreparationOutcome::Stopped) + } + PreparationStage::Failed(error) => { + let request = services.store.get(token.request_id).await?; + if request.stage == DeletionStage::Approved + && request.lease_owner.as_deref() == Some(token.owner.as_str()) + && request.lease_generation == token.generation + && services + .store + .verify_execution_token(&token, DeletionStage::Approved) + .await + .is_ok() + { + return Ok(OwnerPreparationOutcome::Prepared(Box::new( + ClaimedDeletion { + request, + lease: token, + }, + ))); + } + if request.lease_owner.as_deref() != Some(token.owner.as_str()) + || request.lease_generation != token.generation + { + return Ok(OwnerPreparationOutcome::Failed(run_output(request))); + } + record_owner_preparation_failure(services, &token, &error).await?; + let request = services.store.get(token.request_id).await?; + Ok(OwnerPreparationOutcome::Failed(run_output(request))) + } + } +} + async fn stop_claim_executor( services: &Services, mode: LoopMode, @@ -1725,6 +1929,242 @@ mod postgres_tests { (db, services, claim) } + async fn claimed_owner_preparation( + prefix: &str, + ) -> (Db, Services, ClaimedDeletion, FrozenInventory) { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .expect("BUZZ_TEST_DATABASE_URL or DATABASE_URL is required"); + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect owner preparation test DB"); + let db = Db::from_pool(pool); + if std::env::var("BUZZ_TEST_SCHEMA_MODE").as_deref() != Ok("desired") { + db.migrate().await.expect("migrate deletion engine test DB"); + } + let store = db.deletion_store(); + let host = format!("{prefix}-{}.example", Uuid::new_v4().simple()); + let owner = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let buzz_db::CreateCommunityWithOwnerResult::Created(community) = db + .create_community_with_owner(&host, &owner) + .await + .expect("create owner preparation community") + else { + panic!("expected fresh owner preparation community") + }; + db.archive_community_owned_by(&host, &owner, "protected.example") + .await + .expect("archive owner preparation community") + .expect("owned community"); + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + store + .admit_owner_request(&host, &owner, operator, 1, Uuid::new_v4()) + .await + .expect("admit owner request"); + let claim = store + .claim_next_owner_submission("test-preparer", DEFAULT_LEASE_DURATION) + .await + .expect("claim owner request") + .expect("owner request is preparable"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community.id) + .await + .expect("inventory owner schema"), + storage: empty_storage_manifest(community.id), + }; + let services = Services { + store, + media: Arc::new( + MediaStorage::new(&buzz_media::MediaConfig { + s3_endpoint: "http://127.0.0.1:1".to_string(), + s3_access_key: "unused".to_string(), + s3_secret_key: "unused".to_string(), + s3_bucket: "unused".to_string(), + s3_region: "us-east-1".to_string(), + s3_addressing_style: buzz_media::S3AddressingStyle::Path, + max_image_bytes: 1, + max_gif_bytes: 1, + max_video_bytes: 1, + max_file_bytes: 1, + public_base_url: "http://localhost/media".to_string(), + upload_records_enabled: false, + upload_ip_header: None, + upload_port_header: None, + }) + .expect("construct unused media service"), + ), + redis: deadpool_redis::Config::from_url("redis://127.0.0.1:1") + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .expect("construct unused Redis pool"), + }; + (db, services, claim, inventory) + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn drain_owner_preparation_approves_for_existing_execution_only() { + let (db, services, claim, inventory) = + claimed_owner_preparation("owner-preparation-success").await; + let operator_host = format!("manual-{}.example", Uuid::new_v4().simple()); + db.ensure_configured_community(&operator_host) + .await + .expect("create manual community"); + let manual = services + .store + .submit(&operator_host, "manual-operator", None) + .await + .expect("submit manual request"); + + let outcome = prepare_owner_claim_with( + &services, + LoopMode::Drain, + claim, + &CancellationToken::new(), + HEARTBEAT_INTERVAL, + |_| async move { Ok(inventory) }, + ) + .await + .expect("prepare owner claim"); + let OwnerPreparationOutcome::Prepared(prepared) = outcome else { + panic!("owner preparation should produce an approved execution claim") + }; + assert_eq!(prepared.request.stage, DeletionStage::Approved); + services + .store + .verify_execution_token(&prepared.lease, DeletionStage::Approved) + .await + .expect("prepared claim enters unchanged execution boundary"); + assert_eq!( + services + .store + .get(manual.id) + .await + .expect("manual request") + .stage, + DeletionStage::Submitted + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn drain_owner_preparation_persists_transient_and_permanent_failures() { + let (_, services, claim, _) = claimed_owner_preparation("owner-preparation-failure").await; + let request_id = claim.request.id; + let outcome = prepare_owner_claim_with( + &services, + LoopMode::Drain, + claim, + &CancellationToken::new(), + HEARTBEAT_INTERVAL, + |_| async { Err(transient("temporary inventory failure")) }, + ) + .await + .expect("record transient preparation failure"); + assert!(matches!(outcome, OwnerPreparationOutcome::Failed(_))); + let retried = services + .store + .get(request_id) + .await + .expect("retried request"); + assert_eq!(retried.retry_stage, Some(DeletionStage::Submitted)); + assert_eq!(retried.retry_count, 1); + + let (_, blocked_services, claim, _) = + claimed_owner_preparation("owner-preparation-permanent").await; + let blocked_request_id = claim.request.id; + let outcome = prepare_owner_claim_with( + &blocked_services, + LoopMode::Drain, + claim, + &CancellationToken::new(), + HEARTBEAT_INTERVAL, + |_| async { Err(permanent("unsafe inventory taxonomy")) }, + ) + .await + .expect("record permanent preparation failure"); + assert!(matches!(outcome, OwnerPreparationOutcome::Failed(_))); + assert!(blocked_services + .store + .get(blocked_request_id) + .await + .expect("blocked request") + .blocked_reason + .is_some()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn drain_owner_preparation_rejects_stale_lease_before_inventory() { + let (_, services, claim, inventory) = + claimed_owner_preparation("owner-preparation-stale").await; + services + .store + .stop_executor(Some(&claim.lease), &claim.lease.owner) + .await + .expect("release preparation lease"); + let built = Arc::new(AtomicBool::new(false)); + let observed = Arc::clone(&built); + let outcome = prepare_owner_claim_with( + &services, + LoopMode::Drain, + claim, + &CancellationToken::new(), + HEARTBEAT_INTERVAL, + move |_| async move { + observed.store(true, Ordering::SeqCst); + Ok(inventory) + }, + ) + .await + .expect("stale preparation converges without mutation"); + assert!(matches!(outcome, OwnerPreparationOutcome::Failed(_))); + assert!(!built.load(Ordering::SeqCst)); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn drain_owner_preparation_cancels_inventory_after_lease_loss() { + struct DropSignal(Arc); + + impl Drop for DropSignal { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } + } + + let (_, services, claim, _) = claimed_owner_preparation("owner-preparation-cancel").await; + let token = claim.lease.clone(); + let dropped = Arc::new(AtomicBool::new(false)); + let observed = Arc::clone(&dropped); + let shutdown = CancellationToken::new(); + let preparation = prepare_owner_claim_with( + &services, + LoopMode::Drain, + claim, + &shutdown, + Duration::from_millis(10), + move |_| async move { + let _drop_signal = DropSignal(observed); + std::future::pending::>().await + }, + ); + let revoke = async { + tokio::time::sleep(Duration::from_millis(20)).await; + services + .store + .stop_executor(Some(&token), &token.owner) + .await + .expect("revoke preparation lease"); + }; + let (outcome, ()) = tokio::join!(preparation, revoke); + assert!(matches!( + outcome.expect("lease loss is typed control flow"), + OwnerPreparationOutcome::Failed(_) + )); + assert!(dropped.load(Ordering::SeqCst)); + } + fn env_of<'a>(set: &'a [(&'a str, &'a str)]) -> impl Fn(&str) -> Option + use<'a> { move |name| { set.iter() diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index d7fd69b3a2a..5d866664ecc 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -5,9 +5,12 @@ Buzz executes whole-community deletion through the typed, one-shot that command as a Kubernetes CronJob; it does not call relay HTTP and it does not add another queue or retry service. -Postgres remains the handoff and source of truth. A run claims only requests -that the deletion store considers runnable, heartbeats the existing lease, and -resumes from durable checkpoints. `concurrencyPolicy: Forbid` prevents scheduled +Postgres remains the handoff and source of truth. A run gives already-approved +work priority. When none is ready, it may claim an authenticated owner-origin +request at `submitted`, build the existing bounded inventory, and atomically +freeze that inventory with a digest-bound `owner_automatic` approval. The same +lease then enters the unchanged executor and resumes from durable checkpoints. +Operator-origin requests never auto-progress. `concurrencyPolicy: Forbid` prevents scheduled pod overlap, `backoffLimit: 0` prevents Kubernetes Job retries, and the deletion store remains authoritative when a pod exits, reaches its deadline, or is replaced. @@ -68,9 +71,11 @@ reports their objects with the `null` version id. ## Runbook -1. Confirm database migrations are current and the deletion request has crossed - the explicit inventory and approval boundary with - `buzz-admin deletions inspect `. +1. Confirm database migrations are current. For operator-origin requests, + confirm explicit inventory and operator approval with + `buzz-admin deletions inspect `. For owner-origin requests, + expect the drain to record `approval_origin: owner_automatic`; `approved_by` + is the immutable mediating operator, not the owner. 2. Confirm the selected Secret contains the required keys and the S3 principal has version-list and exact-version delete permissions. 3. Enable the CronJob and inspect its rendered command and environment before @@ -93,6 +98,10 @@ reports their objects with the `null` version id. 7. If a run fails or times out, fix the recorded dependency or permission failure. Do not add Kubernetes retries: the next scheduled drain consults the durable retry/checkpoint state and resumes only when the store allows it. +7. Use `buzz-admin deletions abort` as privileged recovery while a request is + still at `submitted` or `inventoried` when safe preparation cannot continue. + Abort preserves the archived community and immutable request evidence. An + operator may also `unblock` a remediated preparation failure. ## Deadlines, termination, and the retry budget @@ -129,12 +138,13 @@ killed process. Expect up to roughly a lease duration of delay before the request is runnable again; do not raise the grace period expecting a clean handoff. -The current owner self-serve relay admission records an owner-origin request at -`submitted` and intentionally performs no inventory or approval synchronously. -The deletion engine rejects `submitted` and `inventoried` requests at its -explicit approval boundary. Automating the privileged inventory/approval step -is therefore a separate control-plane slice; enabling this CronJob alone does -not make a newly accepted owner request destructive. +Owner self-serve relay admission still records only a `submitted` row and does +no inventory, approval, S3 work, or execution synchronously. A successful drain +has no human approval step or cooling-off period: authenticated owner intent is +prepared automatically under privileged policy and becomes immediately +eligible for execution. Transient preparation failures use the existing retry +schedule; permanent or exhausted failures block durably. Owner-facing +admission has no cancellation endpoint. The chart has no existing PrometheusRule or provider-neutral CronJob alert integration. Operators must alert on failed/missed Jobs and long-running active diff --git a/migrations/0051_owner_deletion_auto_approval.sql b/migrations/0051_owner_deletion_auto_approval.sql new file mode 100644 index 00000000000..139ffb0bcf2 --- /dev/null +++ b/migrations/0051_owner_deletion_auto_approval.sql @@ -0,0 +1,22 @@ +-- Owner-origin deletion requests may be prepared automatically by the +-- privileged one-shot deletion drain. Manual approvals retain operator +-- provenance and the existing exact inventory-digest foreign key. +SET LOCAL lock_timeout = '5s'; + +ALTER TABLE community_deletion_approvals + ADD COLUMN approval_origin TEXT NOT NULL DEFAULT 'operator' + CHECK (approval_origin IN ('operator', 'owner_automatic')); + +ALTER TABLE community_deletion_requests + DROP CONSTRAINT community_deletion_requests_retry_stage_check, + ADD CONSTRAINT community_deletion_requests_retry_stage_check + CHECK (retry_stage IS NULL OR retry_stage IN ( + 'submitted', 'approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified' + )); + +CREATE INDEX community_deletion_requests_owner_preparable + ON community_deletion_requests (next_attempt_at, created_at) + WHERE request_origin = 'owner' + AND stage = 'submitted' + AND blocked_at IS NULL; diff --git a/schema/schema.sql b/schema/schema.sql index c91cd7375d6..1dc48d555a0 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1252,7 +1252,7 @@ CREATE TABLE community_deletion_requests ( attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), retry_stage TEXT CHECK (retry_stage IS NULL OR retry_stage IN ( - 'approved', 'fenced', 'drained', 'bindings_removed', + 'submitted', 'approved', 'fenced', 'drained', 'bindings_removed', 'postgres_purged', 'cache_purged', 'logically_verified' )), next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), @@ -1300,12 +1300,19 @@ CREATE INDEX community_deletion_requests_runnable 'postgres_purged', 'cache_purged', 'logically_verified'); CREATE INDEX community_deletion_requests_lease ON community_deletion_requests (lease_until) WHERE lease_owner IS NOT NULL; +CREATE INDEX community_deletion_requests_owner_preparable + ON community_deletion_requests (next_attempt_at, created_at) + WHERE request_origin = 'owner' + AND stage = 'submitted' + AND blocked_at IS NULL; CREATE TABLE community_deletion_approvals ( request_id UUID PRIMARY KEY, community_id UUID NOT NULL, inventory_digest BYTEA NOT NULL CHECK (length(inventory_digest) = 32), approved_by TEXT NOT NULL, + approval_origin TEXT NOT NULL DEFAULT 'operator' + CHECK (approval_origin IN ('operator', 'owner_automatic')), note TEXT, approved_at TIMESTAMPTZ NOT NULL DEFAULT now(), FOREIGN KEY (request_id, community_id, inventory_digest) From d9c153f51d6684fdf953ee0a335bbcd5ac759201 Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 09/46] Reject reserved deletion job pod labels Co-authored-by: Codex Signed-off-by: tornquist --- .../charts/buzz/templates/_operator-jobs.tpl | 5 +++++ .../charts/buzz/tests/deletion_drain_test.yaml | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/deploy/charts/buzz/templates/_operator-jobs.tpl b/deploy/charts/buzz/templates/_operator-jobs.tpl index aec91d52ff2..f273fe8ebd3 100644 --- a/deploy/charts/buzz/templates/_operator-jobs.tpl +++ b/deploy/charts/buzz/templates/_operator-jobs.tpl @@ -4,6 +4,11 @@ {{- $root := .root -}} {{- if eq .type "deletionDrain" -}} {{- $job := $root.Values.operatorJobs.deletionDrain -}} +{{- range $label := list "app.kubernetes.io/name" "app.kubernetes.io/instance" "app.kubernetes.io/component" -}} +{{- if hasKey $job.podLabels $label -}} +{{- fail (printf "operatorJobs.deletionDrain.podLabels may not set chart-owned label %q" $label) -}} +{{- end -}} +{{- end -}} apiVersion: batch/v1 kind: CronJob metadata: diff --git a/deploy/charts/buzz/tests/deletion_drain_test.yaml b/deploy/charts/buzz/tests/deletion_drain_test.yaml index 18e0d1a9ef2..5053c9911d3 100644 --- a/deploy/charts/buzz/tests/deletion_drain_test.yaml +++ b/deploy/charts/buzz/tests/deletion_drain_test.yaml @@ -200,6 +200,24 @@ tests: content: name: RELAY_URL + - it: rejects chart-owned pod identity labels + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain: + enabled: true + podLabels: + app.kubernetes.io/component: relay + asserts: + - failedTemplate: + errorPattern: 'operatorJobs.deletionDrain.podLabels may not set chart-owned label "app.kubernetes.io/component"' + - it: rejects enablement without a Redis source set: relayUrl: wss://buzz.example.com From 8ebe3b881c98f19cfc93da7c1d00d4b0618a356e Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 10/46] Record that owner deletion consent is an operator assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The admission path reads as if the relay had verified the owner's consent. It has not: the mediating operator authenticates the owner and collects the acknowledgement out of band, and the request reaching the relay carries only that operator's NIP-98 signature. The relay checks operator authority and current ownership, and stores the owner pubkey, operator pubkey, and acknowledgement version as provenance for the upstream ceremony — never an owner-signed attestation. Also record how manual handoff converges. Owner provenance pins `requested_by` to `owner_pubkey`, so `buzz-admin deletions submit` must repeat the owner pubkey to adopt an admitted request; the operator's own pubkey conflicts with the one-active-request invariant instead. Co-Authored-By: Claude Opus 5 Signed-off-by: tornquist --- ARCHITECTURE.md | 18 ++++++++++++++++++ crates/buzz-db/src/store/deletion.rs | 12 ++++++++++++ crates/buzz-deletion/src/lib.rs | 6 +++++- crates/buzz-relay/src/api/operator.rs | 7 +++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ede53a4b838..a3fa2af013a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -26,6 +26,24 @@ management lists suppress the archived row. Replaying the same UUID converges to its current stage; a different UUID conflicts with the existing one-active- request invariant until that request is aborted. +Owner consent on this path is asserted, not proven. The mediating operator +authenticates the owner and collects the deletion acknowledgement out of band, +upstream of the relay; the request itself carries only the operator's NIP-98 +signature. The relay verifies operator authority and that the asserted pubkey +is still the community's owner, then records the owner pubkey, mediating +operator pubkey, and acknowledgement version as durable provenance for that +upstream ceremony. No owner-signed attestation is required or checked, and +owners have no self-service cancellation. Recovery is a privileged abort, +which stays open across the reversible `submitted`, `inventoried`, `approved`, +and `fenced` stages — releasing the request fence while leaving the community +archived — and closes from `drained` onward, once tenant state is destroyed. + +Manual operator handoff converges on an admitted owner request only when +`buzz-admin deletions submit --requested-by` repeats the owner pubkey recorded +on the row. Owner provenance pins `requested_by` to `owner_pubkey`, so passing +the operator's own pubkey does not converge — it conflicts with the existing +one-active-request invariant instead. + Ownership is mutable only while a community is active. Archiving freezes the current owner. Normal transfer and deployment-root legacy convergence take the same community-row lock as owner-deletion admission, then reject archived, diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 073e9cafd08..333b4da5728 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -748,6 +748,12 @@ impl DeletionStore { } /// Persist a request. Only active non-tombstone communities may be submitted. + /// + /// `requested_by` is recorded on a new row and is also the convergence key + /// when a `submitted` request already exists. Owner provenance pins an + /// owner-origin request's `requested_by` to its `owner_pubkey`, so taking + /// one over manually means passing that owner pubkey; the operator's own + /// pubkey conflicts with the existing request instead of converging. pub async fn submit( &self, community_host: &str, @@ -799,6 +805,12 @@ impl DeletionStore { /// correlation/idempotency identity. Replays return the existing request at /// its current stage. This operation only persists intent; it never /// inventories, approves, quiesces, or executes deletion. + /// + /// The owner's consent arrives as the calling operator's assertion: the + /// operator authenticated the owner and collected the acknowledgement + /// upstream. This layer records that provenance and checks that + /// `owner_pubkey` is still the community's owner; it never verifies an + /// owner-signed attestation. pub async fn admit_owner_request( &self, normalized_community_host: &str, diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 714cf7eaff2..2814c486917 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -227,7 +227,11 @@ pub enum Command { /// Canonical community host. Defaults to RELAY_URL's authority. #[arg(long)] host: Option, - /// Operator identity recorded on the request. + /// Identity recorded on the request. + /// + /// Also the convergence key for an existing `submitted` request: to + /// take over an owner-origin request, pass its owner pubkey. The + /// operator's own pubkey conflicts with it instead of converging. #[arg(long)] requested_by: String, /// Optional reason for the request. diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index eea2dd7c1f1..f02fd3e4ea7 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -343,6 +343,13 @@ pub async fn unarchive_community( /// The request UUID is the correlation/idempotency key. Acceptance is a fast /// PostgreSQL-only transaction and returns `202`; inventory, approval, /// quiescing, object-store access, and executor work remain asynchronous. +/// +/// Owner consent is asserted by the operator, not proven to the relay. The +/// operator authenticates the owner and collects the acknowledgement upstream; +/// this request carries only the operator's NIP-98 signature. Authorization is +/// therefore operator authority plus "the asserted pubkey is still the owner". +/// `owner_pubkey` and `acknowledgement_version` are recorded as provenance for +/// that upstream ceremony, not verified as cryptographic owner consent. pub async fn delete_community( State(state): State>, headers: HeaderMap, From 98f4e91ef117f0f667f8391edd5e154be664405d Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 11/46] Harden owner deletion preparation coverage Co-authored-by: Codex Co-Authored-By: Claude Opus 5 Signed-off-by: tornquist --- crates/buzz-db/src/runtime/migration.rs | 37 ++++++++ crates/buzz-db/src/store/deletion.rs | 69 +++++++++++--- crates/buzz-deletion/src/lib.rs | 116 ++++++++++++++++++++++-- 3 files changed, 201 insertions(+), 21 deletions(-) diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 348d24dc0a8..38a94d4bcf4 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -1838,6 +1838,43 @@ mod postgres_tests { ); } } + let migration_approval_table = migration + .tables + .get("community_deletion_approvals") + .expect("0029 approval table"); + let schema_approval_table = schema + .tables + .get("community_deletion_approvals") + .expect("schema.sql approval table"); + for invariant in [ + "inventory_digest bytea not null check (length(inventory_digest) = 32)", + "foreign key (request_id, community_id, inventory_digest) references community_deletion_requests(id, community_id, inventory_digest) on delete restrict", + ] { + assert!( + migration_approval_table.contains(invariant), + "0029 deletion approvals are missing {invariant}" + ); + assert!( + schema_approval_table.contains(invariant), + "schema.sql deletion approvals are missing {invariant}" + ); + } + let migration_request_table = migration + .tables + .get("community_deletion_requests") + .expect("0029 deletion request table"); + for request_table in [ + migration_request_table, + schema + .tables + .get("community_deletion_requests") + .expect("schema.sql deletion request table"), + ] { + assert!( + request_table.contains("unique (id, community_id, inventory_digest)"), + "deletion requests must expose the exact composite approval target" + ); + } for (function, definition) in &migration.functions { let in_schema = schema .functions diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 167e4503a1e..9682f96f3e4 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -1304,13 +1304,35 @@ impl DeletionStore { &self, owner: &str, lease_duration: Duration, + ) -> Result> { + self.claim_owner_submission(None, owner, lease_duration) + .await + } + + /// Claim one due authenticated owner submission by request id. + pub async fn claim_specific_owner_submission( + &self, + request_id: Uuid, + owner: &str, + lease_duration: Duration, + ) -> Result> { + self.claim_owner_submission(Some(request_id), owner, lease_duration) + .await + } + + async fn claim_owner_submission( + &self, + request_id: Option, + owner: &str, + lease_duration: Duration, ) -> Result> { let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); let mut tx = self.pool.begin().await?; let row = sqlx::query( r#"WITH candidate AS ( SELECT id FROM community_deletion_requests - WHERE request_origin = 'owner' AND acknowledgement_version = $3 + WHERE ($1::uuid IS NULL OR id = $1) + AND request_origin = 'owner' AND acknowledgement_version = $4 AND stage = 'submitted' AND blocked_at IS NULL AND next_attempt_at <= now() AND (lease_until IS NULL OR lease_until < now()) @@ -1318,12 +1340,13 @@ impl DeletionStore { FOR UPDATE SKIP LOCKED LIMIT 1 ) UPDATE community_deletion_requests request - SET lease_owner = $1, lease_generation = lease_generation + 1, - lease_until = now() + make_interval(secs => $2), + SET lease_owner = $2, lease_generation = lease_generation + 1, + lease_until = now() + make_interval(secs => $3), attempts = attempts + 1, updated_at = now() FROM candidate WHERE request.id = candidate.id RETURNING request.*"#, ) + .bind(request_id) .bind(owner) .bind(lease_seconds) .bind(OWNER_DELETION_ACKNOWLEDGEMENT_VERSION) @@ -4954,8 +4977,16 @@ mod postgres_tests { .expect("submit manual request"); let (first, second) = tokio::join!( - store.claim_next_owner_submission("preparer-a", DEFAULT_LEASE_DURATION), - store.claim_next_owner_submission("preparer-b", DEFAULT_LEASE_DURATION), + store.claim_specific_owner_submission( + owner_request.id, + "preparer-a", + DEFAULT_LEASE_DURATION, + ), + store.claim_specific_owner_submission( + owner_request.id, + "preparer-b", + DEFAULT_LEASE_DURATION, + ), ); let claims = [first.expect("first claim"), second.expect("second claim")]; assert_eq!(claims.iter().filter(|claim| claim.is_some()).count(), 1); @@ -4963,6 +4994,15 @@ mod postgres_tests { assert_eq!(claim.request.id, owner_request.id); assert_eq!(claim.request.request_origin, DeletionRequestOrigin::Owner); assert_ne!(claim.request.id, operator_request.id); + assert!(store + .claim_specific_owner_submission( + operator_request.id, + "operator-preparer", + DEFAULT_LEASE_DURATION, + ) + .await + .expect("operator request selection") + .is_none()); store .heartbeat_owner_submission(&claim.lease, "drain", DEFAULT_LEASE_DURATION, false) @@ -4976,7 +5016,7 @@ mod postgres_tests { .await .expect("expire preparation lease"); let successor = store - .claim_next_owner_submission("preparer-c", DEFAULT_LEASE_DURATION) + .claim_specific_owner_submission(owner_request.id, "preparer-c", DEFAULT_LEASE_DURATION) .await .expect("reclaim expired preparation") .expect("expired owner preparation is reclaimable"); @@ -5000,7 +5040,7 @@ mod postgres_tests { .await .expect("admit owner request"); let claim = store - .claim_next_owner_submission("preparer", DEFAULT_LEASE_DURATION) + .claim_specific_owner_submission(request_id, "preparer", DEFAULT_LEASE_DURATION) .await .expect("claim owner request") .expect("owner request is preparable"); @@ -5071,12 +5111,13 @@ mod postgres_tests { let (db, store) = store().await; let (host, owner, community) = archived_owned_community(&db).await; let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); store - .admit_owner_request(&host, &owner, operator, 1, Uuid::new_v4()) + .admit_owner_request(&host, &owner, operator, 1, request_id) .await .expect("admit owner request"); let stale = store - .claim_next_owner_submission("stale-preparer", DEFAULT_LEASE_DURATION) + .claim_specific_owner_submission(request_id, "stale-preparer", DEFAULT_LEASE_DURATION) .await .expect("claim owner request") .expect("owner request is preparable"); @@ -5088,7 +5129,11 @@ mod postgres_tests { .await .expect("expire stale lease"); let successor = store - .claim_next_owner_submission("successor-preparer", DEFAULT_LEASE_DURATION) + .claim_specific_owner_submission( + request_id, + "successor-preparer", + DEFAULT_LEASE_DURATION, + ) .await .expect("reclaim owner request") .expect("expired request is reclaimable"); @@ -5124,7 +5169,7 @@ mod postgres_tests { .await .expect("admit owner request"); let claim = store - .claim_next_owner_submission("preparer", DEFAULT_LEASE_DURATION) + .claim_specific_owner_submission(request_id, "preparer", DEFAULT_LEASE_DURATION) .await .expect("claim owner request") .expect("owner request is preparable"); @@ -5143,7 +5188,7 @@ mod postgres_tests { assert!(retried.lease_owner.is_none()); let claim = store - .claim_next_owner_submission("preparer-2", DEFAULT_LEASE_DURATION) + .claim_specific_owner_submission(request_id, "preparer-2", DEFAULT_LEASE_DURATION) .await .expect("reclaim owner request") .expect("retried request is due"); diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index f7768385568..fcdd69bbd58 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -1929,9 +1929,9 @@ mod postgres_tests { (db, services, claim) } - async fn claimed_owner_preparation( + async fn owner_preparation_fixture( prefix: &str, - ) -> (Db, Services, ClaimedDeletion, FrozenInventory) { + ) -> (Db, Services, DeletionRequest, FrozenInventory) { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) .expect("BUZZ_TEST_DATABASE_URL or DATABASE_URL is required"); @@ -1957,15 +1957,13 @@ mod postgres_tests { .expect("archive owner preparation community") .expect("owned community"); let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; - store + let buzz_db::deletion::OwnerDeletionAdmission::Accepted(request) = store .admit_owner_request(&host, &owner, operator, 1, Uuid::new_v4()) .await - .expect("admit owner request"); - let claim = store - .claim_next_owner_submission("test-preparer", DEFAULT_LEASE_DURATION) - .await - .expect("claim owner request") - .expect("owner request is preparable"); + .expect("admit owner request") + else { + panic!("owner request must be accepted") + }; let inventory = FrozenInventory { schema: store .inventory_schema(community.id) @@ -1998,9 +1996,109 @@ mod postgres_tests { .create_pool(Some(deadpool_redis::Runtime::Tokio1)) .expect("construct unused Redis pool"), }; + (db, services, *request, inventory) + } + + async fn claimed_owner_preparation( + prefix: &str, + ) -> (Db, Services, ClaimedDeletion, FrozenInventory) { + let (db, services, request, inventory) = owner_preparation_fixture(prefix).await; + let claim = services + .store + .claim_specific_owner_submission(request.id, "test-preparer", DEFAULT_LEASE_DURATION) + .await + .expect("claim owner request") + .expect("owner request is preparable"); (db, services, claim, inventory) } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn drain_loop_prioritizes_approved_work_then_dispatches_owner_submission() { + let (db, services, owner_request, _) = + owner_preparation_fixture("owner-preparation-dispatch").await; + let approved_host = format!("approved-{}.example", Uuid::new_v4().simple()); + let approved_community = db + .ensure_configured_community(&approved_host) + .await + .expect("create approved community"); + let approved = services + .store + .submit(&approved_host, "manual-operator", None) + .await + .expect("submit approved request"); + let inventory = FrozenInventory { + schema: services + .store + .inventory_schema(approved_community.id) + .await + .expect("inventory approved community"), + storage: empty_storage_manifest(approved_community.id), + }; + services + .store + .freeze_inventory(approved.id, &inventory) + .await + .expect("freeze approved request"); + services + .store + .approve(approved.id, "manual-approver", None) + .await + .expect("approve request"); + + assert_eq!( + run_loop( + services.clone(), + LoopMode::Drain, + None, + "priority-executor".to_string(), + ) + .await + .expect("approved work failure remains typed"), + 1 + ); + let approved_after = services + .store + .get(approved.id) + .await + .expect("approved request after drain"); + assert!( + approved_after.attempts > 0, + "approved work must be claimed first" + ); + let owner_after_priority = services + .store + .get(owner_request.id) + .await + .expect("owner request after approved work"); + assert_eq!(owner_after_priority.stage, DeletionStage::Submitted); + assert_eq!(owner_after_priority.attempts, 0); + assert!(owner_after_priority.lease_owner.is_none()); + + assert_eq!( + run_loop( + services.clone(), + LoopMode::Drain, + None, + "owner-dispatch-executor".to_string(), + ) + .await + .expect("owner inventory failure remains typed"), + 1 + ); + let owner_after_dispatch = services + .store + .get(owner_request.id) + .await + .expect("owner request after dispatch"); + assert_eq!(owner_after_dispatch.attempts, 1); + assert_eq!( + owner_after_dispatch.retry_stage, + Some(DeletionStage::Submitted) + ); + assert!(owner_after_dispatch.lease_owner.is_none()); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn drain_owner_preparation_approves_for_existing_execution_only() { From 494d5744360f2d39c464e8ac8b83ae2276ccf0f6 Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 12/46] Correct the deletion drain runbook's operational claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runbook told operators to target `cronjob/-buzz-deletion-drain`. `buzz.fullname` collapses to the release name when it already contains the chart name, so the documented name is wrong for the documented install: `helm install buzz ...` renders `buzz-deletion-drain`. Discover the CronJob by its component label instead, and describe the name rule rather than a single guessed spelling. Three more corrections: `activeDeadlineSeconds` was presented as if a timed-out run were just another retry. It is not recorded as one — shutdown releases the claim without recording a retry, and only the object-store drain resumes mid-stage — so a deadline landing repeatedly inside a non-resumable stage loops forever with a rising `attempts`, a flat `retry_count`, and no block. Document how to spot that from both the request and Kubernetes, how to size the deadline, and how to recover. `terminationGracePeriodSeconds` was presented as a clean handoff. Document that a pod still working at the end of the window is SIGKILLed holding its lease, and that recovery is lease expiry plus reclaim under a new generation. An empty `serviceAccountName` was described as a neutral default. It inherits the relay's service account; `automountServiceAccountToken: false` hides the projected token but does not detach cloud IAM bindings resolved through the node metadata path. Recommend a dedicated pre-created account when the executor's IAM blast radius should be smaller than the relay's. Co-Authored-By: Claude Opus 5 Signed-off-by: tornquist --- deploy/charts/buzz/values.yaml | 4 +- docs/operator-community-deletion.md | 67 ++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/deploy/charts/buzz/values.yaml b/deploy/charts/buzz/values.yaml index f43925346e5..d51d37b4bb7 100644 --- a/deploy/charts/buzz/values.yaml +++ b/deploy/charts/buzz/values.yaml @@ -402,7 +402,9 @@ operatorJobs: successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 3 terminationGracePeriodSeconds: 30 - serviceAccountName: "" # defaults to the main Buzz service account + # Empty inherits the relay service account, and with it any cloud IAM + # binding on that account. Name a dedicated account to narrow the executor. + serviceAccountName: "" podLabels: {} podAnnotations: sidecar.istio.io/inject: "false" diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index ec2c20f0004..d7fd69b3a2a 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -15,8 +15,7 @@ replaced. ## Enablement The CronJob is disabled by default. Production deployments should use an -existing Secret and a dedicated service account when their cluster policy -supports one: +existing Secret and a dedicated, pre-created service account: ```yaml secrets: @@ -52,6 +51,16 @@ The pod also disables service-account token automounting and Kubernetes service link environment injection because the executor does not call the Kubernetes API or discover cluster Services. +Leaving `operatorJobs.deletionDrain.serviceAccountName` empty falls back to the +relay's own service account. `automountServiceAccountToken: false` only +suppresses the projected token inside the pod; it does not detach the identity. +Cloud IAM bindings attached to that service account — IRSA on EKS, Workload +Identity on GKE — are resolved by the node/metadata path and still apply, so the +drain pod inherits the relay's cloud permissions. Create a dedicated service +account with only the object-store permissions listed below and name it +explicitly if you want the executor's IAM blast radius to be smaller than the +relay's. + The S3 principal needs the relay's normal object permissions plus bucket-level `s3:ListBucketVersions` and object-level `s3:DeleteObjectVersion` for every tenant-owned prefix. This also applies to never-versioned buckets because S3 @@ -66,14 +75,60 @@ reports their objects with the `null` version id. has version-list and exact-version delete permissions. 3. Enable the CronJob and inspect its rendered command and environment before rollout. -4. Start one staffed manual run with - `kubectl create job --from=cronjob/-buzz-deletion-drain `. -5. Follow pod logs and re-run `buzz-admin deletions inspect ` to +4. Locate the rendered CronJob by label rather than by guessing its name: + + ```sh + kubectl get cronjob -n \ + -l app.kubernetes.io/component=deletion-drain,app.kubernetes.io/instance= + ``` + + The name is `-deletion-drain`. `buzz.fullname` collapses to the + release name when the release name already contains the chart name, so + `helm install buzz ...` renders `buzz-deletion-drain`, not + `buzz-buzz-deletion-drain`. +5. Start one staffed manual run with + `kubectl create job --from=cronjob/ `. +6. Follow pod logs and re-run `buzz-admin deletions inspect ` to verify lease, checkpoint, retry, blocked, and terminal state. -6. If a run fails or times out, fix the recorded dependency or permission +7. If a run fails or times out, fix the recorded dependency or permission failure. Do not add Kubernetes retries: the next scheduled drain consults the durable retry/checkpoint state and resumes only when the store allows it. +## Deadlines, termination, and the retry budget + +`activeDeadlineSeconds` is a Kubernetes-side limit, and the deletion store does +not learn why a pod went away. Two consequences matter when reading state: + +- A `DeadlineExceeded` Job is not recorded as a deletion retry. Shutdown + releases the claim without recording one, so `retry_count` and `blocked_at` + do not advance. Only the object-store drain checkpoints per manifest chunk + and resumes mid-stage; every other stage restarts from its beginning on the + next run. A deadline that keeps landing inside one of those non-resumable + stages therefore repeats indefinitely: each run increments `attempts` and + burns the window again while the retry budget never moves and the request is + never blocked. +- Diagnose this from both sides. `buzz-admin deletions inspect ` + shows a rising `attempts` with a flat `retry_count` and no `last_error`; + Kubernetes holds the reason. The chart labels the CronJob and the drain pods, + but not the generated Jobs, so find the attempts with + `kubectl get pods -n -l app.kubernetes.io/component=deletion-drain` + and read the condition with `kubectl describe job `. + +Size `activeDeadlineSeconds` for the longest single stage this community will +run, not for the average run. To recover, either raise the deadline and let the +schedule pick the request back up, or take one staffed run with +`buzz-admin deletions run ` outside the CronJob's deadline. + +`terminationGracePeriodSeconds` is a best-effort window, not a guarantee. The +drain command handles `SIGTERM` and releases its lease cleanly when it wins the +race, but a pod that is still working when the grace period expires is +`SIGKILL`ed with the lease still held. Nothing is lost: the durable lease simply +expires (60s by default, heartbeated every 10s) and the next run reclaims the +request with a fresh lease generation, which fences any straggler write from the +killed process. Expect up to roughly a lease duration of delay before the +request is runnable again; do not raise the grace period expecting a clean +handoff. + The current owner self-serve relay admission records an owner-origin request at `submitted` and intentionally performs no inventory or approval synchronously. The deletion engine rejects `submitted` and `inventoried` requests at its From e5b3d465748a5eb2d5552ca844c4cd2decdebf8e Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 20:27:15 +0000 Subject: [PATCH 13/46] Serialize owner convergence before deletion abort Co-Authored-By: Claude Opus 5 Signed-off-by: tornquist --- crates/buzz-db/src/store/deletion.rs | 202 +++++++++++++++++++++- crates/buzz-db/src/store/relay_members.rs | 1 + 2 files changed, 202 insertions(+), 1 deletion(-) diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 333b4da5728..09752c10093 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -2874,7 +2874,7 @@ async fn lock_community_deletion( Ok(()) } -async fn lock_community_deletion_shared( +pub(crate) async fn lock_community_deletion_shared( tx: &mut Transaction<'_, Postgres>, community: CommunityId, ) -> Result<()> { @@ -4018,6 +4018,111 @@ mod postgres_tests { .expect("release admission fixture serialization"); } + struct OwnerConvergenceGate { + connection: sqlx::pool::PoolConnection, + trigger_name: String, + function_name: String, + first_key: i32, + second_key: i32, + } + + async fn install_owner_convergence_gate( + db: &Db, + community: CommunityId, + owner: &str, + ) -> OwnerConvergenceGate { + let suffix = Uuid::new_v4().simple().to_string(); + let trigger_name = format!("a_owner_convergence_gate_{suffix}"); + let function_name = format!("owner_convergence_gate_fn_{suffix}"); + let gate_id = Uuid::new_v4(); + let first_key = (gate_id.as_u128() as u32 & 0x7fff_ffff) as i32; + let second_key = ((gate_id.as_u128() >> 32) as u32 & 0x7fff_ffff) as i32; + let mut connection = db.pool.acquire().await.expect("acquire gate connection"); + sqlx::query("SELECT pg_advisory_lock($1, $2)") + .bind(first_key) + .bind(second_key) + .execute(&mut *connection) + .await + .expect("hold owner convergence gate"); + sqlx::query(AssertSqlSafe(format!( + "CREATE FUNCTION {function_name}() RETURNS trigger LANGUAGE plpgsql AS $$ \ + BEGIN \ + IF NEW.community_id = '{}'::uuid AND NEW.pubkey = '{}' THEN \ + PERFORM pg_advisory_xact_lock({first_key}, {second_key}); \ + END IF; \ + RETURN NEW; \ + END $$", + community.as_uuid(), + owner + ))) + .execute(&db.pool) + .await + .expect("install owner convergence gate function"); + sqlx::query(AssertSqlSafe(format!( + "CREATE TRIGGER {trigger_name} BEFORE INSERT ON relay_members \ + FOR EACH ROW EXECUTE FUNCTION {function_name}()" + ))) + .execute(&db.pool) + .await + .expect("install owner convergence gate trigger"); + OwnerConvergenceGate { + connection, + trigger_name, + function_name, + first_key, + second_key, + } + } + + async fn wait_for_owner_convergence_gate(db: &Db, gate: &OwnerConvergenceGate) { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let waiting: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pg_locks \ + WHERE locktype = 'advisory' AND classid = $1::oid AND objid = $2::oid \ + AND objsubid = 2 AND NOT granted)", + ) + .bind(gate.first_key) + .bind(gate.second_key) + .fetch_one(&db.pool) + .await + .expect("inspect owner convergence gate"); + if waiting { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("owner convergence reached gated member update"); + } + + async fn release_owner_convergence_gate(gate: &mut OwnerConvergenceGate) { + sqlx::query("SELECT pg_advisory_unlock($1, $2)") + .bind(gate.first_key) + .bind(gate.second_key) + .execute(&mut *gate.connection) + .await + .expect("release owner convergence gate"); + } + + async fn remove_owner_convergence_gate(db: &Db, gate: OwnerConvergenceGate) { + sqlx::query(AssertSqlSafe(format!( + "DROP TRIGGER {} ON relay_members", + gate.trigger_name + ))) + .execute(&db.pool) + .await + .expect("remove owner convergence gate trigger"); + sqlx::query(AssertSqlSafe(format!( + "DROP FUNCTION {}()", + gate.function_name + ))) + .execute(&db.pool) + .await + .expect("remove owner convergence gate function"); + } + async fn contender_db(application_name: &str) -> Db { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) @@ -4203,6 +4308,101 @@ mod postgres_tests { assert_eq!(membership_roles(&db, community).await, before); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn same_owner_convergence_serializes_with_abort_without_deadlock() { + let mut deadlocks = Vec::new(); + for stage in [DeletionStage::Submitted, DeletionStage::Inventoried] { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let OwnerDeletionAdmission::Accepted(request) = store + .admit_owner_request( + &host, + &owner, + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + 1, + Uuid::new_v4(), + ) + .await + .expect("admit owner deletion") + else { + panic!("expected accepted owner deletion") + }; + if stage == DeletionStage::Inventoried { + let inventory = FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("inventory schema"), + storage: empty_storage_manifest(community), + }; + let inventoried = store + .freeze_inventory(request.id, &inventory) + .await + .expect("freeze inventory"); + assert_eq!(inventoried.stage, DeletionStage::Inventoried); + } + + let mut gate = install_owner_convergence_gate(&db, community, &owner).await; + let convergence_db = + contender_db(&format!("owner-convergence-{}", Uuid::new_v4().simple())).await; + let converging = tokio::spawn({ + let owner = owner.clone(); + async move { convergence_db.provision_owner(community, &owner).await } + }); + wait_for_owner_convergence_gate(&db, &gate).await; + + let abort_application = format!("owner-abort-{}", Uuid::new_v4().simple()); + let abort_store = contender_db(&abort_application).await.deletion_store(); + let aborting = tokio::spawn(async move { + abort_store + .abort(request.id, "operator", "same-owner convergence race") + .await + }); + wait_for_contender_lock(&db, &abort_application).await; + release_owner_convergence_gate(&mut gate).await; + + let (convergence_join, abort_join) = tokio::join!( + tokio::time::timeout(Duration::from_secs(5), converging), + tokio::time::timeout(Duration::from_secs(5), aborting), + ); + remove_owner_convergence_gate(&db, gate).await; + let convergence_result = convergence_join + .expect("same-owner convergence must not hang") + .expect("join same-owner convergence"); + let abort_result = abort_join + .expect("abort must not hang") + .expect("join abort"); + if matches!( + &convergence_result, + Err(DbError::Sqlx(sqlx::Error::Database(error))) + if error.code().as_deref() == Some("40P01") + ) || matches!( + &abort_result, + Err(DbError::Sqlx(sqlx::Error::Database(error))) + if error.code().as_deref() == Some("40P01") + ) { + deadlocks.push(format!( + "{stage}: convergence={convergence_result:?}, abort={abort_result:?}" + )); + continue; + } + assert_eq!( + convergence_result.expect("same-owner convergence"), + ProvisionOwnerResult::Applied + ); + assert_eq!( + abort_result.expect("abort request").stage, + DeletionStage::Aborted + ); + } + assert!( + deadlocks.is_empty(), + "same-owner convergence and abort must serialize without 40P01:\n{}", + deadlocks.join("\n") + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn concurrent_owner_admission_serializes_before_normal_transfer() { diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 3bd09c625fe..20309022cbb 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -422,6 +422,7 @@ async fn lock_owner_mutation_admission( proposed_owner: &str, mode: OwnerMutationMode, ) -> Result { + crate::deletion::lock_community_deletion_shared(tx, community).await?; let target = sqlx::query( "SELECT archived_at, deletion_state, deleted_at FROM communities \ WHERE id = $1 FOR UPDATE", From b02926e2350602a3c82647203c04764538ce989d Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Mon, 28 Sep 2026 09:04:16 -0400 Subject: [PATCH 14/46] Test blocked owner deletion preparation guards Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/src/store/deletion.rs | 96 ++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 9682f96f3e4..1639311873f 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -5028,6 +5028,102 @@ mod postgres_tests { .is_err()); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn blocked_owner_submission_cannot_be_claimed() { + let (db, store) = store().await; + let (host, owner, _) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + store + .admit_owner_request(&host, &owner, operator, 1, request_id) + .await + .expect("admit owner request"); + sqlx::query( + "UPDATE community_deletion_requests SET blocked_at = now(), \ + blocked_reason = 'operator hold' WHERE id = $1", + ) + .bind(request_id) + .execute(&db.pool) + .await + .expect("block owner submission before claim"); + + assert!(store + .claim_specific_owner_submission( + request_id, + "specific-preparer", + DEFAULT_LEASE_DURATION + ) + .await + .expect("specific claim selection") + .is_none()); + assert!(store + .claim_next_owner_submission("next-preparer", DEFAULT_LEASE_DURATION) + .await + .expect("next claim selection") + .is_none()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn blocked_claimed_owner_submission_cannot_heartbeat_or_approve() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + store + .admit_owner_request(&host, &owner, operator, 1, request_id) + .await + .expect("admit owner request"); + let claim = store + .claim_specific_owner_submission(request_id, "preparer", DEFAULT_LEASE_DURATION) + .await + .expect("claim owner request") + .expect("owner request is preparable"); + sqlx::query( + "UPDATE community_deletion_requests SET blocked_at = now(), \ + blocked_reason = 'operator hold' WHERE id = $1", + ) + .bind(request_id) + .execute(&db.pool) + .await + .expect("block owner submission while retaining lease"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("schema inventory"), + storage: empty_storage_manifest(community), + }; + + assert!(store + .heartbeat_owner_submission(&claim.lease, "drain", DEFAULT_LEASE_DURATION, false) + .await + .is_err()); + assert!(store + .complete_owner_preparation(&claim.lease, &inventory) + .await + .is_err()); + assert_eq!( + store + .get(request_id) + .await + .expect("load blocked request") + .stage, + DeletionStage::Submitted + ); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM community_deletion_approvals WHERE request_id = $1", + ) + .bind(request_id) + .fetch_one(&db.pool) + .await + .expect("count automatic approvals"), + 0 + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn owner_preparation_atomically_approves_exact_inventory_and_converges() { From e6714ec408a350761b58c43f8d4152745a5041b4 Mon Sep 17 00:00:00 2001 From: tornquist Date: Mon, 28 Sep 2026 13:24:52 +0000 Subject: [PATCH 15/46] Cover deletion drain chart contracts Exercise both supported credential shapes through the CI render matrix and bind the typed job guards, schema boundary, pod identity, and default sidecar annotation. Co-authored-by: Codex Signed-off-by: tornquist --- .../buzz/tests/deletion_drain_test.yaml | 81 ++++++++++++++++++- .../deletion-drain-bundled-values.yaml | 11 +++ .../production-existing-secret-values.yaml | 3 + 3 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 deploy/charts/buzz/tests/fixtures/deletion-drain-bundled-values.yaml diff --git a/deploy/charts/buzz/tests/deletion_drain_test.yaml b/deploy/charts/buzz/tests/deletion_drain_test.yaml index 5053c9911d3..1f8a6315ced 100644 --- a/deploy/charts/buzz/tests/deletion_drain_test.yaml +++ b/deploy/charts/buzz/tests/deletion_drain_test.yaml @@ -38,7 +38,6 @@ tests: podLabels: tags.datadoghq.com/service: buzz-deletion-drain podAnnotations: - sidecar.istio.io/inject: "false" example.com/operator-job: deletion-drain resources: requests: @@ -96,6 +95,18 @@ tests: - equal: path: spec.jobTemplate.spec.template.metadata.labels["tags.datadoghq.com/service"] value: buzz-deletion-drain + - equal: + path: spec.jobTemplate.spec.template.metadata.labels["app.kubernetes.io/name"] + value: buzz + - equal: + path: spec.jobTemplate.spec.template.metadata.labels["app.kubernetes.io/instance"] + value: RELEASE-NAME + - equal: + path: spec.jobTemplate.spec.template.metadata.labels["app.kubernetes.io/component"] + value: deletion-drain + - equal: + path: spec.jobTemplate.spec.template.metadata.annotations["sidecar.istio.io/inject"] + value: "false" - equal: path: spec.jobTemplate.spec.template.metadata.annotations["example.com/operator-job"] value: deletion-drain @@ -200,7 +211,43 @@ tests: content: name: RELAY_URL - - it: rejects chart-owned pod identity labels + - it: rejects the chart-owned name pod label with an explicit guard error + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain: + enabled: true + podLabels: + app.kubernetes.io/name: relay + asserts: + - failedTemplate: + errorPattern: 'operatorJobs.deletionDrain.podLabels may not set chart-owned label "app.kubernetes.io/name"' + + - it: rejects the chart-owned instance pod label with an explicit guard error + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain: + enabled: true + podLabels: + app.kubernetes.io/instance: another-release + asserts: + - failedTemplate: + errorPattern: 'operatorJobs.deletionDrain.podLabels may not set chart-owned label "app.kubernetes.io/instance"' + + - it: rejects the chart-owned component pod label with an explicit guard error set: relayUrl: wss://buzz.example.com ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" @@ -218,6 +265,36 @@ tests: - failedTemplate: errorPattern: 'operatorJobs.deletionDrain.podLabels may not set chart-owned label "app.kubernetes.io/component"' + - it: rejects existing-secret enablement without an explicit S3 endpoint + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + secrets.existingSecret: buzz-operator-secrets + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.bucket: buzz-media-example + operatorJobs.deletionDrain.enabled: true + asserts: + - failedTemplate: + errorPattern: "s3.endpoint is required when operatorJobs.deletionDrain.enabled=true" + + - it: rejects unknown deletion drain keys through the values schema + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain: + enabled: true + arbitraryCommand: ["unsafe"] + asserts: + - failedTemplate: + errorPattern: "Additional property arbitraryCommand is not allowed" + - it: rejects enablement without a Redis source set: relayUrl: wss://buzz.example.com diff --git a/deploy/charts/buzz/tests/fixtures/deletion-drain-bundled-values.yaml b/deploy/charts/buzz/tests/fixtures/deletion-drain-bundled-values.yaml new file mode 100644 index 00000000000..18d30a696da --- /dev/null +++ b/deploy/charts/buzz/tests/fixtures/deletion-drain-bundled-values.yaml @@ -0,0 +1,11 @@ +relayUrl: wss://buzz.example.com +ownerPubkey: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" +postgresql: + enabled: true +redis: + enabled: true +minio: + enabled: true +operatorJobs: + deletionDrain: + enabled: true diff --git a/deploy/charts/buzz/tests/fixtures/production-existing-secret-values.yaml b/deploy/charts/buzz/tests/fixtures/production-existing-secret-values.yaml index fd4bdadfeda..7020fb32b0b 100644 --- a/deploy/charts/buzz/tests/fixtures/production-existing-secret-values.yaml +++ b/deploy/charts/buzz/tests/fixtures/production-existing-secret-values.yaml @@ -8,6 +8,9 @@ externalPostgresql: url: "postgres://buzz:pw@postgres.example.com:5432/buzz" externalRedis: url: "redis://:pw@redis.example.com:6379" +operatorJobs: + deletionDrain: + enabled: true s3: endpoint: "https://s3.us-east-1.amazonaws.com" bucket: "buzz-media" From 10372630dcb1447bd6f654cfca7b59bc0bed908e Mon Sep 17 00:00:00 2001 From: tornquist Date: Mon, 28 Sep 2026 13:25:36 +0000 Subject: [PATCH 16/46] Bound operator CronJob names Share a suffix-preserving 52-character name helper between the deletion drain and storage accounting CronJobs while retaining their existing short names. Co-authored-by: Codex Signed-off-by: tornquist --- deploy/charts/buzz/templates/_helpers.tpl | 7 ++++++ .../charts/buzz/templates/_operator-jobs.tpl | 2 +- .../templates/storage-accounting-cronjob.yaml | 2 +- .../buzz/tests/deletion_drain_test.yaml | 19 ++++++++++++++++ .../buzz/tests/storage_accounting_test.yaml | 22 +++++++++++++++++++ docs/operator-community-deletion.md | 6 +++-- 6 files changed, 54 insertions(+), 4 deletions(-) diff --git a/deploy/charts/buzz/templates/_helpers.tpl b/deploy/charts/buzz/templates/_helpers.tpl index e5ec9182d68..c00e643fdb3 100644 --- a/deploy/charts/buzz/templates/_helpers.tpl +++ b/deploy/charts/buzz/templates/_helpers.tpl @@ -17,6 +17,13 @@ {{- end -}} {{- end -}} +{{/* Kubernetes CronJob names are limited to 52 characters. */}} +{{- define "buzz.cronJobName" -}} +{{- $maxBaseLength := sub 51 (len .suffix) | int -}} +{{- $base := include "buzz.fullname" .root | trunc $maxBaseLength | trimSuffix "-" -}} +{{- printf "%s-%s" $base .suffix -}} +{{- end -}} + {{- define "buzz.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} diff --git a/deploy/charts/buzz/templates/_operator-jobs.tpl b/deploy/charts/buzz/templates/_operator-jobs.tpl index f273fe8ebd3..49a204f9ec9 100644 --- a/deploy/charts/buzz/templates/_operator-jobs.tpl +++ b/deploy/charts/buzz/templates/_operator-jobs.tpl @@ -12,7 +12,7 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "buzz.fullname" $root }}-deletion-drain + name: {{ include "buzz.cronJobName" (dict "root" $root "suffix" "deletion-drain") }} labels: {{- include "buzz.labels" $root | nindent 4 }} app.kubernetes.io/component: deletion-drain diff --git a/deploy/charts/buzz/templates/storage-accounting-cronjob.yaml b/deploy/charts/buzz/templates/storage-accounting-cronjob.yaml index deaa816888e..f753bd3b412 100644 --- a/deploy/charts/buzz/templates/storage-accounting-cronjob.yaml +++ b/deploy/charts/buzz/templates/storage-accounting-cronjob.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "buzz.fullname" . }}-storage-accounting + name: {{ include "buzz.cronJobName" (dict "root" . "suffix" "storage-accounting") }} labels: {{- include "buzz.labels" . | nindent 4 }} app.kubernetes.io/component: storage-accounting diff --git a/deploy/charts/buzz/tests/deletion_drain_test.yaml b/deploy/charts/buzz/tests/deletion_drain_test.yaml index 1f8a6315ced..cf2b7e8556a 100644 --- a/deploy/charts/buzz/tests/deletion_drain_test.yaml +++ b/deploy/charts/buzz/tests/deletion_drain_test.yaml @@ -307,3 +307,22 @@ tests: asserts: - failedTemplate: errorPattern: "operatorJobs.deletionDrain requires Redis" + + - it: preserves the deletion suffix while bounding a long release and name override + release: + name: very-long-release-name-for-operator-jobs + set: + nameOverride: custom-buzz-name + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain.enabled: true + asserts: + - equal: + path: metadata.name + value: very-long-release-name-for-operator-j-deletion-drain diff --git a/deploy/charts/buzz/tests/storage_accounting_test.yaml b/deploy/charts/buzz/tests/storage_accounting_test.yaml index d37d8aefe98..563e1ec2283 100644 --- a/deploy/charts/buzz/tests/storage_accounting_test.yaml +++ b/deploy/charts/buzz/tests/storage_accounting_test.yaml @@ -53,6 +53,10 @@ tests: path: kind value: CronJob template: templates/storage-accounting-cronjob.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-buzz-storage-accounting + template: templates/storage-accounting-cronjob.yaml - equal: path: spec.concurrencyPolicy value: Forbid @@ -143,3 +147,21 @@ tests: content: name: BUZZ_GIT_HOOK_HMAC_SECRET template: templates/storage-accounting-cronjob.yaml + + - it: preserves the storage suffix without a trailing separator after truncation + set: + fullnameOverride: storage-accounting-fullname-that-needs-truncation-at-the-separator + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + storageAccounting.enabled: true + asserts: + - equal: + path: metadata.name + value: storage-accounting-fullname-that-storage-accounting + template: templates/storage-accounting-cronjob.yaml diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index d7fd69b3a2a..546f73cf3d2 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -82,8 +82,10 @@ reports their objects with the `null` version id. -l app.kubernetes.io/component=deletion-drain,app.kubernetes.io/instance= ``` - The name is `-deletion-drain`. `buzz.fullname` collapses to the - release name when the release name already contains the chart name, so + The name is `-deletion-drain`. The chart truncates only the + fullname portion when needed so the suffix remains stable within Kubernetes' + 52-character CronJob name limit. `buzz.fullname` collapses to the release + name when the release name already contains the chart name, so `helm install buzz ...` renders `buzz-deletion-drain`, not `buzz-buzz-deletion-drain`. 5. Start one staffed manual run with From 7e3e142e630041fda803cc9de88eff346ad436ed Mon Sep 17 00:00:00 2001 From: tornquist Date: Mon, 28 Sep 2026 13:25:59 +0000 Subject: [PATCH 17/46] Clarify deletion job workload identity Distinguish the disabled Kubernetes API token mount from provider workload-identity credentials while retaining the dedicated service-account recommendation. Co-authored-by: Codex Signed-off-by: tornquist --- deploy/charts/buzz/README.md | 6 ++++-- docs/operator-community-deletion.md | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index 288681b0c98..4e72ee5debf 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -125,8 +125,10 @@ checkpoints remain the execution authority. The pod receives only `DATABASE_URL`, `REDIS_URL`, and required S3 configuration/credential variables. It does not receive the relay private key, -git-hook secret, relay URL, service-account token, service links, or a generic -environment registry. Schedule, +git-hook secret, relay URL, service links, or a generic environment registry. +The chart disables the ordinary Kubernetes API service-account token mount; +platform workload-identity admission may still inject its own projected token +and provider environment variables. Schedule, deadline, history, termination grace, resources, service account, pod labels, and pod annotations are independently configurable under `operatorJobs.deletionDrain`. diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index 546f73cf3d2..61d8fd89456 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -49,17 +49,19 @@ non-secret S3 settings. It does not receive `BUZZ_RELAY_PRIVATE_KEY`, `BUZZ_GIT_HOOK_HMAC_SECRET`, `RELAY_URL`, or the full Secret through `envFrom`. The pod also disables service-account token automounting and Kubernetes service link environment injection because the executor does not call the Kubernetes -API or discover cluster Services. +API or discover cluster Services. This disables the ordinary Kubernetes API +token mount, not credentials injected by a platform workload-identity +mechanism. Leaving `operatorJobs.deletionDrain.serviceAccountName` empty falls back to the -relay's own service account. `automountServiceAccountToken: false` only -suppresses the projected token inside the pod; it does not detach the identity. -Cloud IAM bindings attached to that service account — IRSA on EKS, Workload -Identity on GKE — are resolved by the node/metadata path and still apply, so the -drain pod inherits the relay's cloud permissions. Create a dedicated service -account with only the object-store permissions listed below and name it -explicitly if you want the executor's IAM blast radius to be smaller than the -relay's. +relay's own service account, including cloud IAM attached through the platform's +workload-identity mechanism. For example, the EKS IRSA admission webhook can +inject its projected web-identity token and AWS environment variables despite +`automountServiceAccountToken: false`; other platforms provide their own +identity mechanism. The drain pod therefore inherits the relay's cloud role by +default. Create a dedicated service account with only the object-store +permissions listed below and name it explicitly if you want the executor's IAM +blast radius to be smaller than the relay's. The S3 principal needs the relay's normal object permissions plus bucket-level `s3:ListBucketVersions` and object-level `s3:DeleteObjectVersion` for every From dfd5db4e087b77215d6a81ab81342d025e28c3e4 Mon Sep 17 00:00:00 2001 From: tornquist Date: Mon, 28 Sep 2026 14:46:10 +0000 Subject: [PATCH 18/46] Scope video menu probes to emitted messages Bind each context-menu interaction to the exact mock message returned by the emitter so same-second ordering cannot select the wrong video. Co-authored-by: Codex Signed-off-by: tornquist --- desktop/tests/e2e/video-attachment.spec.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/desktop/tests/e2e/video-attachment.spec.ts b/desktop/tests/e2e/video-attachment.spec.ts index 2c86f5a9bda..060d356d28e 100644 --- a/desktop/tests/e2e/video-attachment.spec.ts +++ b/desktop/tests/e2e/video-attachment.spec.ts @@ -1513,12 +1513,14 @@ test("right-click menus expose distinct selectors for links, relay video, and of // ── Relay video menu: Download video + Copy link, appearing only once the // relay origin resolves (the reactivity fix) ───────────────────────────── - await emitVideoMessage(page, { + const relayMessage = (await emitVideoMessage(page, { url: MENU_RELAY_VIDEO_URL, sha: MENU_RELAY_VIDEO_SHA, filename: "relay-clip.mp4", - }); - const relayPlayer = page.getByTestId("video-player").last(); + })) as { id: string }; + const relayPlayer = page + .locator(`[data-message-id="${relayMessage.id}"]`) + .getByTestId("video-player"); await expect(relayPlayer).toBeVisible(); // Right-click the player surface. `force` skips the actionability guard: the // Play-button overlay sits above the video, but the contextmenu event still @@ -1559,12 +1561,14 @@ test("right-click menus expose distinct selectors for links, relay video, and of await expect(page.locator("[data-video-context-menu]")).toHaveCount(0); // ── Off-relay video control: renders and offers Copy link, never Download ─ - await emitVideoMessage(page, { + const offRelayMessage = (await emitVideoMessage(page, { url: MENU_OFF_RELAY_VIDEO_URL, sha: MENU_OFF_RELAY_VIDEO_SHA, filename: "external-clip.mp4", - }); - const offRelayPlayer = page.getByTestId("video-player").last(); + })) as { id: string }; + const offRelayPlayer = page + .locator(`[data-message-id="${offRelayMessage.id}"]`) + .getByTestId("video-player"); await expect(offRelayPlayer).toBeVisible(); await offRelayPlayer.click({ button: "right", force: true }); From fcd831ea241985eab065233869642d1f891ba183 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 15:21:08 +0000 Subject: [PATCH 19/46] Renumber owner deletion admission migration Signed-off-by: Codex Co-authored-by: Codex --- crates/buzz-db/src/runtime/migration.rs | 12 ++++++------ .../runtime/tests/thread_window_postgres_tests.rs | 2 +- ...l => 0051_owner_community_deletion_admission.sql} | 0 3 files changed, 7 insertions(+), 7 deletions(-) rename migrations/{0050_owner_community_deletion_admission.sql => 0051_owner_community_deletion_admission.sql} (100%) diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 7b64547c2f5..9bcc5654eeb 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -705,7 +705,7 @@ mod postgres_tests { assert_eq!(migrations.len(), 50); assert_eq!(migrations[48].version, 49); - assert_eq!(migrations[49].version, 50); + assert_eq!(migrations[49].version, 51); assert!(migrations[48] .sql .as_str() @@ -1770,10 +1770,10 @@ mod postgres_tests { .expect("embedded migration 0029") .sql .as_ref(); - let migration_0050: &str = MIGRATOR + let migration_0051: &str = MIGRATOR .iter() - .find(|migration| migration.version == 50) - .expect("embedded migration 0050") + .find(|migration| migration.version == 51) + .expect("embedded migration 0051") .sql .as_ref(); let workspace_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) @@ -1784,7 +1784,7 @@ mod postgres_tests { .expect("read schema/schema.sql"); let migration = surface(migration_0029); - let owner_admission_migration = surface(migration_0050); + let owner_admission_migration = surface(migration_0051); let schema = surface(&schema_sql); assert_eq!( @@ -1830,7 +1830,7 @@ mod postgres_tests { owner_admission_migration .functions .get("prevent_community_deletion_request_retargeting") - .expect("0050 deletion retargeting guard"), + .expect("0051 deletion retargeting guard"), "schema.sql must carry the latest immutable owner-provenance guard" ); let request_table = schema diff --git a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs index 5ca0185af56..5d51f4348d5 100644 --- a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs @@ -311,7 +311,7 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() production_result.is_ok(), "production migrator must preserve ingestion progress: {production_result:?}" ); - assert_eq!(version, 50); + assert_eq!(version, 51); assert_eq!(final_oid, oid, "prebuild must not be replaced"); assert_eq!(count, 4, "all writer witnesses must persist"); } diff --git a/migrations/0050_owner_community_deletion_admission.sql b/migrations/0051_owner_community_deletion_admission.sql similarity index 100% rename from migrations/0050_owner_community_deletion_admission.sql rename to migrations/0051_owner_community_deletion_admission.sql From 1f1b5f4d39f2639053ba58bdd66213609df8dea3 Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Mon, 28 Sep 2026 13:13:09 -0400 Subject: [PATCH 20/46] docs: clarify community deletion abort boundary Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- ARCHITECTURE.md | 3 ++- crates/buzz-db/src/store/deletion.rs | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a3fa2af013a..2de723c5e3e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -36,7 +36,8 @@ upstream ceremony. No owner-signed attestation is required or checked, and owners have no self-service cancellation. Recovery is a privileged abort, which stays open across the reversible `submitted`, `inventoried`, `approved`, and `fenced` stages — releasing the request fence while leaving the community -archived — and closes from `drained` onward, once tenant state is destroyed. +archived — and closes from `drained` onward, when tenant-state destruction may +have begun. Manual operator handoff converges on an admitted owner request only when `buzz-admin deletions submit --requested-by` repeats the owner pubkey recorded diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 09752c10093..cdd13d94c81 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -2207,7 +2207,8 @@ impl DeletionStore { /// privileged path is the only recovery when preparation cannot continue. /// Abort reverses deletion intent, not the owner's archive decision: the /// community stays archived and the owner restores it explicitly. - /// Stages from `drained` onward have destroyed tenant state and stay closed. + /// Stages from `drained` onward stay closed because tenant-state destruction + /// may have begun. pub async fn abort( &self, request_id: Uuid, @@ -4758,8 +4759,8 @@ mod postgres_tests { ); } - /// The reversible boundary stops at `inventoried`. Once execution has - /// destroyed anything, abort must stay closed. + /// The reversible boundary extends through `fenced`. From `drained` + /// onward, destruction may have begun, so abort must stay closed. #[tokio::test] #[ignore = "requires Postgres"] async fn privileged_abort_spans_only_the_reversible_pre_destruction_boundary() { From dc0a125cc20f727a546b4afb24305a4c7e2562d9 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 18:19:00 +0000 Subject: [PATCH 21/46] test(deletion): synchronize lease-loss cancellation Signed-off-by: Codex Co-authored-by: Codex --- crates/buzz-deletion/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index fcdd69bbd58..ae32af57eeb 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -2236,6 +2236,7 @@ mod postgres_tests { let dropped = Arc::new(AtomicBool::new(false)); let observed = Arc::clone(&dropped); let shutdown = CancellationToken::new(); + let (inventory_started_tx, inventory_started_rx) = tokio::sync::oneshot::channel(); let preparation = prepare_owner_claim_with( &services, LoopMode::Drain, @@ -2244,11 +2245,14 @@ mod postgres_tests { Duration::from_millis(10), move |_| async move { let _drop_signal = DropSignal(observed); + inventory_started_tx + .send(()) + .expect("signal inventory started"); std::future::pending::>().await }, ); let revoke = async { - tokio::time::sleep(Duration::from_millis(20)).await; + inventory_started_rx.await.expect("inventory started"); services .store .stop_executor(Some(&token), &token.owner) From e56f4d4c3aaf2099effe421ba15e76cc4a83caa4 Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 22/46] Add typed deletion drain operator CronJob Co-authored-by: Codex Signed-off-by: tornquist --- ARCHITECTURE.md | 9 + deploy/charts/buzz/Chart.yaml | 4 +- deploy/charts/buzz/README.md | 22 +- .../charts/buzz/templates/_operator-jobs.tpl | 86 +++++++ deploy/charts/buzz/templates/_validate.tpl | 7 + .../templates/deletion-drain-cronjob.yaml | 4 + .../buzz/tests/deletion_drain_test.yaml | 214 ++++++++++++++++++ deploy/charts/buzz/values.schema.json | 28 +++ deploy/charts/buzz/values.yaml | 23 ++ docs/operator-community-deletion.md | 87 +++++++ 10 files changed, 481 insertions(+), 3 deletions(-) create mode 100644 deploy/charts/buzz/templates/_operator-jobs.tpl create mode 100644 deploy/charts/buzz/templates/deletion-drain-cronjob.yaml create mode 100644 deploy/charts/buzz/tests/deletion_drain_test.yaml create mode 100644 docs/operator-community-deletion.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8aecbd61cc4..d30b9c9a2f5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -750,10 +750,19 @@ Subcommands: | `remove-member` | Remove a pubkey from the relay membership list (`--pubkey`, optional `--role` guard); publishes kind:13534 roster | | `list-members` | List all relay members | | `generate-key` | Generate a new Nostr keypair (for bootstrapping) | +| `deletions` | Submit, inspect, approve, abort, unblock, run, or drain durable whole-community deletion requests | +| `storage-snapshot` | Run one isolated S3 accounting scan and publish its complete Postgres snapshot | | `reconcile-channels` | Emit kind:39000/39002 discovery events for channels missing them (idempotent) | The `buzz-admin` binary is shipped in the relay Docker image (`/usr/local/bin/buzz-admin`) and is the recommended way to manage relay membership in production. Use `./run.sh add-member`, `./run.sh remove-member`, and `./run.sh list-members` in Docker Compose deployments. +Kubernetes deployments may schedule the typed one-shot +`buzz-admin deletions drain` command directly. The pod owns its bounded +Postgres/Redis clients and S3 client; it does not call relay HTTP. Durable +requests, leases, retry timing, and checkpoints in Postgres are the handoff and +execution authority, so Kubernetes uses `Forbid` concurrency and zero Job +retries rather than introducing a second retry system. + --- ### buzz-test-client — Integration Test Harness diff --git a/deploy/charts/buzz/Chart.yaml b/deploy/charts/buzz/Chart.yaml index e26a2815fff..55d9db5488f 100644 --- a/deploy/charts/buzz/Chart.yaml +++ b/deploy/charts/buzz/Chart.yaml @@ -7,7 +7,7 @@ description: | PostgreSQL and Redis. Configurable for single-node evaluation (subcharts on) and HA production (external services, existingSecret). type: application -version: 0.1.9 +version: 0.1.10 appVersion: "0.1.0" home: https://github.com/block/buzz sources: @@ -24,7 +24,7 @@ maintainers: annotations: artifacthub.io/changes: | - kind: added - description: Optional isolated storage-accounting CronJob with durable relay snapshots. + description: Optional typed deletion-drain operator CronJob using durable database leases. artifacthub.io/license: Apache-2.0 # Optional eval-only subcharts. Production deploys disable both and point diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index f5075fd19d5..1bdff7592e9 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -12,7 +12,7 @@ This chart has two operating profiles selected by values: ## Quickstart (eval only) ```sh -helm install buzz oci://ghcr.io/block/buzz/charts/buzz --version 0.1.8 \ +helm install buzz oci://ghcr.io/block/buzz/charts/buzz --version 0.1.10 \ --create-namespace --namespace buzz \ --set quickstart=true \ --set postgresql.enabled=true \ @@ -116,6 +116,26 @@ is still parsed strictly, but reachability and addressing errors surface on the first storage operation. `/_readiness` tests no external dependency in either case — see the readiness contract below. +## Community deletion operator job + +`operatorJobs.deletionDrain` is a disabled-by-default, typed CronJob for +`/usr/local/bin/buzz-admin deletions drain`. It runs inside the relay image with +bounded Job lifetime/history, `concurrencyPolicy: Forbid`, `backoffLimit: 0`, +and no relay HTTP call. Postgres deletion requests, leases, retries, and +checkpoints remain the execution authority. + +The pod receives only `DATABASE_URL`, `REDIS_URL`, and required S3 +configuration/credential variables. It does not receive the relay private key, +git-hook secret, relay URL, service-account token, service links, or a generic +environment registry. Schedule, +deadline, history, termination grace, resources, service account, pod labels, +and pod annotations are independently configurable under +`operatorJobs.deletionDrain`. + +See [`docs/operator-community-deletion.md`](../../../docs/operator-community-deletion.md) +for enablement, permissions, the staffed first-run procedure, failure recovery, +and the current explicit approval/alerting boundaries. + ### Early-startup telemetry contract `buzz_process_lifecycle` JSON records are the authoritative history for the diff --git a/deploy/charts/buzz/templates/_operator-jobs.tpl b/deploy/charts/buzz/templates/_operator-jobs.tpl new file mode 100644 index 00000000000..aec91d52ff2 --- /dev/null +++ b/deploy/charts/buzz/templates/_operator-jobs.tpl @@ -0,0 +1,86 @@ +{{/* Closed rendering foundation for typed Buzz operator jobs. */}} + +{{- define "buzz.operatorCronJob" -}} +{{- $root := .root -}} +{{- if eq .type "deletionDrain" -}} +{{- $job := $root.Values.operatorJobs.deletionDrain -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "buzz.fullname" $root }}-deletion-drain + labels: + {{- include "buzz.labels" $root | nindent 4 }} + app.kubernetes.io/component: deletion-drain +spec: + schedule: {{ $job.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: {{ $job.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ $job.failedJobsHistoryLimit }} + jobTemplate: + spec: + activeDeadlineSeconds: {{ $job.activeDeadlineSeconds }} + backoffLimit: 0 + template: + metadata: + labels: + {{- include "buzz.selectorLabels" $root | nindent 12 }} + app.kubernetes.io/component: deletion-drain + {{- with $job.podLabels }} + {{- toYaml . | nindent 12 }} + {{- end }} + annotations: + {{- toYaml $job.podAnnotations | nindent 12 }} + spec: + restartPolicy: Never + terminationGracePeriodSeconds: {{ $job.terminationGracePeriodSeconds }} + serviceAccountName: {{ default (include "buzz.serviceAccountName" $root) $job.serviceAccountName }} + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + {{- toYaml $root.Values.relay.securityContext | nindent 12 }} + {{- with $root.Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: deletion-drain + image: {{ include "buzz.image" $root }} + imagePullPolicy: {{ $root.Values.image.pullPolicy }} + securityContext: + {{- omit $root.Values.relay.containerSecurityContext "readOnlyRootFilesystem" | toYaml | nindent 16 }} + readOnlyRootFilesystem: true + command: ["/usr/local/bin/buzz-admin"] + args: ["deletions", "drain"] + env: + - { name: BUZZ_S3_ENDPOINT, value: {{ required "s3.endpoint is required when operatorJobs.deletionDrain.enabled=true" (include "buzz.s3Endpoint" $root) | quote }} } + - { name: BUZZ_S3_BUCKET, value: {{ required "s3.bucket is required when operatorJobs.deletionDrain.enabled=true" $root.Values.s3.bucket | quote }} } + - { name: BUZZ_S3_REGION, value: {{ $root.Values.s3.region | quote }} } + - { name: BUZZ_S3_ADDRESSING_STYLE, value: {{ $root.Values.s3.addressingStyle | quote }} } + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "buzz.envSecretName" $root }} + key: DATABASE_URL + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: {{ include "buzz.envSecretName" $root }} + key: REDIS_URL + - name: BUZZ_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "buzz.envSecretName" $root }} + key: BUZZ_S3_ACCESS_KEY + optional: true + - name: BUZZ_S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "buzz.envSecretName" $root }} + key: BUZZ_S3_SECRET_KEY + optional: true + resources: + {{- toYaml $job.resources | nindent 16 }} +{{- else -}} +{{- fail (printf "unsupported typed operator job %q" .type) -}} +{{- end -}} +{{- end -}} diff --git a/deploy/charts/buzz/templates/_validate.tpl b/deploy/charts/buzz/templates/_validate.tpl index aa7f7ac13cf..8d055f825ed 100644 --- a/deploy/charts/buzz/templates/_validate.tpl +++ b/deploy/charts/buzz/templates/_validate.tpl @@ -18,6 +18,13 @@ surface at template time regardless of which manifest helm renders first. {{- end -}} {{- end -}} +{{/* The deletion executor always uses Redis for tenant-scoped invalidation. */}} +{{- if .Values.operatorJobs.deletionDrain.enabled -}} + {{- if and (not .Values.redis.enabled) (not .Values.externalRedis.url) (not .Values.secrets.existingSecret) -}} + {{- fail "operatorJobs.deletionDrain requires Redis. Enable redis.enabled=true, set externalRedis.url, or provide secrets.existingSecret with key REDIS_URL." -}} + {{- end -}} +{{- end -}} + {{/* Multiple replicas do NOT require ReadWriteMany git storage. Git ref/object state is object-store-backed: every read and write hydrates diff --git a/deploy/charts/buzz/templates/deletion-drain-cronjob.yaml b/deploy/charts/buzz/templates/deletion-drain-cronjob.yaml new file mode 100644 index 00000000000..84acc56590a --- /dev/null +++ b/deploy/charts/buzz/templates/deletion-drain-cronjob.yaml @@ -0,0 +1,4 @@ +{{- include "buzz.validate" . -}} +{{- if .Values.operatorJobs.deletionDrain.enabled }} +{{- include "buzz.operatorCronJob" (dict "root" . "type" "deletionDrain") }} +{{- end }} diff --git a/deploy/charts/buzz/tests/deletion_drain_test.yaml b/deploy/charts/buzz/tests/deletion_drain_test.yaml new file mode 100644 index 00000000000..18e0d1a9ef2 --- /dev/null +++ b/deploy/charts/buzz/tests/deletion_drain_test.yaml @@ -0,0 +1,214 @@ +suite: typed deletion drain operator job +templates: + - templates/deletion-drain-cronjob.yaml +tests: + - it: renders no deletion executor by default + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.accessKey: test + s3.secretKey: test + asserts: + - hasDocuments: + count: 0 + + - it: renders the bounded typed drain command with isolated pod controls + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.region: us-west-2 + s3.addressingStyle: virtual + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain: + enabled: true + schedule: "*/7 * * * *" + activeDeadlineSeconds: 900 + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 4 + terminationGracePeriodSeconds: 45 + serviceAccountName: buzz-deletion-drain + podLabels: + tags.datadoghq.com/service: buzz-deletion-drain + podAnnotations: + sidecar.istio.io/inject: "false" + example.com/operator-job: deletion-drain + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + asserts: + - hasDocuments: + count: 1 + - isAPIVersion: + of: batch/v1 + - isKind: + of: CronJob + - equal: + path: metadata.name + value: RELEASE-NAME-buzz-deletion-drain + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: deletion-drain + - equal: + path: spec.schedule + value: "*/7 * * * *" + - equal: + path: spec.concurrencyPolicy + value: Forbid + - equal: + path: spec.successfulJobsHistoryLimit + value: 2 + - equal: + path: spec.failedJobsHistoryLimit + value: 4 + - equal: + path: spec.jobTemplate.spec.activeDeadlineSeconds + value: 900 + - equal: + path: spec.jobTemplate.spec.backoffLimit + value: 0 + - equal: + path: spec.jobTemplate.spec.template.spec.restartPolicy + value: Never + - equal: + path: spec.jobTemplate.spec.template.spec.terminationGracePeriodSeconds + value: 45 + - equal: + path: spec.jobTemplate.spec.template.spec.serviceAccountName + value: buzz-deletion-drain + - equal: + path: spec.jobTemplate.spec.template.spec.automountServiceAccountToken + value: false + - equal: + path: spec.jobTemplate.spec.template.spec.enableServiceLinks + value: false + - equal: + path: spec.jobTemplate.spec.template.metadata.labels["tags.datadoghq.com/service"] + value: buzz-deletion-drain + - equal: + path: spec.jobTemplate.spec.template.metadata.annotations["example.com/operator-job"] + value: deletion-drain + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].command + value: ["/usr/local/bin/buzz-admin"] + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].args + value: ["deletions", "drain"] + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.requests.cpu + value: 250m + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources.limits.memory + value: 1Gi + + - it: exposes only database redis and deletion S3 configuration + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + secrets.existingSecret: buzz-operator-secrets + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.region: us-west-2 + s3.addressingStyle: virtual + operatorJobs.deletionDrain.enabled: true + asserts: + - lengthEqual: + path: spec.jobTemplate.spec.template.spec.containers[0].env + count: 8 + - notExists: + path: spec.jobTemplate.spec.template.spec.containers[0].envFrom + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: DATABASE_URL + valueFrom: + secretKeyRef: + name: buzz-operator-secrets + key: DATABASE_URL + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: REDIS_URL + valueFrom: + secretKeyRef: + name: buzz-operator-secrets + key: REDIS_URL + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_ENDPOINT + value: https://s3.example.com + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_BUCKET + value: buzz-media-example + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_REGION + value: us-west-2 + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_ADDRESSING_STYLE + value: virtual + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: buzz-operator-secrets + key: BUZZ_S3_ACCESS_KEY + optional: true + - contains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: buzz-operator-secrets + key: BUZZ_S3_SECRET_KEY + optional: true + - notContains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_RELAY_PRIVATE_KEY + - notContains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: BUZZ_GIT_HOOK_HMAC_SECRET + - notContains: + path: spec.jobTemplate.spec.template.spec.containers[0].env + content: + name: RELAY_URL + + - it: rejects enablement without a Redis source + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + s3.endpoint: https://s3.example.com + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain.enabled: true + asserts: + - failedTemplate: + errorPattern: "operatorJobs.deletionDrain requires Redis" diff --git a/deploy/charts/buzz/values.schema.json b/deploy/charts/buzz/values.schema.json index ea5db6398dd..309b2f5c37f 100644 --- a/deploy/charts/buzz/values.schema.json +++ b/deploy/charts/buzz/values.schema.json @@ -234,6 +234,34 @@ "resources": { "type": "object" } } }, + "operatorJobs": { + "type": "object", + "additionalProperties": false, + "properties": { + "deletionDrain": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" }, + "schedule": { "type": "string", "minLength": 1 }, + "activeDeadlineSeconds": { "type": "integer", "minimum": 1, "maximum": 86400 }, + "successfulJobsHistoryLimit": { "type": "integer", "minimum": 0, "maximum": 10 }, + "failedJobsHistoryLimit": { "type": "integer", "minimum": 0, "maximum": 10 }, + "terminationGracePeriodSeconds": { "type": "integer", "minimum": 1, "maximum": 300 }, + "serviceAccountName": { "type": "string" }, + "podLabels": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "podAnnotations": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "resources": { "type": "object" } + } + } + } + }, "minio": { "type": "object", "additionalProperties": false, diff --git a/deploy/charts/buzz/values.yaml b/deploy/charts/buzz/values.yaml index 496c32e979f..f43925346e5 100644 --- a/deploy/charts/buzz/values.yaml +++ b/deploy/charts/buzz/values.yaml @@ -391,6 +391,29 @@ storageAccounting: cpu: "1" memory: 20Gi +# Typed, one-shot operator jobs. The chart intentionally exposes no generic +# command or environment registry: each job has a reviewed executable and +# least-privilege environment contract. +operatorJobs: + deletionDrain: + enabled: false + schedule: "*/5 * * * *" + activeDeadlineSeconds: 3600 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + terminationGracePeriodSeconds: 30 + serviceAccountName: "" # defaults to the main Buzz service account + podLabels: {} + podAnnotations: + sidecar.istio.io/inject: "false" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + # In-cluster MinIO for the quickstart profile only. Production deploys leave # this disabled and use s3.* (or secrets.existingSecret) against managed S3. minio: diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md new file mode 100644 index 00000000000..ec2c20f0004 --- /dev/null +++ b/docs/operator-community-deletion.md @@ -0,0 +1,87 @@ +# Community Deletion Operator Job + +Buzz executes whole-community deletion through the typed, one-shot +`/usr/local/bin/buzz-admin deletions drain` command. The Helm chart can schedule +that command as a Kubernetes CronJob; it does not call relay HTTP and it does +not add another queue or retry service. + +Postgres remains the handoff and source of truth. A run claims only requests +that the deletion store considers runnable, heartbeats the existing lease, and +resumes from durable checkpoints. `concurrencyPolicy: Forbid` prevents scheduled +pod overlap, `backoffLimit: 0` prevents Kubernetes Job retries, and the deletion +store remains authoritative when a pod exits, reaches its deadline, or is +replaced. + +## Enablement + +The CronJob is disabled by default. Production deployments should use an +existing Secret and a dedicated service account when their cluster policy +supports one: + +```yaml +secrets: + existingSecret: buzz-operator-secrets + +operatorJobs: + deletionDrain: + enabled: true + schedule: "*/5 * * * *" + activeDeadlineSeconds: 3600 + terminationGracePeriodSeconds: 30 + serviceAccountName: buzz-deletion-drain + podLabels: + tags.datadoghq.com/service: buzz-deletion-drain + podAnnotations: + sidecar.istio.io/inject: "false" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi +``` + +Set `s3.endpoint`, `s3.bucket`, `s3.region`, and `s3.addressingStyle` in chart +values. The selected Secret must contain `DATABASE_URL` and `REDIS_URL`; it may +contain `BUZZ_S3_ACCESS_KEY` and `BUZZ_S3_SECRET_KEY` when the object store uses +static credentials. The pod receives only those connection values and the four +non-secret S3 settings. It does not receive `BUZZ_RELAY_PRIVATE_KEY`, +`BUZZ_GIT_HOOK_HMAC_SECRET`, `RELAY_URL`, or the full Secret through `envFrom`. +The pod also disables service-account token automounting and Kubernetes service +link environment injection because the executor does not call the Kubernetes +API or discover cluster Services. + +The S3 principal needs the relay's normal object permissions plus bucket-level +`s3:ListBucketVersions` and object-level `s3:DeleteObjectVersion` for every +tenant-owned prefix. This also applies to never-versioned buckets because S3 +reports their objects with the `null` version id. + +## Runbook + +1. Confirm database migrations are current and the deletion request has crossed + the explicit inventory and approval boundary with + `buzz-admin deletions inspect `. +2. Confirm the selected Secret contains the required keys and the S3 principal + has version-list and exact-version delete permissions. +3. Enable the CronJob and inspect its rendered command and environment before + rollout. +4. Start one staffed manual run with + `kubectl create job --from=cronjob/-buzz-deletion-drain `. +5. Follow pod logs and re-run `buzz-admin deletions inspect ` to + verify lease, checkpoint, retry, blocked, and terminal state. +6. If a run fails or times out, fix the recorded dependency or permission + failure. Do not add Kubernetes retries: the next scheduled drain consults the + durable retry/checkpoint state and resumes only when the store allows it. + +The current owner self-serve relay admission records an owner-origin request at +`submitted` and intentionally performs no inventory or approval synchronously. +The deletion engine rejects `submitted` and `inventoried` requests at its +explicit approval boundary. Automating the privileged inventory/approval step +is therefore a separate control-plane slice; enabling this CronJob alone does +not make a newly accepted owner request destructive. + +The chart has no existing PrometheusRule or provider-neutral CronJob alert +integration. Operators must alert on failed/missed Jobs and long-running active +Jobs in their deployment platform. Adding a chart-native alert abstraction is +debt, not part of this job contract. From 07312054dbff1618485be2af333b1a94f57be5dc Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 23/46] Reject reserved deletion job pod labels Co-authored-by: Codex Signed-off-by: tornquist --- .../charts/buzz/templates/_operator-jobs.tpl | 5 +++++ .../charts/buzz/tests/deletion_drain_test.yaml | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/deploy/charts/buzz/templates/_operator-jobs.tpl b/deploy/charts/buzz/templates/_operator-jobs.tpl index aec91d52ff2..f273fe8ebd3 100644 --- a/deploy/charts/buzz/templates/_operator-jobs.tpl +++ b/deploy/charts/buzz/templates/_operator-jobs.tpl @@ -4,6 +4,11 @@ {{- $root := .root -}} {{- if eq .type "deletionDrain" -}} {{- $job := $root.Values.operatorJobs.deletionDrain -}} +{{- range $label := list "app.kubernetes.io/name" "app.kubernetes.io/instance" "app.kubernetes.io/component" -}} +{{- if hasKey $job.podLabels $label -}} +{{- fail (printf "operatorJobs.deletionDrain.podLabels may not set chart-owned label %q" $label) -}} +{{- end -}} +{{- end -}} apiVersion: batch/v1 kind: CronJob metadata: diff --git a/deploy/charts/buzz/tests/deletion_drain_test.yaml b/deploy/charts/buzz/tests/deletion_drain_test.yaml index 18e0d1a9ef2..5053c9911d3 100644 --- a/deploy/charts/buzz/tests/deletion_drain_test.yaml +++ b/deploy/charts/buzz/tests/deletion_drain_test.yaml @@ -200,6 +200,24 @@ tests: content: name: RELAY_URL + - it: rejects chart-owned pod identity labels + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain: + enabled: true + podLabels: + app.kubernetes.io/component: relay + asserts: + - failedTemplate: + errorPattern: 'operatorJobs.deletionDrain.podLabels may not set chart-owned label "app.kubernetes.io/component"' + - it: rejects enablement without a Redis source set: relayUrl: wss://buzz.example.com From 5c3af3b4c012da1166a2ef1e440d1c9e6892ca21 Mon Sep 17 00:00:00 2001 From: tornquist Date: Fri, 25 Sep 2026 18:03:39 +0000 Subject: [PATCH 24/46] Correct the deletion drain runbook's operational claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runbook told operators to target `cronjob/-buzz-deletion-drain`. `buzz.fullname` collapses to the release name when it already contains the chart name, so the documented name is wrong for the documented install: `helm install buzz ...` renders `buzz-deletion-drain`. Discover the CronJob by its component label instead, and describe the name rule rather than a single guessed spelling. Three more corrections: `activeDeadlineSeconds` was presented as if a timed-out run were just another retry. It is not recorded as one — shutdown releases the claim without recording a retry, and only the object-store drain resumes mid-stage — so a deadline landing repeatedly inside a non-resumable stage loops forever with a rising `attempts`, a flat `retry_count`, and no block. Document how to spot that from both the request and Kubernetes, how to size the deadline, and how to recover. `terminationGracePeriodSeconds` was presented as a clean handoff. Document that a pod still working at the end of the window is SIGKILLed holding its lease, and that recovery is lease expiry plus reclaim under a new generation. An empty `serviceAccountName` was described as a neutral default. It inherits the relay's service account; `automountServiceAccountToken: false` hides the projected token but does not detach cloud IAM bindings resolved through the node metadata path. Recommend a dedicated pre-created account when the executor's IAM blast radius should be smaller than the relay's. Co-Authored-By: Claude Opus 5 Signed-off-by: tornquist --- deploy/charts/buzz/values.yaml | 4 +- docs/operator-community-deletion.md | 67 ++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/deploy/charts/buzz/values.yaml b/deploy/charts/buzz/values.yaml index f43925346e5..d51d37b4bb7 100644 --- a/deploy/charts/buzz/values.yaml +++ b/deploy/charts/buzz/values.yaml @@ -402,7 +402,9 @@ operatorJobs: successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 3 terminationGracePeriodSeconds: 30 - serviceAccountName: "" # defaults to the main Buzz service account + # Empty inherits the relay service account, and with it any cloud IAM + # binding on that account. Name a dedicated account to narrow the executor. + serviceAccountName: "" podLabels: {} podAnnotations: sidecar.istio.io/inject: "false" diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index ec2c20f0004..d7fd69b3a2a 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -15,8 +15,7 @@ replaced. ## Enablement The CronJob is disabled by default. Production deployments should use an -existing Secret and a dedicated service account when their cluster policy -supports one: +existing Secret and a dedicated, pre-created service account: ```yaml secrets: @@ -52,6 +51,16 @@ The pod also disables service-account token automounting and Kubernetes service link environment injection because the executor does not call the Kubernetes API or discover cluster Services. +Leaving `operatorJobs.deletionDrain.serviceAccountName` empty falls back to the +relay's own service account. `automountServiceAccountToken: false` only +suppresses the projected token inside the pod; it does not detach the identity. +Cloud IAM bindings attached to that service account — IRSA on EKS, Workload +Identity on GKE — are resolved by the node/metadata path and still apply, so the +drain pod inherits the relay's cloud permissions. Create a dedicated service +account with only the object-store permissions listed below and name it +explicitly if you want the executor's IAM blast radius to be smaller than the +relay's. + The S3 principal needs the relay's normal object permissions plus bucket-level `s3:ListBucketVersions` and object-level `s3:DeleteObjectVersion` for every tenant-owned prefix. This also applies to never-versioned buckets because S3 @@ -66,14 +75,60 @@ reports their objects with the `null` version id. has version-list and exact-version delete permissions. 3. Enable the CronJob and inspect its rendered command and environment before rollout. -4. Start one staffed manual run with - `kubectl create job --from=cronjob/-buzz-deletion-drain `. -5. Follow pod logs and re-run `buzz-admin deletions inspect ` to +4. Locate the rendered CronJob by label rather than by guessing its name: + + ```sh + kubectl get cronjob -n \ + -l app.kubernetes.io/component=deletion-drain,app.kubernetes.io/instance= + ``` + + The name is `-deletion-drain`. `buzz.fullname` collapses to the + release name when the release name already contains the chart name, so + `helm install buzz ...` renders `buzz-deletion-drain`, not + `buzz-buzz-deletion-drain`. +5. Start one staffed manual run with + `kubectl create job --from=cronjob/ `. +6. Follow pod logs and re-run `buzz-admin deletions inspect ` to verify lease, checkpoint, retry, blocked, and terminal state. -6. If a run fails or times out, fix the recorded dependency or permission +7. If a run fails or times out, fix the recorded dependency or permission failure. Do not add Kubernetes retries: the next scheduled drain consults the durable retry/checkpoint state and resumes only when the store allows it. +## Deadlines, termination, and the retry budget + +`activeDeadlineSeconds` is a Kubernetes-side limit, and the deletion store does +not learn why a pod went away. Two consequences matter when reading state: + +- A `DeadlineExceeded` Job is not recorded as a deletion retry. Shutdown + releases the claim without recording one, so `retry_count` and `blocked_at` + do not advance. Only the object-store drain checkpoints per manifest chunk + and resumes mid-stage; every other stage restarts from its beginning on the + next run. A deadline that keeps landing inside one of those non-resumable + stages therefore repeats indefinitely: each run increments `attempts` and + burns the window again while the retry budget never moves and the request is + never blocked. +- Diagnose this from both sides. `buzz-admin deletions inspect ` + shows a rising `attempts` with a flat `retry_count` and no `last_error`; + Kubernetes holds the reason. The chart labels the CronJob and the drain pods, + but not the generated Jobs, so find the attempts with + `kubectl get pods -n -l app.kubernetes.io/component=deletion-drain` + and read the condition with `kubectl describe job `. + +Size `activeDeadlineSeconds` for the longest single stage this community will +run, not for the average run. To recover, either raise the deadline and let the +schedule pick the request back up, or take one staffed run with +`buzz-admin deletions run ` outside the CronJob's deadline. + +`terminationGracePeriodSeconds` is a best-effort window, not a guarantee. The +drain command handles `SIGTERM` and releases its lease cleanly when it wins the +race, but a pod that is still working when the grace period expires is +`SIGKILL`ed with the lease still held. Nothing is lost: the durable lease simply +expires (60s by default, heartbeated every 10s) and the next run reclaims the +request with a fresh lease generation, which fences any straggler write from the +killed process. Expect up to roughly a lease duration of delay before the +request is runnable again; do not raise the grace period expecting a clean +handoff. + The current owner self-serve relay admission records an owner-origin request at `submitted` and intentionally performs no inventory or approval synchronously. The deletion engine rejects `submitted` and `inventoried` requests at its From d791f86a3d342dd86a6656703bacfd817a9726ed Mon Sep 17 00:00:00 2001 From: tornquist Date: Mon, 28 Sep 2026 13:24:52 +0000 Subject: [PATCH 25/46] Cover deletion drain chart contracts Exercise both supported credential shapes through the CI render matrix and bind the typed job guards, schema boundary, pod identity, and default sidecar annotation. Co-authored-by: Codex Signed-off-by: tornquist --- .../buzz/tests/deletion_drain_test.yaml | 81 ++++++++++++++++++- .../deletion-drain-bundled-values.yaml | 11 +++ .../production-existing-secret-values.yaml | 3 + 3 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 deploy/charts/buzz/tests/fixtures/deletion-drain-bundled-values.yaml diff --git a/deploy/charts/buzz/tests/deletion_drain_test.yaml b/deploy/charts/buzz/tests/deletion_drain_test.yaml index 5053c9911d3..1f8a6315ced 100644 --- a/deploy/charts/buzz/tests/deletion_drain_test.yaml +++ b/deploy/charts/buzz/tests/deletion_drain_test.yaml @@ -38,7 +38,6 @@ tests: podLabels: tags.datadoghq.com/service: buzz-deletion-drain podAnnotations: - sidecar.istio.io/inject: "false" example.com/operator-job: deletion-drain resources: requests: @@ -96,6 +95,18 @@ tests: - equal: path: spec.jobTemplate.spec.template.metadata.labels["tags.datadoghq.com/service"] value: buzz-deletion-drain + - equal: + path: spec.jobTemplate.spec.template.metadata.labels["app.kubernetes.io/name"] + value: buzz + - equal: + path: spec.jobTemplate.spec.template.metadata.labels["app.kubernetes.io/instance"] + value: RELEASE-NAME + - equal: + path: spec.jobTemplate.spec.template.metadata.labels["app.kubernetes.io/component"] + value: deletion-drain + - equal: + path: spec.jobTemplate.spec.template.metadata.annotations["sidecar.istio.io/inject"] + value: "false" - equal: path: spec.jobTemplate.spec.template.metadata.annotations["example.com/operator-job"] value: deletion-drain @@ -200,7 +211,43 @@ tests: content: name: RELAY_URL - - it: rejects chart-owned pod identity labels + - it: rejects the chart-owned name pod label with an explicit guard error + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain: + enabled: true + podLabels: + app.kubernetes.io/name: relay + asserts: + - failedTemplate: + errorPattern: 'operatorJobs.deletionDrain.podLabels may not set chart-owned label "app.kubernetes.io/name"' + + - it: rejects the chart-owned instance pod label with an explicit guard error + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain: + enabled: true + podLabels: + app.kubernetes.io/instance: another-release + asserts: + - failedTemplate: + errorPattern: 'operatorJobs.deletionDrain.podLabels may not set chart-owned label "app.kubernetes.io/instance"' + + - it: rejects the chart-owned component pod label with an explicit guard error set: relayUrl: wss://buzz.example.com ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" @@ -218,6 +265,36 @@ tests: - failedTemplate: errorPattern: 'operatorJobs.deletionDrain.podLabels may not set chart-owned label "app.kubernetes.io/component"' + - it: rejects existing-secret enablement without an explicit S3 endpoint + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + secrets.existingSecret: buzz-operator-secrets + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.bucket: buzz-media-example + operatorJobs.deletionDrain.enabled: true + asserts: + - failedTemplate: + errorPattern: "s3.endpoint is required when operatorJobs.deletionDrain.enabled=true" + + - it: rejects unknown deletion drain keys through the values schema + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain: + enabled: true + arbitraryCommand: ["unsafe"] + asserts: + - failedTemplate: + errorPattern: "Additional property arbitraryCommand is not allowed" + - it: rejects enablement without a Redis source set: relayUrl: wss://buzz.example.com diff --git a/deploy/charts/buzz/tests/fixtures/deletion-drain-bundled-values.yaml b/deploy/charts/buzz/tests/fixtures/deletion-drain-bundled-values.yaml new file mode 100644 index 00000000000..18d30a696da --- /dev/null +++ b/deploy/charts/buzz/tests/fixtures/deletion-drain-bundled-values.yaml @@ -0,0 +1,11 @@ +relayUrl: wss://buzz.example.com +ownerPubkey: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" +postgresql: + enabled: true +redis: + enabled: true +minio: + enabled: true +operatorJobs: + deletionDrain: + enabled: true diff --git a/deploy/charts/buzz/tests/fixtures/production-existing-secret-values.yaml b/deploy/charts/buzz/tests/fixtures/production-existing-secret-values.yaml index fd4bdadfeda..7020fb32b0b 100644 --- a/deploy/charts/buzz/tests/fixtures/production-existing-secret-values.yaml +++ b/deploy/charts/buzz/tests/fixtures/production-existing-secret-values.yaml @@ -8,6 +8,9 @@ externalPostgresql: url: "postgres://buzz:pw@postgres.example.com:5432/buzz" externalRedis: url: "redis://:pw@redis.example.com:6379" +operatorJobs: + deletionDrain: + enabled: true s3: endpoint: "https://s3.us-east-1.amazonaws.com" bucket: "buzz-media" From c0a392a48788d81ac3adfe663f5647aa426f1607 Mon Sep 17 00:00:00 2001 From: tornquist Date: Mon, 28 Sep 2026 13:25:36 +0000 Subject: [PATCH 26/46] Bound operator CronJob names Share a suffix-preserving 52-character name helper between the deletion drain and storage accounting CronJobs while retaining their existing short names. Co-authored-by: Codex Signed-off-by: tornquist --- deploy/charts/buzz/templates/_helpers.tpl | 7 ++++++ .../charts/buzz/templates/_operator-jobs.tpl | 2 +- .../templates/storage-accounting-cronjob.yaml | 2 +- .../buzz/tests/deletion_drain_test.yaml | 19 ++++++++++++++++ .../buzz/tests/storage_accounting_test.yaml | 22 +++++++++++++++++++ docs/operator-community-deletion.md | 6 +++-- 6 files changed, 54 insertions(+), 4 deletions(-) diff --git a/deploy/charts/buzz/templates/_helpers.tpl b/deploy/charts/buzz/templates/_helpers.tpl index e5ec9182d68..c00e643fdb3 100644 --- a/deploy/charts/buzz/templates/_helpers.tpl +++ b/deploy/charts/buzz/templates/_helpers.tpl @@ -17,6 +17,13 @@ {{- end -}} {{- end -}} +{{/* Kubernetes CronJob names are limited to 52 characters. */}} +{{- define "buzz.cronJobName" -}} +{{- $maxBaseLength := sub 51 (len .suffix) | int -}} +{{- $base := include "buzz.fullname" .root | trunc $maxBaseLength | trimSuffix "-" -}} +{{- printf "%s-%s" $base .suffix -}} +{{- end -}} + {{- define "buzz.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} diff --git a/deploy/charts/buzz/templates/_operator-jobs.tpl b/deploy/charts/buzz/templates/_operator-jobs.tpl index f273fe8ebd3..49a204f9ec9 100644 --- a/deploy/charts/buzz/templates/_operator-jobs.tpl +++ b/deploy/charts/buzz/templates/_operator-jobs.tpl @@ -12,7 +12,7 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "buzz.fullname" $root }}-deletion-drain + name: {{ include "buzz.cronJobName" (dict "root" $root "suffix" "deletion-drain") }} labels: {{- include "buzz.labels" $root | nindent 4 }} app.kubernetes.io/component: deletion-drain diff --git a/deploy/charts/buzz/templates/storage-accounting-cronjob.yaml b/deploy/charts/buzz/templates/storage-accounting-cronjob.yaml index deaa816888e..f753bd3b412 100644 --- a/deploy/charts/buzz/templates/storage-accounting-cronjob.yaml +++ b/deploy/charts/buzz/templates/storage-accounting-cronjob.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: {{ include "buzz.fullname" . }}-storage-accounting + name: {{ include "buzz.cronJobName" (dict "root" . "suffix" "storage-accounting") }} labels: {{- include "buzz.labels" . | nindent 4 }} app.kubernetes.io/component: storage-accounting diff --git a/deploy/charts/buzz/tests/deletion_drain_test.yaml b/deploy/charts/buzz/tests/deletion_drain_test.yaml index 1f8a6315ced..cf2b7e8556a 100644 --- a/deploy/charts/buzz/tests/deletion_drain_test.yaml +++ b/deploy/charts/buzz/tests/deletion_drain_test.yaml @@ -307,3 +307,22 @@ tests: asserts: - failedTemplate: errorPattern: "operatorJobs.deletionDrain requires Redis" + + - it: preserves the deletion suffix while bounding a long release and name override + release: + name: very-long-release-name-for-operator-jobs + set: + nameOverride: custom-buzz-name + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + operatorJobs.deletionDrain.enabled: true + asserts: + - equal: + path: metadata.name + value: very-long-release-name-for-operator-j-deletion-drain diff --git a/deploy/charts/buzz/tests/storage_accounting_test.yaml b/deploy/charts/buzz/tests/storage_accounting_test.yaml index d37d8aefe98..563e1ec2283 100644 --- a/deploy/charts/buzz/tests/storage_accounting_test.yaml +++ b/deploy/charts/buzz/tests/storage_accounting_test.yaml @@ -53,6 +53,10 @@ tests: path: kind value: CronJob template: templates/storage-accounting-cronjob.yaml + - equal: + path: metadata.name + value: RELEASE-NAME-buzz-storage-accounting + template: templates/storage-accounting-cronjob.yaml - equal: path: spec.concurrencyPolicy value: Forbid @@ -143,3 +147,21 @@ tests: content: name: BUZZ_GIT_HOOK_HMAC_SECRET template: templates/storage-accounting-cronjob.yaml + + - it: preserves the storage suffix without a trailing separator after truncation + set: + fullnameOverride: storage-accounting-fullname-that-needs-truncation-at-the-separator + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: https://s3.example.com + s3.bucket: buzz-media-example + s3.accessKey: test + s3.secretKey: test + storageAccounting.enabled: true + asserts: + - equal: + path: metadata.name + value: storage-accounting-fullname-that-storage-accounting + template: templates/storage-accounting-cronjob.yaml diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index d7fd69b3a2a..546f73cf3d2 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -82,8 +82,10 @@ reports their objects with the `null` version id. -l app.kubernetes.io/component=deletion-drain,app.kubernetes.io/instance= ``` - The name is `-deletion-drain`. `buzz.fullname` collapses to the - release name when the release name already contains the chart name, so + The name is `-deletion-drain`. The chart truncates only the + fullname portion when needed so the suffix remains stable within Kubernetes' + 52-character CronJob name limit. `buzz.fullname` collapses to the release + name when the release name already contains the chart name, so `helm install buzz ...` renders `buzz-deletion-drain`, not `buzz-buzz-deletion-drain`. 5. Start one staffed manual run with From cdef0269269d36e51209cebd3d7d9e2c20ee4533 Mon Sep 17 00:00:00 2001 From: tornquist Date: Mon, 28 Sep 2026 13:25:59 +0000 Subject: [PATCH 27/46] Clarify deletion job workload identity Distinguish the disabled Kubernetes API token mount from provider workload-identity credentials while retaining the dedicated service-account recommendation. Co-authored-by: Codex Signed-off-by: tornquist --- deploy/charts/buzz/README.md | 6 ++++-- docs/operator-community-deletion.md | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index 1bdff7592e9..b290cff902d 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -126,8 +126,10 @@ checkpoints remain the execution authority. The pod receives only `DATABASE_URL`, `REDIS_URL`, and required S3 configuration/credential variables. It does not receive the relay private key, -git-hook secret, relay URL, service-account token, service links, or a generic -environment registry. Schedule, +git-hook secret, relay URL, service links, or a generic environment registry. +The chart disables the ordinary Kubernetes API service-account token mount; +platform workload-identity admission may still inject its own projected token +and provider environment variables. Schedule, deadline, history, termination grace, resources, service account, pod labels, and pod annotations are independently configurable under `operatorJobs.deletionDrain`. diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index 546f73cf3d2..61d8fd89456 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -49,17 +49,19 @@ non-secret S3 settings. It does not receive `BUZZ_RELAY_PRIVATE_KEY`, `BUZZ_GIT_HOOK_HMAC_SECRET`, `RELAY_URL`, or the full Secret through `envFrom`. The pod also disables service-account token automounting and Kubernetes service link environment injection because the executor does not call the Kubernetes -API or discover cluster Services. +API or discover cluster Services. This disables the ordinary Kubernetes API +token mount, not credentials injected by a platform workload-identity +mechanism. Leaving `operatorJobs.deletionDrain.serviceAccountName` empty falls back to the -relay's own service account. `automountServiceAccountToken: false` only -suppresses the projected token inside the pod; it does not detach the identity. -Cloud IAM bindings attached to that service account — IRSA on EKS, Workload -Identity on GKE — are resolved by the node/metadata path and still apply, so the -drain pod inherits the relay's cloud permissions. Create a dedicated service -account with only the object-store permissions listed below and name it -explicitly if you want the executor's IAM blast radius to be smaller than the -relay's. +relay's own service account, including cloud IAM attached through the platform's +workload-identity mechanism. For example, the EKS IRSA admission webhook can +inject its projected web-identity token and AWS environment variables despite +`automountServiceAccountToken: false`; other platforms provide their own +identity mechanism. The drain pod therefore inherits the relay's cloud role by +default. Create a dedicated service account with only the object-store +permissions listed below and name it explicitly if you want the executor's IAM +blast radius to be smaller than the relay's. The S3 principal needs the relay's normal object permissions plus bucket-level `s3:ListBucketVersions` and object-level `s3:DeleteObjectVersion` for every From 38db86befc9baac48010bc9ab48b7d184a739703 Mon Sep 17 00:00:00 2001 From: tornquist Date: Mon, 28 Sep 2026 14:46:10 +0000 Subject: [PATCH 28/46] Scope video menu probes to emitted messages Bind each context-menu interaction to the exact mock message returned by the emitter so same-second ordering cannot select the wrong video. Co-authored-by: Codex Signed-off-by: tornquist --- desktop/tests/e2e/video-attachment.spec.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/desktop/tests/e2e/video-attachment.spec.ts b/desktop/tests/e2e/video-attachment.spec.ts index 2c86f5a9bda..060d356d28e 100644 --- a/desktop/tests/e2e/video-attachment.spec.ts +++ b/desktop/tests/e2e/video-attachment.spec.ts @@ -1513,12 +1513,14 @@ test("right-click menus expose distinct selectors for links, relay video, and of // ── Relay video menu: Download video + Copy link, appearing only once the // relay origin resolves (the reactivity fix) ───────────────────────────── - await emitVideoMessage(page, { + const relayMessage = (await emitVideoMessage(page, { url: MENU_RELAY_VIDEO_URL, sha: MENU_RELAY_VIDEO_SHA, filename: "relay-clip.mp4", - }); - const relayPlayer = page.getByTestId("video-player").last(); + })) as { id: string }; + const relayPlayer = page + .locator(`[data-message-id="${relayMessage.id}"]`) + .getByTestId("video-player"); await expect(relayPlayer).toBeVisible(); // Right-click the player surface. `force` skips the actionability guard: the // Play-button overlay sits above the video, but the contextmenu event still @@ -1559,12 +1561,14 @@ test("right-click menus expose distinct selectors for links, relay video, and of await expect(page.locator("[data-video-context-menu]")).toHaveCount(0); // ── Off-relay video control: renders and offers Copy link, never Download ─ - await emitVideoMessage(page, { + const offRelayMessage = (await emitVideoMessage(page, { url: MENU_OFF_RELAY_VIDEO_URL, sha: MENU_OFF_RELAY_VIDEO_SHA, filename: "external-clip.mp4", - }); - const offRelayPlayer = page.getByTestId("video-player").last(); + })) as { id: string }; + const offRelayPlayer = page + .locator(`[data-message-id="${offRelayMessage.id}"]`) + .getByTestId("video-player"); await expect(offRelayPlayer).toBeVisible(); await offRelayPlayer.click({ button: "right", force: true }); From 131ef21a253367db5e092fd6a392398e4b6eae93 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 20:06:50 +0000 Subject: [PATCH 29/46] test(desktop): await channel head before scroll summary check Signed-off-by: Codex Co-authored-by: Codex --- desktop/tests/e2e/scroll-history.spec.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/desktop/tests/e2e/scroll-history.spec.ts b/desktop/tests/e2e/scroll-history.spec.ts index 27bf08b731f..5282c4c977e 100644 --- a/desktop/tests/e2e/scroll-history.spec.ts +++ b/desktop/tests/e2e/scroll-history.spec.ts @@ -2172,6 +2172,11 @@ test("thread summary badge survives a retained older-history prepend", async ({ await page.getByTestId("channel-deep-history").click(); await expect(page.getByTestId("chat-title")).toHaveText("deep-history"); + await waitForMockChannelHeadReady( + page, + "deep-history", + "feedf00d-0000-4000-8000-000000000007", + ); const timeline = page.getByTestId("message-timeline"); const badgeSelector = '[data-testid="message-thread-summary"][data-thread-head-id="mock-deep-history-599"]'; From 5650ae7a5dcdbbc88dedd0e4b60a3270bd9ec959 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 20:50:54 +0000 Subject: [PATCH 30/46] test(desktop): record transient snapshot before fallback Signed-off-by: Codex --- desktop/tests/e2e/sidebar-snapshot.spec.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/desktop/tests/e2e/sidebar-snapshot.spec.ts b/desktop/tests/e2e/sidebar-snapshot.spec.ts index 6d1b349e884..bff5af61209 100644 --- a/desktop/tests/e2e/sidebar-snapshot.spec.ts +++ b/desktop/tests/e2e/sidebar-snapshot.spec.ts @@ -487,6 +487,7 @@ test("mismatched not-modified hash falls back to a full list", async ({ page, }) => { await seedSnapshot(page, { hash: "persisted-stale-hash" }); + await trackSnapshotRows(page); await installMockBridge(page, { channelsReadDelayMs: READ_DELAY_MS, channelsNotModifiedResponses: 1, @@ -494,9 +495,11 @@ test("mismatched not-modified hash falls back to a full list", async ({ await page.goto("/"); const snapshotRows = page.locator('[data-channel-id^="snapshot-"]'); - await expect(snapshotRows).toHaveCount(FULL_SNAPSHOT.length, { - timeout: 500, - }); + // Record the transient boot frame independently of test-runner scheduling; + // cold-boot coverage separately enforces the snapshot paint deadline. + await expect + .poll(() => getTrackedSnapshotRows(page)) + .toEqual(FULL_SNAPSHOT.map((channel) => channel.id)); await expect .poll(() => getChannelsPayloads(page)) .toEqual([{ knownHash: "persisted-stale-hash" }, { knownHash: null }]); From e024135d6859fd32fc10fbf4645973b51c6689c9 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 21:58:29 +0000 Subject: [PATCH 31/46] feat: add owner deletion receipt and quota reservation Signed-off-by: Codex Co-authored-by: Codex --- crates/buzz-db/src/runtime/migration.rs | 33 ++- crates/buzz-db/src/store/community.rs | 222 ++++++++++++++-- crates/buzz-db/src/store/deletion.rs | 10 + crates/buzz-db/src/store/relay_members.rs | 39 ++- crates/buzz-relay/src/api/operator.rs | 250 +++++++++++++++++- crates/buzz-relay/src/router.rs | 4 + .../0053_owner_deletion_quota_reservation.sql | 10 + schema/schema.sql | 6 + 8 files changed, 541 insertions(+), 33 deletions(-) create mode 100644 migrations/0053_owner_deletion_quota_reservation.sql diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 1300b01bb3d..9063bf1b6b7 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -705,11 +705,12 @@ mod postgres_tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 52); + assert_eq!(migrations.len(), 53); assert_eq!(migrations[48].version, 49); assert_eq!(migrations[49].version, 50); assert_eq!(migrations[50].version, 51); assert_eq!(migrations[51].version, 52); + assert_eq!(migrations[52].version, 53); assert!(migrations[48] .sql .as_str() @@ -718,6 +719,10 @@ mod postgres_tests { .sql .as_str() .contains("community_deletion_owner_provenance")); + assert!(migrations[52] + .sql + .as_str() + .contains("community_deletion_requests_owner_quota_reservations")); assert!(migrations[51].sql.as_str().contains("approval_origin")); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); @@ -1748,6 +1753,32 @@ mod postgres_tests { assert!(migration.contains("set local lock_timeout = '5s'")); } + #[test] + fn owner_deletion_quota_reservation_index_matches_desired_schema() { + let migration = MIGRATOR + .iter() + .find(|migration| migration.version == 53) + .expect("embedded migration 0053") + .sql + .as_ref() + .to_ascii_lowercase(); + let workspace_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(std::path::Path::parent) + .expect("workspace root"); + let schema = std::fs::read_to_string(workspace_root.join("schema/schema.sql")) + .expect("read schema/schema.sql") + .to_ascii_lowercase(); + + for sql in [&migration, &schema] { + assert!(sql.contains("community_deletion_requests_owner_quota_reservations")); + assert!(sql.contains("request_origin = 'owner'")); + assert!(sql.contains("stage <> 'aborted'")); + assert!(sql.contains("completed_at is null")); + } + assert!(migration.contains("set local lock_timeout = '5s'")); + } + /// Structural parity between migration 0029's deletion surface and the /// desired-state bootstrap schema (`schema/schema.sql`). /// diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index 461348a5cd3..2b822f848b8 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -61,6 +61,15 @@ pub struct OwnedCommunityRecord { pub archived_at: Option>, } +/// Owner-list rows plus the authoritative quota projection from one snapshot. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OwnedCommunitiesPage { + pub communities: Vec, + pub quota_used: i64, + pub quota_limit: i64, + pub can_create: bool, +} + /// Community row returned by an owner-authorized archive operation. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ArchivedCommunityRecord { @@ -206,13 +215,17 @@ impl Db { pub async fn list_communities_owned_by( &self, owner_pubkey: &str, - ) -> Result> { + ) -> Result { let owner_pubkey = owner_pubkey.to_ascii_lowercase(); - let mut connection = crate::observability::acquire_writer( + let connection = crate::observability::acquire_writer( &self.pool, crate::observability::WriterOperation::Authorization, ) .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + .execute(&mut *tx) + .await?; let rows = sqlx::query( r#" SELECT c.id, c.host, c.created_at, c.archived_at @@ -227,11 +240,11 @@ impl Db { ORDER BY c.created_at ASC, c.host ASC "#, ) - .bind(owner_pubkey) - .fetch_all(&mut *connection) + .bind(&owner_pubkey) + .fetch_all(&mut *tx) .await?; - - rows.into_iter() + let communities = rows + .into_iter() .map(|row| { let id: Uuid = row.try_get("id")?; let host: String = row.try_get("host")?; @@ -244,7 +257,17 @@ impl Db { archived_at, }) }) - .collect() + .collect::>>()?; + let quota_used = + relay_members::owner_quota_used_in_transaction(&mut tx, &owner_pubkey).await?; + tx.commit().await?; + let quota_limit = relay_members::max_communities_per_owner(); + Ok(OwnedCommunitiesPage { + communities, + quota_used, + quota_limit, + can_create: quota_used < quota_limit, + }) } /// Returns the normalized host mapped to a community id, if the community @@ -450,12 +473,8 @@ impl Db { let host: String = row.try_get("host")?; // Enforce the limit before inserting the new owner row. - let owned_count: i64 = sqlx::query_scalar( - "SELECT count(*) FROM relay_members WHERE pubkey = $1 AND role = 'owner'", - ) - .bind(&owner_pubkey) - .fetch_one(&mut *tx) - .await?; + let owned_count = + relay_members::owner_quota_used_in_transaction(&mut tx, &owner_pubkey).await?; if owned_count >= relay_members::max_communities_per_owner() { tx.rollback().await?; @@ -793,6 +812,7 @@ mod postgres_tests { "EnsuredCommunityRecord", "CreatedCommunityRecord", "OwnedCommunityRecord", + "OwnedCommunitiesPage", "ArchivedCommunityRecord", "UnarchivedCommunityRecord", ]; @@ -1093,8 +1113,180 @@ mod postgres_tests { .await .expect("list owned communities"); - assert_eq!(owned.len(), 1); - assert_eq!(owned[0].id, community_a); + assert_eq!(owned.communities.len(), 1); + assert_eq!(owned.communities[0].id, community_a); + assert_eq!(owned.quota_used, 1); + assert_eq!( + owned.quota_limit, + crate::relay_members::max_communities_per_owner() + ); + assert!(owned.can_create); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_quota_reservation_survives_purge_stages_and_releases_terminally() { + let db = setup_db().await; + let owner = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let host = format!("quota-reservation-{}.example", Uuid::new_v4().simple()); + let created = db + .create_community_with_owner(&host, &owner) + .await + .expect("create owned community"); + let CreateCommunityWithOwnerResult::Created(created) = created else { + panic!("expected created community") + }; + let request_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO community_deletion_requests \ + (id, community_id, community_host, requested_by, request_origin, owner_pubkey, \ + mediating_operator_pubkey, acknowledgement_version) \ + VALUES ($1, $2, $3, $4, 'owner', $4, $5, 1)", + ) + .bind(request_id) + .bind(created.id.as_uuid()) + .bind(&host) + .bind(&owner) + .bind("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + .execute(&db.pool) + .await + .expect("insert owner deletion request"); + + let pre_purge = db + .list_communities_owned_by(&owner) + .await + .expect("pre-purge quota"); + assert!( + pre_purge.communities.is_empty(), + "pending row is suppressed" + ); + assert_eq!( + pre_purge.quota_used, 1, + "membership and request deduplicate" + ); + + sqlx::query("DELETE FROM relay_members WHERE community_id = $1") + .bind(created.id.as_uuid()) + .execute(&db.pool) + .await + .expect("simulate membership purge"); + for stage in ["postgres_purged", "cache_purged", "logically_verified"] { + sqlx::query( + "UPDATE community_deletion_requests SET stage = $2, completed_at = NULL WHERE id = $1", + ) + .bind(request_id) + .bind(stage) + .execute(&db.pool) + .await + .expect("advance quota fixture"); + assert_eq!( + db.list_communities_owned_by(&owner) + .await + .expect("reserved quota") + .quota_used, + 1, + "{stage} must retain the slot" + ); + } + + sqlx::query( + "UPDATE community_deletion_requests \ + SET stage = 'retention_pending', completed_at = now() WHERE id = $1", + ) + .bind(request_id) + .execute(&db.pool) + .await + .expect("complete quota fixture"); + assert_eq!( + db.list_communities_owned_by(&owner) + .await + .expect("released quota") + .quota_used, + 0 + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_quota_serializes_concurrent_create_and_transfer_with_reservation() { + let db = setup_db().await; + let recipient = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + let source_owner = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); + + for index in 0..3 { + let host = format!("quota-live-{index}-{}.example", Uuid::new_v4().simple()); + assert!(matches!( + db.create_community_with_owner(&host, &recipient) + .await + .expect("create live quota fixture"), + CreateCommunityWithOwnerResult::Created(_) + )); + } + let reserved_host = format!("quota-held-{}.example", Uuid::new_v4().simple()); + let reserved = db + .create_community_with_owner(&reserved_host, &recipient) + .await + .expect("create reservation fixture"); + let CreateCommunityWithOwnerResult::Created(reserved) = reserved else { + panic!("expected reservation community") + }; + sqlx::query( + "INSERT INTO community_deletion_requests \ + (id, community_id, community_host, requested_by, request_origin, owner_pubkey, \ + mediating_operator_pubkey, acknowledgement_version) \ + VALUES ($1, $2, $3, $4, 'owner', $4, $5, 1)", + ) + .bind(Uuid::new_v4()) + .bind(reserved.id.as_uuid()) + .bind(&reserved_host) + .bind(&recipient) + .bind("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") + .execute(&db.pool) + .await + .expect("insert reservation"); + sqlx::query("DELETE FROM relay_members WHERE community_id = $1") + .bind(reserved.id.as_uuid()) + .execute(&db.pool) + .await + .expect("simulate purged membership"); + + let transfer_host = format!("quota-transfer-{}.example", Uuid::new_v4().simple()); + let transfer_target = db + .create_community_with_owner(&transfer_host, &source_owner) + .await + .expect("create transfer target"); + let CreateCommunityWithOwnerResult::Created(transfer_target) = transfer_target else { + panic!("expected transfer target") + }; + let create_host = format!("quota-race-{}.example", Uuid::new_v4().simple()); + + let (create, transfer) = tokio::join!( + db.create_community_with_owner(&create_host, &recipient), + db.transfer_ownership(transfer_target.id, &recipient, &source_owner), + ); + let create = create.expect("concurrent create result"); + let transfer = transfer.expect("concurrent transfer result"); + let create_won = matches!(create, CreateCommunityWithOwnerResult::Created(_)); + let transfer_won = matches!( + transfer, + crate::relay_members::TransferResult::Transferred { .. } + ); + assert_ne!(create_won, transfer_won, "exactly one owner grant may win"); + assert!( + matches!(create, CreateCommunityWithOwnerResult::LimitReached) || create_won, + "create loser must observe the quota" + ); + assert!( + matches!(transfer, crate::relay_members::TransferResult::LimitReached) || transfer_won, + "transfer loser must observe the quota" + ); + assert_eq!( + db.list_communities_owned_by(&recipient) + .await + .expect("post-race quota") + .quota_used, + crate::relay_members::max_communities_per_owner() + ); } #[tokio::test] diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 9346bea4570..259307cd28f 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -5013,6 +5013,7 @@ mod postgres_tests { db.list_communities_owned_by(&owner) .await .expect("owner list") + .communities .iter() .all(|row| row.id != community), "accepted deletion requests must not remain actionable archived rows" @@ -5069,6 +5070,14 @@ mod postgres_tests { .expect("privileged abort at the reversible submitted boundary"); assert_eq!(aborted.stage, DeletionStage::Aborted); assert_eq!(aborted.aborted_by.as_deref(), Some("recovery-operator")); + assert_eq!( + db.list_communities_owned_by(&owner) + .await + .expect("quota after abort") + .quota_used, + 1, + "abort releases the request reservation but preserved membership still counts" + ); // Abort reverses deletion intent, not the owner's archive decision. let (deletion_state, archived_at): (String, Option>) = @@ -5087,6 +5096,7 @@ mod postgres_tests { db.list_communities_owned_by(&owner) .await .expect("owner list after abort") + .communities .iter() .any(|row| row.id == community), "aborting the request must restore the owner's actionable archived row" diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 20309022cbb..8b1e9a3e07a 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -604,6 +604,38 @@ pub fn owner_count_advisory_lock_key(pubkey_hex: &str) -> i64 { h as i64 } +/// Count live ownership plus incomplete owner-deletion reservations. +/// +/// `UNION` deliberately de-duplicates the live membership and deletion row +/// before PostgreSQL purges membership. The reservation remains until the +/// logical-completion transition records `completed_at`. +pub(crate) async fn owner_quota_used_in_transaction( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + owner_pubkey: &str, +) -> Result { + sqlx::query_scalar( + r#" + SELECT count(*)::BIGINT + FROM ( + SELECT community_id + FROM relay_members + WHERE pubkey = $1 AND role = 'owner' + UNION + SELECT community_id + FROM community_deletion_requests + WHERE request_origin = 'owner' + AND owner_pubkey = $1 + AND stage <> 'aborted' + AND completed_at IS NULL + ) quota_reservations + "#, + ) + .bind(owner_pubkey) + .fetch_one(&mut **tx) + .await + .map_err(Into::into) +} + /// Atomically transfers ownership of `community` to `new_owner_pubkey`. /// /// Runs in a single transaction: @@ -698,12 +730,7 @@ pub async fn transfer_ownership( // 4. Enforce the transferee's community ownership limit inside the same // transaction that holds the advisory lock. This is the authoritative // check — kgoose's preflight count is advisory only. - let owned_count: i64 = sqlx::query_scalar( - "SELECT count(*) FROM relay_members WHERE pubkey = $1 AND role = 'owner'", - ) - .bind(&pubkey) - .fetch_one(&mut *tx) - .await?; + let owned_count = owner_quota_used_in_transaction(&mut tx, &pubkey).await?; if owned_count >= max_communities_per_owner() { tx.rollback().await?; diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index 64095decc1a..e841bc2b81e 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -24,6 +24,17 @@ use crate::state::AppState; use super::{api_error, bridge, internal_error}; +fn deletion_api_error( + status: StatusCode, + code: &'static str, + message: &str, +) -> (StatusCode, Json) { + ( + status, + Json(serde_json::json!({ "error": message, "code": code })), + ) +} + /// Query parameters for `GET /operator/communities`. #[derive(Debug, Deserialize)] pub struct ListCommunitiesQuery { @@ -327,6 +338,8 @@ pub struct DeleteCommunityRequest { acknowledgement_version: i32, } +const DELETE_RECEIPT_PATH: &str = "/operator/communities/delete/receipt"; + /// Idempotently archive a community owned by the asserted end-user identity. pub async fn archive_community( State(state): State>, @@ -470,23 +483,26 @@ pub async fn delete_community( let operator = authorize_operator_request(&state, &headers, "POST", PATH, None, Some(&body)).await?; let request: DeleteCommunityRequest = serde_json::from_slice(&body).map_err(|e| { - api_error( + deletion_api_error( StatusCode::BAD_REQUEST, + "invalid_request", &format!("invalid delete-community JSON: {e}"), ) })?; let normalized_host = normalize_candidate_host(&request.host) - .map_err(|msg| api_error(StatusCode::BAD_REQUEST, &msg))?; + .map_err(|msg| deletion_api_error(StatusCode::BAD_REQUEST, "invalid_request", &msg))?; let deployment_host = buzz_core::tenant::relay_url_authority(&state.config.relay_url); if normalized_host == deployment_host { - return Err(api_error( + return Err(deletion_api_error( StatusCode::CONFLICT, + "protected_community", "the deployment community cannot be deleted", )); } let owner = validate_pubkey_hex(&request.owner_pubkey).ok_or_else(|| { - api_error( + deletion_api_error( StatusCode::BAD_REQUEST, + "invalid_request", "invalid owner_pubkey: expected 64-char hex pubkey", ) })?; @@ -506,29 +522,37 @@ pub async fn delete_community( let accepted = match admission { buzz_db::deletion::OwnerDeletionAdmission::Accepted(request) => request, buzz_db::deletion::OwnerDeletionAdmission::NotFoundOrNotOwner => { - return Err(api_error(StatusCode::NOT_FOUND, "community not found")); + return Err(deletion_api_error( + StatusCode::NOT_FOUND, + "community_not_found", + "community not found", + )); } buzz_db::deletion::OwnerDeletionAdmission::NotArchived => { - return Err(api_error( + return Err(deletion_api_error( StatusCode::CONFLICT, + "community_not_archived", "community must be archived before deletion", )); } buzz_db::deletion::OwnerDeletionAdmission::LifecycleConflict => { - return Err(api_error( + return Err(deletion_api_error( StatusCode::CONFLICT, + "deletion_lifecycle_conflict", "community deletion lifecycle is already active", )); } buzz_db::deletion::OwnerDeletionAdmission::RequestConflict => { - return Err(api_error( + return Err(deletion_api_error( StatusCode::CONFLICT, + "deletion_request_conflict", "deletion request conflicts with existing intent", )); } buzz_db::deletion::OwnerDeletionAdmission::UnsupportedAcknowledgementVersion => { - return Err(api_error( + return Err(deletion_api_error( StatusCode::BAD_REQUEST, + "unsupported_acknowledgement_version", "unsupported acknowledgement_version", )); } @@ -539,11 +563,73 @@ pub async fn delete_community( "request_id": accepted.id, "community_id": accepted.community_id.to_string(), "host": accepted.community_host, + "acknowledgement_version": accepted.acknowledgement_version, "status": accepted.stage.to_string(), })), )) } +/// Read one durable owner-deletion receipt without admitting or mutating work. +pub async fn delete_community_receipt( + State(state): State>, + headers: HeaderMap, + body: axum::body::Bytes, +) -> Result, (StatusCode, Json)> { + authorize_operator_request( + &state, + &headers, + "POST", + DELETE_RECEIPT_PATH, + None, + Some(&body), + ) + .await?; + let request: DeleteCommunityRequest = serde_json::from_slice(&body).map_err(|e| { + deletion_api_error( + StatusCode::BAD_REQUEST, + "invalid_request", + &format!("invalid delete-receipt JSON: {e}"), + ) + })?; + let owner = validate_pubkey_hex(&request.owner_pubkey).ok_or_else(|| { + deletion_api_error( + StatusCode::BAD_REQUEST, + "invalid_request", + "invalid owner_pubkey: expected 64-char hex pubkey", + ) + })?; + let receipt = state.db.deletion_store().get(request.request_id).await; + let receipt = match receipt { + Ok(receipt) + if receipt.request_origin == buzz_db::deletion::DeletionRequestOrigin::Owner + && receipt.owner_pubkey.as_deref() == Some(owner.as_str()) + && receipt.community_host == request.host + && receipt.acknowledgement_version == Some(request.acknowledgement_version) => + { + receipt + } + Ok(_) | Err(buzz_db::DbError::NotFound(_)) => { + return Err(deletion_api_error( + StatusCode::NOT_FOUND, + "deletion_receipt_not_found", + "deletion receipt not found", + )); + } + Err(error) => { + return Err(internal_error(&format!( + "read owner deletion receipt: {error}" + ))); + } + }; + Ok(Json(serde_json::json!({ + "request_id": receipt.id, + "community_id": receipt.community_id.to_string(), + "host": receipt.community_host, + "acknowledgement_version": receipt.acknowledgement_version, + "status": receipt.stage.to_string(), + }))) +} + /// List communities where a pubkey currently holds the `owner` role. pub async fn list_owned_communities( State(state): State>, @@ -568,7 +654,7 @@ pub async fn list_owned_communities( ) })?; - let rows = state + let page = state .db .list_communities_owned_by(&owner_pubkey) .await @@ -576,12 +662,15 @@ pub async fn list_owned_communities( Ok(Json(serde_json::json!({ "owner_pubkey": owner_pubkey, - "communities": rows.into_iter().map(|row| serde_json::json!({ + "communities": page.communities.into_iter().map(|row| serde_json::json!({ "community_id": row.id.to_string(), "host": row.host, "created_at": row.created_at, "archived_at": row.archived_at, })).collect::>(), + "quota_used": page.quota_used, + "quota_limit": page.quota_limit, + "can_create": page.can_create, }))) } @@ -1111,6 +1200,7 @@ mod postgres_tests { assert_eq!(response.status(), StatusCode::ACCEPTED); let json = read_json(response).await; assert_eq!(json["request_id"], request_id.to_string()); + assert_eq!(json["acknowledgement_version"], 1); assert_eq!(json["status"], "submitted"); let request = state .db @@ -1127,6 +1217,136 @@ mod postgres_tests { ); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_delete_receipt_is_bound_read_only_and_reports_aborted() { + let operator = Keys::generate(); + let outsider = Keys::generate(); + let owner = Keys::generate(); + let Some(state) = operator_test_state(std::slice::from_ref(&operator)).await else { + return; + }; + let host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(Arc::clone(&state), &operator, &host, &owner) + .await + .status(), + StatusCode::OK + ); + archive_for_owner_deletion(&state, &host, &owner).await; + let request_id = Uuid::new_v4(); + let body = owner_delete_body(&host, &owner, request_id); + assert_eq!( + signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(body.clone()), + ) + .await + .status(), + StatusCode::ACCEPTED + ); + let before = state + .db + .deletion_store() + .list(1_000) + .await + .expect("list receipts before reads") + .into_iter() + .filter(|request| request.id == request_id) + .count(); + + let receipt = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete/receipt", + Some(body.clone()), + ) + .await; + assert_eq!(receipt.status(), StatusCode::OK); + let receipt = read_json(receipt).await; + assert_eq!(receipt["request_id"], request_id.to_string()); + assert_eq!(receipt["host"], host); + assert_eq!(receipt["acknowledgement_version"], 1); + assert_eq!(receipt["status"], "submitted"); + + for mismatch in [ + serde_json::json!({ + "host": format!("wrong-{host}"), + "owner_pubkey": owner.public_key().to_hex(), + "request_id": request_id, + "acknowledgement_version": 1, + }), + serde_json::json!({ + "host": host, + "owner_pubkey": Keys::generate().public_key().to_hex(), + "request_id": request_id, + "acknowledgement_version": 1, + }), + serde_json::json!({ + "host": host, + "owner_pubkey": owner.public_key().to_hex(), + "request_id": request_id, + "acknowledgement_version": 2, + }), + ] { + let response = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete/receipt", + Some(mismatch.to_string()), + ) + .await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + read_json(response).await["code"], + "deletion_receipt_not_found" + ); + } + + let outsider_response = signed_operator_request( + Arc::clone(&state), + &outsider, + "POST", + "/operator/communities/delete/receipt", + Some(body.clone()), + ) + .await; + assert_eq!(outsider_response.status(), StatusCode::FORBIDDEN); + + state + .db + .deletion_store() + .abort(request_id, &operator.public_key().to_hex(), "test abort") + .await + .expect("abort receipt"); + let aborted = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete/receipt", + Some(body), + ) + .await; + assert_eq!(aborted.status(), StatusCode::OK); + assert_eq!(read_json(aborted).await["status"], "aborted"); + + let after = state + .db + .deletion_store() + .list(1_000) + .await + .expect("list receipts after reads") + .into_iter() + .filter(|request| request.id == request_id) + .count(); + assert_eq!(before, after, "receipt lookups must not add request rows"); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn owner_delete_endpoint_rejects_protected_host_and_malformed_or_mismatched_target() { @@ -1145,6 +1365,7 @@ mod postgres_tests { ) .await; assert_eq!(protected.status(), StatusCode::CONFLICT); + assert_eq!(read_json(protected).await["code"], "protected_community"); // Each malformed case keeps every unrelated field valid so it reaches // the guard under test instead of tripping an earlier one, and none of @@ -1170,6 +1391,7 @@ mod postgres_tests { ) .await; assert_eq!(bad_host.status(), StatusCode::BAD_REQUEST); + assert_eq!(read_json(bad_host).await["code"], "invalid_request"); assert_no_persisted_request(&state, bad_host_id, "unnormalizable host").await; let bad_pubkey_id = Uuid::new_v4(); @@ -1219,6 +1441,10 @@ mod postgres_tests { ) .await; assert_eq!(bad_version.status(), StatusCode::BAD_REQUEST); + assert_eq!( + read_json(bad_version).await["code"], + "unsupported_acknowledgement_version" + ); assert_no_persisted_request(&state, bad_version_id, "unsupported acknowledgement").await; let unknown_host_id = Uuid::new_v4(); @@ -1239,6 +1465,7 @@ mod postgres_tests { ) .await; assert_eq!(unknown_host.status(), StatusCode::NOT_FOUND); + assert_eq!(read_json(unknown_host).await["code"], "community_not_found"); assert_no_persisted_request(&state, unknown_host_id, "unprovisioned host").await; let bad_uuid = signed_operator_request( @@ -2027,6 +2254,7 @@ mod postgres_tests { .await .expect("owned communities"); let row = owned + .communities .iter() .find(|row| row.host == host) .expect("archived row"); diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index 9993ea4c8ad..31690f9a423 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -334,6 +334,10 @@ pub fn build_router(state: Arc) -> Router { "/operator/communities/delete", post(api::operator::delete_community), ) + .route( + "/operator/communities/delete/receipt", + post(api::operator::delete_community_receipt), + ) .route( "/operator/communities/availability", get(api::operator::community_availability), diff --git a/migrations/0053_owner_deletion_quota_reservation.sql b/migrations/0053_owner_deletion_quota_reservation.sql new file mode 100644 index 00000000000..32331e4ae1d --- /dev/null +++ b/migrations/0053_owner_deletion_quota_reservation.sql @@ -0,0 +1,10 @@ +-- Keep owner quota reservation lookups bounded after relay membership is +-- purged but logical deletion has not yet completed. +SET LOCAL lock_timeout = '5s'; + +CREATE INDEX community_deletion_requests_owner_quota_reservations + ON community_deletion_requests (owner_pubkey) + INCLUDE (community_id) + WHERE request_origin = 'owner' + AND stage <> 'aborted' + AND completed_at IS NULL; diff --git a/schema/schema.sql b/schema/schema.sql index 1c75db8bc38..8ffbfcd533b 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1309,6 +1309,12 @@ CREATE INDEX community_deletion_requests_owner_preparable WHERE request_origin = 'owner' AND stage = 'submitted' AND blocked_at IS NULL; +CREATE INDEX community_deletion_requests_owner_quota_reservations + ON community_deletion_requests (owner_pubkey) + INCLUDE (community_id) + WHERE request_origin = 'owner' + AND stage <> 'aborted' + AND completed_at IS NULL; CREATE TABLE community_deletion_approvals ( request_id UUID PRIMARY KEY, From 3a7f9ab38a1b3416edc6da4292c07059209247f4 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 22:13:45 +0000 Subject: [PATCH 32/46] fix: route deletion receipts through writer acquisition Signed-off-by: Codex Co-authored-by: Codex --- crates/buzz-db/src/store/deletion.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 259307cd28f..4f07c126cab 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -969,9 +969,14 @@ impl DeletionStore { /// Read one request. pub async fn get(&self, request_id: Uuid) -> Result { + let mut conn = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1") .bind(request_id) - .fetch_optional(&self.pool) + .fetch_optional(&mut *conn) .await? .ok_or_else(|| DbError::NotFound(format!("community deletion {request_id}")))?; row_to_request(row) From b50d7f833e7e99a5fc2c1c3c62acd1d3a3cff590 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 22:17:20 +0000 Subject: [PATCH 33/46] docs: describe authoritative owner quota projection Signed-off-by: Codex Co-authored-by: Codex --- crates/buzz-db/src/store/community.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index 2b822f848b8..01eea254599 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -64,9 +64,13 @@ pub struct OwnedCommunityRecord { /// Owner-list rows plus the authoritative quota projection from one snapshot. #[derive(Debug, Clone, PartialEq, Eq)] pub struct OwnedCommunitiesPage { + /// Visible live owner memberships, excluding communities already under deletion. pub communities: Vec, + /// De-duplicated live memberships and incomplete owner deletion reservations. pub quota_used: i64, + /// Configured maximum hosted communities for one owner. pub quota_limit: i64, + /// Whether the authoritative snapshot leaves capacity for another community. pub can_create: bool, } From 0a4717d5a27185f3d6e810576d59ebf5ea230360 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 22:20:41 +0000 Subject: [PATCH 34/46] test: advance migration lock expectation to 0053 Signed-off-by: Codex Co-authored-by: Codex --- .../buzz-db/src/runtime/tests/thread_window_postgres_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs index 82b7d2fb0a6..c6d38474e1d 100644 --- a/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs +++ b/crates/buzz-db/src/runtime/tests/thread_window_postgres_tests.rs @@ -311,7 +311,7 @@ async fn migration_schema_thread_window_prebuild_does_not_queue_behind_writer() production_result.is_ok(), "production migrator must preserve ingestion progress: {production_result:?}" ); - assert_eq!(version, 52); + assert_eq!(version, 53); assert_eq!(final_oid, oid, "prebuild must not be replaced"); assert_eq!(count, 4, "all writer witnesses must persist"); } From 340ca56d36e29610a17c0169b9ee8ae98fa12d1d Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 22:23:38 +0000 Subject: [PATCH 35/46] test: cover aborted owner quota reservation Signed-off-by: Codex Co-authored-by: Codex --- crates/buzz-db/src/store/community.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index 01eea254599..8acd359dad9 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -1169,6 +1169,26 @@ mod postgres_tests { "membership and request deduplicate" ); + sqlx::query("UPDATE community_deletion_requests SET stage = 'aborted' WHERE id = $1") + .bind(request_id) + .execute(&db.pool) + .await + .expect("abort quota fixture"); + let aborted = db + .list_communities_owned_by(&owner) + .await + .expect("aborted quota"); + assert_eq!(aborted.communities.len(), 1, "abort restores the live row"); + assert_eq!( + aborted.quota_used, 1, + "abort must fall back to the preserved membership" + ); + sqlx::query("UPDATE community_deletion_requests SET stage = 'submitted' WHERE id = $1") + .bind(request_id) + .execute(&db.pool) + .await + .expect("restore pending quota fixture"); + sqlx::query("DELETE FROM relay_members WHERE community_id = $1") .bind(created.id.as_uuid()) .execute(&db.pool) From 546b68640aacc87a80807fdc85c43f6a42299820 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 28 Sep 2026 22:24:21 +0000 Subject: [PATCH 36/46] test: abort quota fixture through store contract Signed-off-by: Codex Co-authored-by: Codex --- crates/buzz-db/src/store/community.rs | 31 +++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index 8acd359dad9..c8a3755b48a 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -1140,7 +1140,7 @@ mod postgres_tests { let CreateCommunityWithOwnerResult::Created(created) = created else { panic!("expected created community") }; - let request_id = Uuid::new_v4(); + let mut request_id = Uuid::new_v4(); sqlx::query( "INSERT INTO community_deletion_requests \ (id, community_id, community_host, requested_by, request_origin, owner_pubkey, \ @@ -1169,9 +1169,12 @@ mod postgres_tests { "membership and request deduplicate" ); - sqlx::query("UPDATE community_deletion_requests SET stage = 'aborted' WHERE id = $1") - .bind(request_id) - .execute(&db.pool) + db.deletion_store() + .abort( + request_id, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "quota reservation test abort", + ) .await .expect("abort quota fixture"); let aborted = db @@ -1183,11 +1186,21 @@ mod postgres_tests { aborted.quota_used, 1, "abort must fall back to the preserved membership" ); - sqlx::query("UPDATE community_deletion_requests SET stage = 'submitted' WHERE id = $1") - .bind(request_id) - .execute(&db.pool) - .await - .expect("restore pending quota fixture"); + request_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO community_deletion_requests \ + (id, community_id, community_host, requested_by, request_origin, owner_pubkey, \ + mediating_operator_pubkey, acknowledgement_version) \ + VALUES ($1, $2, $3, $4, 'owner', $4, $5, 1)", + ) + .bind(request_id) + .bind(created.id.as_uuid()) + .bind(&host) + .bind(&owner) + .bind("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + .execute(&db.pool) + .await + .expect("insert replacement pending quota fixture"); sqlx::query("DELETE FROM relay_members WHERE community_id = $1") .bind(created.id.as_uuid()) From 99fa5aef4b8774cb2219d00debf9f9c01eed6b91 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 29 Sep 2026 02:21:58 +0000 Subject: [PATCH 37/46] chore: reserve migration 0054 for deletion quota Signed-off-by: Codex Co-authored-by: Codex --- ..._reservation.sql => 0054_owner_deletion_quota_reservation.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename migrations/{0053_owner_deletion_quota_reservation.sql => 0054_owner_deletion_quota_reservation.sql} (100%) diff --git a/migrations/0053_owner_deletion_quota_reservation.sql b/migrations/0054_owner_deletion_quota_reservation.sql similarity index 100% rename from migrations/0053_owner_deletion_quota_reservation.sql rename to migrations/0054_owner_deletion_quota_reservation.sql From bd95c4a98496e2905e26fb39509a2b0667cc8c3d Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 29 Sep 2026 02:27:25 +0000 Subject: [PATCH 38/46] fix(relay): return stable deletion lifecycle conflicts Signed-off-by: Codex Co-authored-by: Codex --- crates/buzz-relay/src/api/operator.rs | 259 +++++++++++++++++++++++++- 1 file changed, 257 insertions(+), 2 deletions(-) diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index e841bc2b81e..cd6abcb9e34 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -432,8 +432,9 @@ pub async fn unarchive_community( let record = match result { buzz_db::UnarchiveCommunityResult::Unarchived(record) => record, buzz_db::UnarchiveCommunityResult::DeletionPending => { - return Err(api_error( + return Err(deletion_api_error( StatusCode::CONFLICT, + "deletion_lifecycle_conflict", "community deletion is pending", )); } @@ -762,8 +763,9 @@ pub async fn transfer_community( )); } buzz_db::relay_members::TransferResult::DeletionPending => { - return Err(api_error( + return Err(deletion_api_error( StatusCode::CONFLICT, + "deletion_lifecycle_conflict", "community deletion is pending", )); } @@ -2151,6 +2153,128 @@ mod postgres_tests { assert_eq!(owner_member.role, "owner"); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn unarchive_pending_deletion_returns_stable_conflict_without_mutation() { + let operator = Keys::generate(); + let owner = Keys::generate(); + let Some(state) = operator_test_state(std::slice::from_ref(&operator)).await else { + return; + }; + let host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(Arc::clone(&state), &operator, &host, &owner) + .await + .status(), + StatusCode::OK + ); + let community = state + .db + .lookup_community_by_host(&host) + .await + .expect("lookup community") + .expect("community exists"); + archive_for_owner_deletion(&state, &host, &owner).await; + let request_id = Uuid::new_v4(); + assert_eq!( + signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(owner_delete_body(&host, &owner, request_id)), + ) + .await + .status(), + StatusCode::ACCEPTED + ); + + let pool = sqlx::PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect lifecycle assertion pool"); + let before_lifecycle: ( + Option>, + String, + Option>, + ) = sqlx::query_as( + "SELECT archived_at, deletion_state, deleted_at FROM communities WHERE id = $1", + ) + .bind(community.id.as_uuid()) + .fetch_one(&pool) + .await + .expect("read lifecycle before unarchive conflict"); + let before_request = state + .db + .deletion_store() + .get(request_id) + .await + .expect("read deletion request before unarchive conflict"); + let owner_hex = owner.public_key().to_hex(); + let before_owner_role = state + .db + .get_relay_member(community.id, &owner_hex) + .await + .expect("read owner before unarchive conflict") + .expect("owner exists before unarchive conflict") + .role; + + let body = serde_json::json!({ + "host": host, + "owner_pubkey": owner_hex, + }) + .to_string(); + let response = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/unarchive", + Some(body), + ) + .await; + assert_eq!(response.status(), StatusCode::CONFLICT); + let json = read_json(response).await; + assert_eq!(json["code"], "deletion_lifecycle_conflict"); + assert_eq!(json["error"], "community deletion is pending"); + + let after_lifecycle: ( + Option>, + String, + Option>, + ) = sqlx::query_as( + "SELECT archived_at, deletion_state, deleted_at FROM communities WHERE id = $1", + ) + .bind(community.id.as_uuid()) + .fetch_one(&pool) + .await + .expect("read lifecycle after unarchive conflict"); + assert_eq!(after_lifecycle, before_lifecycle); + assert_eq!( + state + .db + .deletion_store() + .get(request_id) + .await + .expect("read deletion request after unarchive conflict"), + before_request + ); + assert_eq!( + state + .db + .get_relay_member(community.id, &owner.public_key().to_hex()) + .await + .expect("read owner after unarchive conflict") + .expect("owner exists after unarchive conflict") + .role, + before_owner_role + ); + assert!(state + .db + .lookup_community_by_host(&host) + .await + .expect("active lookup after unarchive conflict") + .is_none()); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn archive_publish_failure_is_retryable_and_preserves_timestamp() { @@ -2485,6 +2609,137 @@ mod postgres_tests { .await; } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn transfer_pending_deletion_returns_stable_conflict_without_mutation() { + let operator = Keys::generate(); + let initial_owner = Keys::generate(); + let new_owner = Keys::generate(); + let Some(state) = operator_test_state(std::slice::from_ref(&operator)).await else { + return; + }; + let host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(Arc::clone(&state), &operator, &host, &initial_owner) + .await + .status(), + StatusCode::OK + ); + let community = state + .db + .lookup_community_by_host(&host) + .await + .expect("lookup community") + .expect("community exists"); + archive_for_owner_deletion(&state, &host, &initial_owner).await; + let request_id = Uuid::new_v4(); + assert_eq!( + signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(owner_delete_body(&host, &initial_owner, request_id)), + ) + .await + .status(), + StatusCode::ACCEPTED + ); + + let pool = sqlx::PgPool::connect(&crate::test_support::database_url()) + .await + .expect("connect lifecycle assertion pool"); + let before_lifecycle: ( + Option>, + String, + Option>, + ) = sqlx::query_as( + "SELECT archived_at, deletion_state, deleted_at FROM communities WHERE id = $1", + ) + .bind(community.id.as_uuid()) + .fetch_one(&pool) + .await + .expect("read lifecycle before transfer conflict"); + let before_request = state + .db + .deletion_store() + .get(request_id) + .await + .expect("read deletion request before transfer conflict"); + let initial_owner_hex = initial_owner.public_key().to_hex(); + let new_owner_hex = new_owner.public_key().to_hex(); + let before_owner_role = state + .db + .get_relay_member(community.id, &initial_owner_hex) + .await + .expect("read owner before transfer conflict") + .expect("owner exists before transfer conflict") + .role; + assert!(state + .db + .get_relay_member(community.id, &new_owner_hex) + .await + .expect("read transferee before transfer conflict") + .is_none()); + + let body = serde_json::json!({ + "community_id": community.id.to_string(), + "new_owner_pubkey": new_owner_hex, + "expected_owner_pubkey": initial_owner_hex, + }) + .to_string(); + let response = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/transfer", + Some(body), + ) + .await; + assert_eq!(response.status(), StatusCode::CONFLICT); + let json = read_json(response).await; + assert_eq!(json["code"], "deletion_lifecycle_conflict"); + assert_eq!(json["error"], "community deletion is pending"); + + let after_lifecycle: ( + Option>, + String, + Option>, + ) = sqlx::query_as( + "SELECT archived_at, deletion_state, deleted_at FROM communities WHERE id = $1", + ) + .bind(community.id.as_uuid()) + .fetch_one(&pool) + .await + .expect("read lifecycle after transfer conflict"); + assert_eq!(after_lifecycle, before_lifecycle); + assert_eq!( + state + .db + .deletion_store() + .get(request_id) + .await + .expect("read deletion request after transfer conflict"), + before_request + ); + assert_eq!( + state + .db + .get_relay_member(community.id, &initial_owner.public_key().to_hex()) + .await + .expect("read owner after transfer conflict") + .expect("owner exists after transfer conflict") + .role, + before_owner_role + ); + assert!(state + .db + .get_relay_member(community.id, &new_owner.public_key().to_hex()) + .await + .expect("read transferee after transfer conflict") + .is_none()); + } + /// Transfer with an invalid community_id returns 400. #[tokio::test] #[ignore = "requires Postgres"] From 0d7798cb4515564c20f4df46e61077ec6d88995d Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 29 Sep 2026 04:43:54 +0000 Subject: [PATCH 39/46] fix deletion admission and quota lifecycle proof Signed-off-by: Codex --- crates/buzz-db/src/store/community.rs | 116 -------------------------- crates/buzz-db/src/store/deletion.rs | 83 ++++++++++++++++++ crates/buzz-relay/src/api/operator.rs | 70 ++++++++++++++++ docs/operator-community-deletion.md | 17 ++++ 4 files changed, 170 insertions(+), 116 deletions(-) diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index c8a3755b48a..82cd510ff82 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -1127,122 +1127,6 @@ mod postgres_tests { assert!(owned.can_create); } - #[tokio::test] - #[ignore = "requires Postgres"] - async fn owner_quota_reservation_survives_purge_stages_and_releases_terminally() { - let db = setup_db().await; - let owner = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple()); - let host = format!("quota-reservation-{}.example", Uuid::new_v4().simple()); - let created = db - .create_community_with_owner(&host, &owner) - .await - .expect("create owned community"); - let CreateCommunityWithOwnerResult::Created(created) = created else { - panic!("expected created community") - }; - let mut request_id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO community_deletion_requests \ - (id, community_id, community_host, requested_by, request_origin, owner_pubkey, \ - mediating_operator_pubkey, acknowledgement_version) \ - VALUES ($1, $2, $3, $4, 'owner', $4, $5, 1)", - ) - .bind(request_id) - .bind(created.id.as_uuid()) - .bind(&host) - .bind(&owner) - .bind("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") - .execute(&db.pool) - .await - .expect("insert owner deletion request"); - - let pre_purge = db - .list_communities_owned_by(&owner) - .await - .expect("pre-purge quota"); - assert!( - pre_purge.communities.is_empty(), - "pending row is suppressed" - ); - assert_eq!( - pre_purge.quota_used, 1, - "membership and request deduplicate" - ); - - db.deletion_store() - .abort( - request_id, - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "quota reservation test abort", - ) - .await - .expect("abort quota fixture"); - let aborted = db - .list_communities_owned_by(&owner) - .await - .expect("aborted quota"); - assert_eq!(aborted.communities.len(), 1, "abort restores the live row"); - assert_eq!( - aborted.quota_used, 1, - "abort must fall back to the preserved membership" - ); - request_id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO community_deletion_requests \ - (id, community_id, community_host, requested_by, request_origin, owner_pubkey, \ - mediating_operator_pubkey, acknowledgement_version) \ - VALUES ($1, $2, $3, $4, 'owner', $4, $5, 1)", - ) - .bind(request_id) - .bind(created.id.as_uuid()) - .bind(&host) - .bind(&owner) - .bind("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") - .execute(&db.pool) - .await - .expect("insert replacement pending quota fixture"); - - sqlx::query("DELETE FROM relay_members WHERE community_id = $1") - .bind(created.id.as_uuid()) - .execute(&db.pool) - .await - .expect("simulate membership purge"); - for stage in ["postgres_purged", "cache_purged", "logically_verified"] { - sqlx::query( - "UPDATE community_deletion_requests SET stage = $2, completed_at = NULL WHERE id = $1", - ) - .bind(request_id) - .bind(stage) - .execute(&db.pool) - .await - .expect("advance quota fixture"); - assert_eq!( - db.list_communities_owned_by(&owner) - .await - .expect("reserved quota") - .quota_used, - 1, - "{stage} must retain the slot" - ); - } - - sqlx::query( - "UPDATE community_deletion_requests \ - SET stage = 'retention_pending', completed_at = now() WHERE id = $1", - ) - .bind(request_id) - .execute(&db.pool) - .await - .expect("complete quota fixture"); - assert_eq!( - db.list_communities_owned_by(&owner) - .await - .expect("released quota") - .quota_used, - 0 - ); - } - #[tokio::test] #[ignore = "requires Postgres"] async fn owner_quota_serializes_concurrent_create_and_transfer_with_reservation() { diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 104344d1a34..979298dd77e 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -5349,12 +5349,28 @@ mod postgres_tests { async fn owner_preparation_atomically_approves_exact_inventory_and_converges() { let (db, store) = store().await; let (host, owner, community) = archived_owned_community(&db).await; + assert_eq!( + db.list_communities_owned_by(&owner) + .await + .expect("initial owner quota") + .quota_used, + 1 + ); let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; let request_id = Uuid::new_v4(); store .admit_owner_request(&host, &owner, operator, 1, request_id) .await .expect("admit owner request"); + let pending_quota = db + .list_communities_owned_by(&owner) + .await + .expect("pending owner quota"); + assert!(pending_quota.communities.is_empty()); + assert_eq!( + pending_quota.quota_used, 1, + "membership and incomplete request must deduplicate" + ); let claim = store .claim_specific_owner_submission(request_id, "preparer", DEFAULT_LEASE_DURATION) .await @@ -5419,6 +5435,73 @@ mod postgres_tests { .complete_owner_preparation(&claim.lease, &changed) .await .is_err()); + + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await + .expect("freeze destructive storage"); + store.mark_drained(&token).await.expect("drain"); + store + .mark_bindings_removed(&token, serde_json::json!({"keys": 0})) + .await + .expect("bindings"); + store.purge_postgres(&token).await.expect("purge postgres"); + assert_eq!( + db.list_communities_owned_by(&owner) + .await + .expect("quota after membership purge") + .quota_used, + 1, + "the incomplete owner request must reserve the slot after membership purge" + ); + store + .mark_cache_purged(&token, serde_json::json!({"keys": 0})) + .await + .expect("cache"); + store + .verify_postgres_logically_deleted(&token) + .await + .expect("logical postgres verify"); + store + .mark_logically_verified(&token, serde_json::json!({"all": true})) + .await + .expect("mark verified"); + assert_eq!( + db.list_communities_owned_by(&owner) + .await + .expect("quota before logical completion") + .quota_used, + 1 + ); + store + .mark_retention_pending(&token, serde_json::json!({"shared_cas": "retained"})) + .await + .expect("production terminal transition"); + assert_eq!( + db.list_communities_owned_by(&owner) + .await + .expect("quota after logical completion") + .quota_used, + 0, + "the production logical-completion transition releases the slot" + ); + assert!(db + .lookup_community_by_host_for_management(&host) + .await + .expect("permanent tombstone lookup") + .is_some()); + assert_eq!( + db.create_community_with_owner(&host, &owner) + .await + .expect("recreate tombstoned host"), + CreateCommunityWithOwnerResult::HostExists + ); } #[tokio::test] diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index cd6abcb9e34..5bf451951ae 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -492,6 +492,13 @@ pub async fn delete_community( })?; let normalized_host = normalize_candidate_host(&request.host) .map_err(|msg| deletion_api_error(StatusCode::BAD_REQUEST, "invalid_request", &msg))?; + if normalized_host != request.host { + return Err(deletion_api_error( + StatusCode::BAD_REQUEST, + "invalid_request", + "host must use its exact canonical authority spelling", + )); + } let deployment_host = buzz_core::tenant::relay_url_authority(&state.config.relay_url); if normalized_host == deployment_host { return Err(deletion_api_error( @@ -1219,6 +1226,69 @@ mod postgres_tests { ); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_delete_admission_requires_exact_canonical_host_without_mutation() { + let operator = Keys::generate(); + let owner = Keys::generate(); + let Some(state) = operator_test_state(std::slice::from_ref(&operator)).await else { + return; + }; + let host = format!("community-{}.example", Uuid::new_v4().simple()); + assert_eq!( + provision_community(Arc::clone(&state), &operator, &host, &owner) + .await + .status(), + StatusCode::OK + ); + archive_for_owner_deletion(&state, &host, &owner).await; + + for (label, repaired) in [ + ("whitespace", format!(" {host}")), + ("case", host.to_uppercase()), + ("url", format!("https://{host}")), + ] { + let request_id = Uuid::new_v4(); + let response = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(owner_delete_body(&repaired, &owner, request_id)), + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{label}"); + assert_eq!(read_json(response).await["code"], "invalid_request"); + assert_no_persisted_request(&state, request_id, label).await; + } + + let request_id = Uuid::new_v4(); + let body = owner_delete_body(&host, &owner, request_id); + let accepted = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(body.clone()), + ) + .await; + assert_eq!(accepted.status(), StatusCode::ACCEPTED); + assert_eq!(read_json(accepted).await["host"], host); + + let receipt = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete/receipt", + Some(body), + ) + .await; + assert_eq!(receipt.status(), StatusCode::OK); + let receipt = read_json(receipt).await; + assert_eq!(receipt["host"], host); + assert_eq!(receipt["request_id"], request_id.to_string()); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn owner_delete_receipt_is_bound_read_only_and_reports_aborted() { diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index ed0df6666fd..03e9f7186e6 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -150,6 +150,23 @@ eligible for execution. Transient preparation failures use the existing retry schedule; permanent or exhausted failures block durably. Owner-facing admission has no cancellation endpoint. +## Client compatibility and rollout order + +Owner-list responses now carry the authoritative quota trio: `quota_used`, +`quota_limit`, and `can_create`. Quota-dependent clients fail closed when any +member of that trio is absent: they disable new Create controls even when the +separate community-deletion capability is absent or false. They do not derive +capacity from the number of visible rows, because an in-progress deletion can +hide a row while still reserving its owner's slot. + +Roll this contract out in dependency order: relay first, then KGoose, then +Desktop and any other quota-dependent clients. This ordering is a compatibility +requirement, not authorization to enable owner deletion or its drain job. +During that rollout, keep the already-published migration 0052 channel-artifact +surface and migration 0053 owner auto-approval surface byte-for-byte intact; +do not rewrite their migration ledger entries or recreate a persistent +database to introduce the quota projection. + The chart has no existing PrometheusRule or provider-neutral CronJob alert integration. Operators must alert on failed/missed Jobs and long-running active Jobs in their deployment platform. Adding a chart-native alert abstraction is From 37e9b68c96f4f1f1de7f1a2456ecd6515bcba2c2 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 29 Sep 2026 17:16:50 +0000 Subject: [PATCH 40/46] fix(db): restore owner deletion approval authority fence Signed-off-by: Codex --- crates/buzz-db/src/store/deletion.rs | 195 +++++++++++++++++++++++++++ docs/operator-community-deletion.md | 13 +- 2 files changed, 202 insertions(+), 6 deletions(-) diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index 31af26c2571..74d463e2977 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -1419,6 +1419,28 @@ impl DeletionStore { let storage = serde_json::to_value(&inventory.storage)?; let frozen = serde_json::to_value(inventory)?; let mut tx = self.pool.begin().await?; + + lock_community_deletion_shared(&mut tx, token.community_id).await?; + let community = sqlx::query( + "SELECT archived_at, deletion_state, deleted_at FROM communities \ + WHERE id = $1 FOR UPDATE", + ) + .bind(token.community_id.as_uuid()) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "owner deletion {} community is missing before automatic approval", + token.request_id + )) + })?; + let current_owners: Vec = sqlx::query_scalar( + "SELECT pubkey FROM relay_members \ + WHERE community_id = $1 AND role = 'owner' ORDER BY pubkey FOR UPDATE", + ) + .bind(token.community_id.as_uuid()) + .fetch_all(&mut *tx) + .await?; let request_row = sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1 FOR UPDATE") .bind(token.request_id) @@ -1446,6 +1468,22 @@ impl DeletionStore { if !lease_matches { return Err(stale_lease_error(token)); } + let archived_at: Option> = community.try_get("archived_at")?; + let deletion_state: String = community.try_get("deletion_state")?; + let deleted_at: Option> = community.try_get("deleted_at")?; + let owner_authority_matches = request.owner_pubkey.as_ref().is_some_and(|owner| { + current_owners.len() == 1 && current_owners.first() == Some(owner) + }); + if archived_at.is_none() + || deletion_state != "active" + || deleted_at.is_some() + || !owner_authority_matches + { + return Err(DbError::DeletionSafety(format!( + "owner deletion {} community is no longer archived under the admitted owner", + token.request_id + ))); + } if request.stage == DeletionStage::Approved { let approval: Option<(Vec, String, String)> = sqlx::query_as( "SELECT inventory_digest, approved_by, approval_origin \ @@ -5128,6 +5166,77 @@ mod postgres_tests { ); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn privileged_abort_fences_a_live_owner_preparation_lease() { + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + store + .admit_owner_request(&host, &owner, operator, 1, request_id) + .await + .expect("admit owner request"); + let claim = store + .claim_specific_owner_submission(request_id, "preparer", DEFAULT_LEASE_DURATION) + .await + .expect("claim owner request") + .expect("owner request is preparable"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("schema inventory"), + storage: empty_storage_manifest(community), + }; + let (lease_owner, lease_generation, lease_is_live): (Option, i64, bool) = + sqlx::query_as( + "SELECT lease_owner, lease_generation, lease_until >= now() \ + FROM community_deletion_requests WHERE id = $1", + ) + .bind(request_id) + .fetch_one(&db.pool) + .await + .expect("read live preparation lease"); + assert_eq!(lease_owner.as_deref(), Some("preparer")); + assert_eq!(lease_generation, claim.lease.generation); + assert!(lease_is_live, "preparation lease must be live before abort"); + + let aborted = store + .abort(request_id, "recovery-operator", "cancel live preparation") + .await + .expect("abort live preparation"); + assert_eq!(aborted.stage, DeletionStage::Aborted); + assert_eq!(aborted.lease_generation, claim.lease.generation + 1); + assert!(aborted.lease_owner.is_none()); + assert!(aborted.lease_until.is_none()); + + let heartbeat_error = store + .heartbeat_owner_submission(&claim.lease, "drain", DEFAULT_LEASE_DURATION, false) + .await + .expect_err("aborted preparation lease cannot heartbeat"); + assert!(is_stale_deletion_lease(&heartbeat_error)); + let completion_error = store + .complete_owner_preparation(&claim.lease, &inventory) + .await + .expect_err("aborted preparation lease cannot approve"); + assert!(is_stale_deletion_lease(&completion_error)); + + let request = store.get(request_id).await.expect("load aborted request"); + assert_eq!(request.stage, DeletionStage::Aborted); + assert!(request.inventory_digest.is_none()); + assert_eq!( + sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM community_deletion_approvals WHERE request_id = $1", + ) + .bind(request_id) + .fetch_one(&db.pool) + .await + .expect("count automatic approvals"), + 0 + ); + } + /// The reversible boundary extends through `fenced`. From `drained` /// onward, destruction may have begun, so abort must stay closed. #[tokio::test] @@ -5504,6 +5613,92 @@ mod postgres_tests { ); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn owner_preparation_rechecks_archived_current_owner_before_automatic_approval() { + enum AuthorityDrift { + Unarchived, + OwnerChanged, + } + + let mut failures = Vec::new(); + for drift in [AuthorityDrift::Unarchived, AuthorityDrift::OwnerChanged] { + let label = match drift { + AuthorityDrift::Unarchived => "unarchived", + AuthorityDrift::OwnerChanged => "owner-changed", + }; + let (db, store) = store().await; + let (host, owner, community) = archived_owned_community(&db).await; + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let request_id = Uuid::new_v4(); + store + .admit_owner_request(&host, &owner, operator, 1, request_id) + .await + .expect("admit owner request"); + let claim = store + .claim_specific_owner_submission(request_id, "preparer", DEFAULT_LEASE_DURATION) + .await + .expect("claim owner request") + .expect("owner request is preparable"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("schema inventory"), + storage: empty_storage_manifest(community), + }; + + match drift { + AuthorityDrift::Unarchived => { + sqlx::query("UPDATE communities SET archived_at = NULL WHERE id = $1") + .bind(community.as_uuid()) + .execute(&db.pool) + .await + .expect("simulate stale archive authority"); + } + AuthorityDrift::OwnerChanged => { + sqlx::query( + "UPDATE relay_members SET role = 'member' \ + WHERE community_id = $1 AND pubkey = $2 AND role = 'owner'", + ) + .bind(community.as_uuid()) + .bind(&owner) + .execute(&db.pool) + .await + .expect("simulate stale owner authority"); + } + } + + let completion = store + .complete_owner_preparation(&claim.lease, &inventory) + .await; + let request = store.get(request_id).await.expect("load request"); + let approval_count = sqlx::query_scalar::<_, i64>( + "SELECT count(*) FROM community_deletion_approvals WHERE request_id = $1", + ) + .bind(request_id) + .fetch_one(&db.pool) + .await + .expect("count automatic approvals"); + if completion.is_ok() + || request.stage != DeletionStage::Submitted + || request.inventory_digest.is_some() + || approval_count != 0 + { + failures.push(format!( + "{label}: completion={completion:?}, stage={}, inventory_frozen={}, approvals={approval_count}", + request.stage, + request.inventory_digest.is_some(), + )); + } + } + + assert!( + failures.is_empty(), + "stale owner authority reached automatic approval: {failures:#?}" + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn stale_owner_preparation_generation_cannot_freeze_or_approve() { diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index cb6b54f5955..5a6dc4cde57 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -6,10 +6,11 @@ that command as a Kubernetes CronJob; it does not call relay HTTP and it does not add another queue or retry service. Postgres remains the handoff and source of truth. A run gives already-approved -work priority. When none is ready, it may claim an operator-attested owner-origin -request at `submitted`, build the existing bounded inventory, and atomically -freeze that inventory with a digest-bound `owner_automatic` approval. The same -lease then enters the unchanged executor and resumes from durable checkpoints. +work priority. When none is ready, it may claim an operator-attested +owner-origin request at `submitted`, build the existing bounded inventory, and +atomically freeze that inventory with a digest-bound `owner_automatic` +approval. The same lease then enters the unchanged executor and resumes from +durable checkpoints. Operator-origin requests never auto-progress. `concurrencyPolicy: Forbid` prevents scheduled pod overlap, `backoffLimit: 0` prevents Kubernetes Job retries, and the deletion store remains authoritative when a pod exits, reaches its deadline, or is @@ -144,8 +145,8 @@ handoff. Owner self-serve relay admission still records only a `submitted` row and does no inventory, approval, S3 work, or execution synchronously. A successful drain -has no human approval step or cooling-off period: operator-attested owner intent is -prepared automatically under privileged policy and becomes immediately +has no human approval step or cooling-off period: operator-attested owner intent +is prepared automatically under privileged policy and becomes immediately eligible for execution. Transient preparation failures use the existing retry schedule; permanent or exhausted failures block durably. Owner-facing admission has no cancellation endpoint. From 7400268206e582445a27d69397b553a512b88e2c Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Tue, 29 Sep 2026 17:22:30 -0400 Subject: [PATCH 41/46] refactor(relay): make owner delete the idempotent recovery call; add lifetime owner cap Drop POST /operator/communities/delete/receipt. Owner deletion admission already converges on an existing request UUID with the same tuple before any owner or archive check, so resending delete returns 202 with the current status at any stage and admits no new work. Count every non-aborted owner deletion, completed or not, toward a lifetime cap of 20 communities per owner (never below the active cap) on create and transfer-in, so create-then-delete cannot squat tombstoned hosts. The owner-list can_create projection reflects both caps. Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/src/runtime/migration.rs | 2 +- crates/buzz-db/src/store/community.rs | 91 +++++++++-- crates/buzz-db/src/store/deletion.rs | 7 +- crates/buzz-db/src/store/relay_members.rs | 76 +++++++--- crates/buzz-relay/src/api/operator.rs | 141 ++++++------------ crates/buzz-relay/src/router.rs | 4 - docs/operator-community-deletion.md | 41 +++-- .../0054_owner_deletion_quota_reservation.sql | 10 +- schema/schema.sql | 5 +- 9 files changed, 217 insertions(+), 160 deletions(-) diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index bbea441502b..ce823310fe6 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -1784,7 +1784,7 @@ mod postgres_tests { assert!(sql.contains("community_deletion_requests_owner_quota_reservations")); assert!(sql.contains("request_origin = 'owner'")); assert!(sql.contains("stage <> 'aborted'")); - assert!(sql.contains("completed_at is null")); + assert!(sql.contains("include (community_id, completed_at)")); } assert!(migration.contains("set local lock_timeout = '5s'")); } diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index 82cd510ff82..46db43b6730 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -70,7 +70,7 @@ pub struct OwnedCommunitiesPage { pub quota_used: i64, /// Configured maximum hosted communities for one owner. pub quota_limit: i64, - /// Whether the authoritative snapshot leaves capacity for another community. + /// Whether the snapshot leaves room under both the active and lifetime caps. pub can_create: bool, } @@ -262,15 +262,13 @@ impl Db { }) }) .collect::>>()?; - let quota_used = - relay_members::owner_quota_used_in_transaction(&mut tx, &owner_pubkey).await?; + let quota = relay_members::owner_quota_in_transaction(&mut tx, &owner_pubkey).await?; tx.commit().await?; - let quota_limit = relay_members::max_communities_per_owner(); Ok(OwnedCommunitiesPage { communities, - quota_used, - quota_limit, - can_create: quota_used < quota_limit, + quota_used: quota.active, + quota_limit: relay_members::max_communities_per_owner(), + can_create: quota.admits(), }) } @@ -477,10 +475,10 @@ impl Db { let host: String = row.try_get("host")?; // Enforce the limit before inserting the new owner row. - let owned_count = - relay_members::owner_quota_used_in_transaction(&mut tx, &owner_pubkey).await?; - - if owned_count >= relay_members::max_communities_per_owner() { + if !relay_members::owner_quota_in_transaction(&mut tx, &owner_pubkey) + .await? + .admits() + { tx.rollback().await?; return Ok(CreateCommunityWithOwnerResult::LimitReached); } @@ -1210,6 +1208,77 @@ mod postgres_tests { ); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn completed_owner_deletions_count_toward_lifetime_cap() { + let db = setup_db().await; + let owner = format!("{:064x}", Uuid::new_v4().as_u128()); + let other_owner = format!("{:064x}", Uuid::new_v4().as_u128()); + let operator = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + + // Create and completely delete up to the lifetime cap: each tombstone + // frees its active slot but keeps its host and its lifetime count. + for i in 0..crate::relay_members::MAX_LIFETIME_COMMUNITIES_PER_OWNER { + let host = format!("lifetime-{i}-{}.example", Uuid::new_v4().simple()); + let CreateCommunityWithOwnerResult::Created(record) = db + .create_community_with_owner(&host, &owner) + .await + .expect("create under lifetime cap") + else { + panic!("create {i} must succeed below the lifetime cap") + }; + sqlx::query( + "INSERT INTO community_deletion_requests \ + (id, community_id, community_host, requested_by, request_origin, owner_pubkey, \ + mediating_operator_pubkey, acknowledgement_version, stage, completed_at) \ + VALUES ($1, $2, $3, $4, 'owner', $4, $5, 1, 'retention_pending', now())", + ) + .bind(Uuid::new_v4()) + .bind(record.id.as_uuid()) + .bind(&host) + .bind(&owner) + .bind(operator) + .execute(&db.pool) + .await + .expect("insert completed owner deletion"); + sqlx::query("DELETE FROM relay_members WHERE community_id = $1") + .bind(record.id.as_uuid()) + .execute(&db.pool) + .await + .expect("simulate purged membership"); + } + + let page = db + .list_communities_owned_by(&owner) + .await + .expect("owner list at lifetime cap"); + assert_eq!(page.quota_used, 0, "completed deletions free active slots"); + assert!(!page.can_create, "the lifetime cap still blocks creation"); + + let host = format!("lifetime-overflow-{}.example", Uuid::new_v4().simple()); + assert_eq!( + db.create_community_with_owner(&host, &owner) + .await + .expect("create past lifetime cap"), + CreateCommunityWithOwnerResult::LimitReached + ); + + let transfer_host = format!("lifetime-transfer-{}.example", Uuid::new_v4().simple()); + let CreateCommunityWithOwnerResult::Created(target) = db + .create_community_with_owner(&transfer_host, &other_owner) + .await + .expect("create transfer target") + else { + panic!("expected transfer target") + }; + assert_eq!( + db.transfer_ownership(target.id, &owner, &other_owner) + .await + .expect("transfer past lifetime cap"), + crate::relay_members::TransferResult::LimitReached + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn communities_of_channels_present_for_existing_absent_for_missing() { diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index c7d4848a4a8..96629dcdeb4 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -995,14 +995,9 @@ impl DeletionStore { /// Read one request. pub async fn get(&self, request_id: Uuid) -> Result { - let mut conn = crate::observability::acquire_writer( - &self.pool, - crate::observability::WriterOperation::Authorization, - ) - .await?; let row = sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1") .bind(request_id) - .fetch_optional(&mut *conn) + .fetch_optional(&self.pool) .await? .ok_or_else(|| DbError::NotFound(format!("community deletion {request_id}")))?; row_to_request(row) diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 8b1e9a3e07a..ba0651475c2 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -604,36 +604,69 @@ pub fn owner_count_advisory_lock_key(pubkey_hex: &str) -> i64 { h as i64 } -/// Count live ownership plus incomplete owner-deletion reservations. +/// Lifetime cap on communities a pubkey may own, counting owner-deleted +/// communities whose tombstones permanently retain their hosts. Bounds +/// create-then-delete host squatting; never below the active limit. +pub const MAX_LIFETIME_COMMUNITIES_PER_OWNER: i64 = 20; + +/// One owner's quota usage, read inside the admitting transaction. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct OwnerQuota { + /// Live ownership plus incomplete owner-deletion reservations. + pub active: i64, + /// Live ownership plus every non-aborted owner deletion, completed or not. + pub lifetime: i64, +} + +impl OwnerQuota { + /// Whether this owner may gain one more community. + pub fn admits(self) -> bool { + let limit = max_communities_per_owner(); + self.active < limit && self.lifetime < MAX_LIFETIME_COMMUNITIES_PER_OWNER.max(limit) + } +} + +/// Read an owner's active and lifetime community counts. /// /// `UNION` deliberately de-duplicates the live membership and deletion row -/// before PostgreSQL purges membership. The reservation remains until the -/// logical-completion transition records `completed_at`. -pub(crate) async fn owner_quota_used_in_transaction( +/// before PostgreSQL purges membership. An active reservation remains until +/// the logical-completion transition records `completed_at`; the lifetime +/// count keeps it forever. Aborted requests restore the community, which is +/// then counted through its live membership. +pub(crate) async fn owner_quota_in_transaction( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, owner_pubkey: &str, -) -> Result { - sqlx::query_scalar( +) -> Result { + let (active, lifetime): (i64, i64) = sqlx::query_as( r#" - SELECT count(*)::BIGINT - FROM ( + WITH owned AS ( SELECT community_id FROM relay_members WHERE pubkey = $1 AND role = 'owner' - UNION - SELECT community_id + ), deleted AS ( + SELECT community_id, completed_at FROM community_deletion_requests WHERE request_origin = 'owner' AND owner_pubkey = $1 AND stage <> 'aborted' - AND completed_at IS NULL - ) quota_reservations + ) + SELECT + (SELECT count(*) FROM ( + SELECT community_id FROM owned + UNION + SELECT community_id FROM deleted WHERE completed_at IS NULL + ) active)::BIGINT, + (SELECT count(*) FROM ( + SELECT community_id FROM owned + UNION + SELECT community_id FROM deleted + ) lifetime)::BIGINT "#, ) .bind(owner_pubkey) .fetch_one(&mut **tx) - .await - .map_err(Into::into) + .await?; + Ok(OwnerQuota { active, lifetime }) } /// Atomically transfers ownership of `community` to `new_owner_pubkey`. @@ -730,9 +763,7 @@ pub async fn transfer_ownership( // 4. Enforce the transferee's community ownership limit inside the same // transaction that holds the advisory lock. This is the authoritative // check — kgoose's preflight count is advisory only. - let owned_count = owner_quota_used_in_transaction(&mut tx, &pubkey).await?; - - if owned_count >= max_communities_per_owner() { + if !owner_quota_in_transaction(&mut tx, &pubkey).await?.admits() { tx.rollback().await?; return Ok(TransferResult::LimitReached); } @@ -1272,6 +1303,17 @@ mod postgres_tests { ); } + #[test] + fn owner_quota_admits_only_under_active_and_lifetime_caps() { + let quota = |active, lifetime| super::OwnerQuota { active, lifetime }; + let limit = super::max_communities_per_owner(); + let lifetime = super::MAX_LIFETIME_COMMUNITIES_PER_OWNER.max(limit); + assert!(quota(0, 0).admits()); + assert!(quota(limit - 1, lifetime - 1).admits()); + assert!(!quota(limit, limit).admits(), "active cap"); + assert!(!quota(0, lifetime).admits(), "tombstones count toward lifetime"); + } + #[test] fn owner_limit_honors_positive_override() { assert_eq!(super::effective_owner_limit(Some("100")), 100); diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index 39d83974a65..a2dcaeed931 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -338,8 +338,6 @@ pub struct DeleteCommunityRequest { acknowledgement_version: i32, } -const DELETE_RECEIPT_PATH: &str = "/operator/communities/delete/receipt"; - /// Idempotently archive a community owned by the asserted end-user identity. pub async fn archive_community( State(state): State>, @@ -469,6 +467,12 @@ pub async fn unarchive_community( /// PostgreSQL-only transaction and returns `202`; inventory, approval, /// quiescing, object-store access, and executor work remain asynchronous. /// +/// Resubmitting the same UUID with the same host, owner, and acknowledgement +/// version returns `202` with that request's current `status`, at any stage and +/// even after membership purge, and never admits new work. Callers recover an +/// ambiguous submission by resending it; a different tuple under a known UUID +/// is `409 deletion_request_conflict`. +/// /// Owner consent is asserted by the operator, not proven to the relay. The /// operator authenticates the owner and collects the acknowledgement upstream; /// this request carries only the operator's NIP-98 signature. Authorization is @@ -577,67 +581,6 @@ pub async fn delete_community( )) } -/// Read one durable owner-deletion receipt without admitting or mutating work. -pub async fn delete_community_receipt( - State(state): State>, - headers: HeaderMap, - body: axum::body::Bytes, -) -> Result, (StatusCode, Json)> { - authorize_operator_request( - &state, - &headers, - "POST", - DELETE_RECEIPT_PATH, - None, - Some(&body), - ) - .await?; - let request: DeleteCommunityRequest = serde_json::from_slice(&body).map_err(|e| { - deletion_api_error( - StatusCode::BAD_REQUEST, - "invalid_request", - &format!("invalid delete-receipt JSON: {e}"), - ) - })?; - let owner = validate_pubkey_hex(&request.owner_pubkey).ok_or_else(|| { - deletion_api_error( - StatusCode::BAD_REQUEST, - "invalid_request", - "invalid owner_pubkey: expected 64-char hex pubkey", - ) - })?; - let receipt = state.db.deletion_store().get(request.request_id).await; - let receipt = match receipt { - Ok(receipt) - if receipt.request_origin == buzz_db::deletion::DeletionRequestOrigin::Owner - && receipt.owner_pubkey.as_deref() == Some(owner.as_str()) - && receipt.community_host == request.host - && receipt.acknowledgement_version == Some(request.acknowledgement_version) => - { - receipt - } - Ok(_) | Err(buzz_db::DbError::NotFound(_)) => { - return Err(deletion_api_error( - StatusCode::NOT_FOUND, - "deletion_receipt_not_found", - "deletion receipt not found", - )); - } - Err(error) => { - return Err(internal_error(&format!( - "read owner deletion receipt: {error}" - ))); - } - }; - Ok(Json(serde_json::json!({ - "request_id": receipt.id, - "community_id": receipt.community_id.to_string(), - "host": receipt.community_host, - "acknowledgement_version": receipt.acknowledgement_version, - "status": receipt.stage.to_string(), - }))) -} - /// List communities where a pubkey currently holds the `owner` role. pub async fn list_owned_communities( State(state): State>, @@ -1263,35 +1206,21 @@ mod postgres_tests { } let request_id = Uuid::new_v4(); - let body = owner_delete_body(&host, &owner, request_id); let accepted = signed_operator_request( Arc::clone(&state), &operator, "POST", "/operator/communities/delete", - Some(body.clone()), + Some(owner_delete_body(&host, &owner, request_id)), ) .await; assert_eq!(accepted.status(), StatusCode::ACCEPTED); assert_eq!(read_json(accepted).await["host"], host); - - let receipt = signed_operator_request( - Arc::clone(&state), - &operator, - "POST", - "/operator/communities/delete/receipt", - Some(body), - ) - .await; - assert_eq!(receipt.status(), StatusCode::OK); - let receipt = read_json(receipt).await; - assert_eq!(receipt["host"], host); - assert_eq!(receipt["request_id"], request_id.to_string()); } #[tokio::test] #[ignore = "requires Postgres"] - async fn owner_delete_receipt_is_bound_read_only_and_reports_aborted() { + async fn owner_delete_resubmission_reports_current_status_without_new_intent() { let operator = Keys::generate(); let outsider = Keys::generate(); let owner = Keys::generate(); @@ -1325,25 +1254,43 @@ mod postgres_tests { .deletion_store() .list(1_000) .await - .expect("list receipts before reads") + .expect("list requests before replays") .into_iter() .filter(|request| request.id == request_id) .count(); - let receipt = signed_operator_request( + let replay = signed_operator_request( Arc::clone(&state), &operator, "POST", - "/operator/communities/delete/receipt", + "/operator/communities/delete", Some(body.clone()), ) .await; - assert_eq!(receipt.status(), StatusCode::OK); - let receipt = read_json(receipt).await; - assert_eq!(receipt["request_id"], request_id.to_string()); - assert_eq!(receipt["host"], host); - assert_eq!(receipt["acknowledgement_version"], 1); - assert_eq!(receipt["status"], "submitted"); + assert_eq!(replay.status(), StatusCode::ACCEPTED); + let replay = read_json(replay).await; + assert_eq!(replay["request_id"], request_id.to_string()); + assert_eq!(replay["host"], host); + assert_eq!(replay["acknowledgement_version"], 1); + assert_eq!(replay["status"], "submitted"); + + // Recovery must survive membership purge: the replay converges on the + // stored tuple before any owner or archive check runs. + sqlx::query("DELETE FROM relay_members WHERE pubkey = $1 AND role = 'owner'") + .bind(owner.public_key().to_hex()) + .execute(state.db.pool()) + .await + .expect("simulate membership purge"); + let purged_replay = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some(body.clone()), + ) + .await; + assert_eq!(purged_replay.status(), StatusCode::ACCEPTED); + assert_eq!(read_json(purged_replay).await["status"], "submitted"); for mismatch in [ serde_json::json!({ @@ -1369,14 +1316,14 @@ mod postgres_tests { Arc::clone(&state), &operator, "POST", - "/operator/communities/delete/receipt", + "/operator/communities/delete", Some(mismatch.to_string()), ) .await; - assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!(response.status(), StatusCode::CONFLICT); assert_eq!( read_json(response).await["code"], - "deletion_receipt_not_found" + "deletion_request_conflict" ); } @@ -1384,7 +1331,7 @@ mod postgres_tests { Arc::clone(&state), &outsider, "POST", - "/operator/communities/delete/receipt", + "/operator/communities/delete", Some(body.clone()), ) .await; @@ -1395,16 +1342,16 @@ mod postgres_tests { .deletion_store() .abort(request_id, &operator.public_key().to_hex(), "test abort") .await - .expect("abort receipt"); + .expect("abort request"); let aborted = signed_operator_request( Arc::clone(&state), &operator, "POST", - "/operator/communities/delete/receipt", + "/operator/communities/delete", Some(body), ) .await; - assert_eq!(aborted.status(), StatusCode::OK); + assert_eq!(aborted.status(), StatusCode::ACCEPTED); assert_eq!(read_json(aborted).await["status"], "aborted"); let after = state @@ -1412,11 +1359,11 @@ mod postgres_tests { .deletion_store() .list(1_000) .await - .expect("list receipts after reads") + .expect("list requests after replays") .into_iter() .filter(|request| request.id == request_id) .count(); - assert_eq!(before, after, "receipt lookups must not add request rows"); + assert_eq!(before, after, "replays must not add request rows"); } #[tokio::test] diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index 231e38eb761..a4a812fa491 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -334,10 +334,6 @@ pub fn build_router(state: Arc) -> Router { "/operator/communities/delete", post(api::operator::delete_community), ) - .route( - "/operator/communities/delete/receipt", - post(api::operator::delete_community_receipt), - ) .route( "/operator/communities/availability", get(api::operator::community_availability), diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index 5a6dc4cde57..0c0c57afd2c 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -151,22 +151,31 @@ eligible for execution. Transient preparation failures use the existing retry schedule; permanent or exhausted failures block durably. Owner-facing admission has no cancellation endpoint. -## Client compatibility and rollout order - -Owner-list responses now carry the authoritative quota trio: `quota_used`, -`quota_limit`, and `can_create`. Quota-dependent clients fail closed when any -member of that trio is absent: they disable new Create controls even when the -separate community-deletion capability is absent or false. They do not derive -capacity from the number of visible rows, because an in-progress deletion can -hide a row while still reserving its owner's slot. - -Roll this contract out in dependency order: relay first, then KGoose, then -Desktop and any other quota-dependent clients. This ordering is a compatibility -requirement, not authorization to enable owner deletion or its drain job. -During that rollout, keep the already-published migration 0052 channel-artifact -surface and migration 0053 owner auto-approval surface byte-for-byte intact; -do not rewrite their migration ledger entries or recreate a persistent -database to introduce the quota projection. +Admission is idempotent on the request UUID. Resending the same UUID with the +same host, owner, and acknowledgement version returns `202` with that request's +current `status` at any stage, including after membership purge, and admits no +new work. Clients recover an ambiguous submission by resending it. The same +UUID with a different tuple returns `409 deletion_request_conflict`. + +## Owner quota + +The relay enforces two per-owner caps on create and on transfer-in, both as +`limit_reached`: + +- **Active:** live ownership plus incomplete owner deletions + (`BUZZ_MAX_COMMUNITIES_PER_OWNER`, default 5). A deletion keeps its slot + until logical completion records `completed_at`. +- **Lifetime:** live ownership plus every non-aborted owner deletion, including + completed ones, capped at 20 (or the active limit if that is higher). + Deleted communities keep their hosts as permanent tombstones, so this bounds + create-then-delete host squatting. Aborted deletions restore the community + and count only through its live membership. + +Owner-list responses carry `quota_used` (active), `quota_limit` (active), and +`can_create` (both caps). The projection is advisory: clients may use it for +UX, but the relay's `limit_reached` is authoritative, and quota changes have no +deployment order. Keep owner deletion off until this relay and the drain +executor are live; on rollback, turn deletion off before rolling back the relay. The chart has no existing PrometheusRule or provider-neutral CronJob alert integration. Operators must alert on failed/missed Jobs and long-running active diff --git a/migrations/0054_owner_deletion_quota_reservation.sql b/migrations/0054_owner_deletion_quota_reservation.sql index 32331e4ae1d..c1487983eb2 100644 --- a/migrations/0054_owner_deletion_quota_reservation.sql +++ b/migrations/0054_owner_deletion_quota_reservation.sql @@ -1,10 +1,10 @@ --- Keep owner quota reservation lookups bounded after relay membership is --- purged but logical deletion has not yet completed. +-- Keep owner quota lookups bounded: incomplete owner deletions reserve an +-- active slot, and every non-aborted owner deletion counts toward the +-- lifetime cap after relay membership is purged. SET LOCAL lock_timeout = '5s'; CREATE INDEX community_deletion_requests_owner_quota_reservations ON community_deletion_requests (owner_pubkey) - INCLUDE (community_id) + INCLUDE (community_id, completed_at) WHERE request_origin = 'owner' - AND stage <> 'aborted' - AND completed_at IS NULL; + AND stage <> 'aborted'; diff --git a/schema/schema.sql b/schema/schema.sql index f7f0ff2eaea..3bf94907a89 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1311,10 +1311,9 @@ CREATE INDEX community_deletion_requests_owner_preparable AND blocked_at IS NULL; CREATE INDEX community_deletion_requests_owner_quota_reservations ON community_deletion_requests (owner_pubkey) - INCLUDE (community_id) + INCLUDE (community_id, completed_at) WHERE request_origin = 'owner' - AND stage <> 'aborted' - AND completed_at IS NULL; + AND stage <> 'aborted'; CREATE TABLE community_deletion_approvals ( request_id UUID PRIMARY KEY, From c35c74fb18e3949424fc390b8391e4d360cedcbf Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Tue, 29 Sep 2026 17:28:24 -0400 Subject: [PATCH 42/46] fix(db): make the lifetime owner cap absolute and rustfmt the quota test Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/src/store/relay_members.rs | 14 +++++++++----- docs/operator-community-deletion.md | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index ba0651475c2..8f2bcd191e6 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -606,7 +606,8 @@ pub fn owner_count_advisory_lock_key(pubkey_hex: &str) -> i64 { /// Lifetime cap on communities a pubkey may own, counting owner-deleted /// communities whose tombstones permanently retain their hosts. Bounds -/// create-then-delete host squatting; never below the active limit. +/// create-then-delete host squatting. Absolute: it does not scale with +/// `BUZZ_MAX_COMMUNITIES_PER_OWNER`. pub const MAX_LIFETIME_COMMUNITIES_PER_OWNER: i64 = 20; /// One owner's quota usage, read inside the admitting transaction. @@ -621,8 +622,8 @@ pub(crate) struct OwnerQuota { impl OwnerQuota { /// Whether this owner may gain one more community. pub fn admits(self) -> bool { - let limit = max_communities_per_owner(); - self.active < limit && self.lifetime < MAX_LIFETIME_COMMUNITIES_PER_OWNER.max(limit) + self.active < max_communities_per_owner() + && self.lifetime < MAX_LIFETIME_COMMUNITIES_PER_OWNER } } @@ -1307,11 +1308,14 @@ mod postgres_tests { fn owner_quota_admits_only_under_active_and_lifetime_caps() { let quota = |active, lifetime| super::OwnerQuota { active, lifetime }; let limit = super::max_communities_per_owner(); - let lifetime = super::MAX_LIFETIME_COMMUNITIES_PER_OWNER.max(limit); + let lifetime = super::MAX_LIFETIME_COMMUNITIES_PER_OWNER; assert!(quota(0, 0).admits()); assert!(quota(limit - 1, lifetime - 1).admits()); assert!(!quota(limit, limit).admits(), "active cap"); - assert!(!quota(0, lifetime).admits(), "tombstones count toward lifetime"); + assert!( + !quota(0, lifetime).admits(), + "tombstones count toward lifetime" + ); } #[test] diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index 0c0c57afd2c..0b2c6dbe5ce 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -166,7 +166,7 @@ The relay enforces two per-owner caps on create and on transfer-in, both as (`BUZZ_MAX_COMMUNITIES_PER_OWNER`, default 5). A deletion keeps its slot until logical completion records `completed_at`. - **Lifetime:** live ownership plus every non-aborted owner deletion, including - completed ones, capped at 20 (or the active limit if that is higher). + completed ones, capped at an absolute 20 regardless of the active limit. Deleted communities keep their hosts as permanent tombstones, so this bounds create-then-delete host squatting. Aborted deletions restore the community and count only through its live membership. From 63623fee55608b6761b16f0a536c185261e6a7dd Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Tue, 29 Sep 2026 17:32:04 -0400 Subject: [PATCH 43/46] docs(relay): state limit_reached for both owner caps and pin the can_create contract Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-db/src/store/community.rs | 4 ++-- crates/buzz-db/src/store/relay_members.rs | 6 +++--- crates/buzz-relay/src/api/operator.rs | 2 +- crates/buzz-relay/src/handlers/community_provisioning.rs | 5 +---- docs/operator-community-deletion.md | 5 ++++- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index 46db43b6730..cf4597662fd 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -44,7 +44,7 @@ pub enum CreateCommunityWithOwnerResult { Created(CreatedCommunityRecord), /// The host already belongs to another owner. HostExists, - /// The intended owner already owns the maximum number of communities. + /// The intended owner has reached the active or lifetime community limit. LimitReached, } @@ -68,7 +68,7 @@ pub struct OwnedCommunitiesPage { pub communities: Vec, /// De-duplicated live memberships and incomplete owner deletion reservations. pub quota_used: i64, - /// Configured maximum hosted communities for one owner. + /// Configured active limit only; the lifetime cap is reflected in `can_create`. pub quota_limit: i64, /// Whether the snapshot leaves room under both the active and lifetime caps. pub can_create: bool, diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 8f2bcd191e6..f3ce399dd7e 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -544,7 +544,7 @@ pub enum TransferResult { LifecycleConflict, /// Durable deletion intent exists and wins over ownership mutation. DeletionPending, - /// The transferee already owns the maximum number of communities. + /// The transferee has reached the active or lifetime community limit. /// Enforced atomically inside the transfer transaction so concurrent /// transfers to the same recipient cannot both pass the limit. LimitReached, @@ -684,8 +684,8 @@ pub(crate) async fn owner_quota_in_transaction( /// 3. Locks the current owner row `FOR UPDATE` and verifies /// `expected_owner_pubkey` matches. This prevents a stale-owner race where /// a delayed/retried request overwrites a completed transfer. -/// 4. Enforces the [`MAX_COMMUNITIES_PER_OWNER`] limit on the transferee by -/// counting owned communities inside the same transaction. +/// 4. Enforces the transferee's active and lifetime limits +/// ([`OwnerQuota::admits`]) inside the same transaction. /// 5. Upserts `new_owner_pubkey` as `owner` (insert or promote). /// 6. Demotes every other owner in this community to `member` — **not** /// `admin`, per product decision: the former owner retains no management diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index a2dcaeed931..52db37c06cf 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -722,7 +722,7 @@ pub async fn transfer_community( buzz_db::relay_members::TransferResult::LimitReached => { return Err(api_error( StatusCode::CONFLICT, - "limit_reached: transferee already owns the maximum number of communities", + "limit_reached: transferee has reached the community limit", )); } }; diff --git a/crates/buzz-relay/src/handlers/community_provisioning.rs b/crates/buzz-relay/src/handlers/community_provisioning.rs index 1b13e6d921a..e8a5ee90d10 100644 --- a/crates/buzz-relay/src/handlers/community_provisioning.rs +++ b/crates/buzz-relay/src/handlers/community_provisioning.rs @@ -287,10 +287,7 @@ pub async fn provision_community( return Err("community already exists".to_string()); } buzz_db::CreateCommunityWithOwnerResult::LimitReached => { - return Err( - "limit_reached: owner already owns the maximum number of communities" - .to_string(), - ); + return Err("limit_reached: owner has reached the community limit".to_string()); } }; diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index 0b2c6dbe5ce..d54093d991e 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -172,7 +172,10 @@ The relay enforces two per-owner caps on create and on transfer-in, both as and count only through its live membership. Owner-list responses carry `quota_used` (active), `quota_limit` (active), and -`can_create` (both caps). The projection is advisory: clients may use it for +`can_create` (both caps). `can_create: false` is the only signal a client needs: +show a generic community-limit message and do not derive a reason from the +counts, because an owner at the lifetime cap can have `quota_used` below +`quota_limit`. The projection is advisory: clients may use it for UX, but the relay's `limit_reached` is authoritative, and quota changes have no deployment order. Keep owner deletion off until this relay and the drain executor are live; on rollback, turn deletion off before rolling back the relay. From 59375c0044a8bae31a0b5ab6cc7207d7000d1245 Mon Sep 17 00:00:00 2001 From: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Date: Tue, 29 Sep 2026 17:37:03 -0400 Subject: [PATCH 44/46] test(relay): pin that a changed acknowledgement version replay is unsupported, not a conflict Signed-off-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> --- crates/buzz-relay/src/api/operator.rs | 30 +++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/crates/buzz-relay/src/api/operator.rs b/crates/buzz-relay/src/api/operator.rs index 52db37c06cf..062a08cd43c 100644 --- a/crates/buzz-relay/src/api/operator.rs +++ b/crates/buzz-relay/src/api/operator.rs @@ -1305,12 +1305,6 @@ mod postgres_tests { "request_id": request_id, "acknowledgement_version": 1, }), - serde_json::json!({ - "host": host, - "owner_pubkey": owner.public_key().to_hex(), - "request_id": request_id, - "acknowledgement_version": 2, - }), ] { let response = signed_operator_request( Arc::clone(&state), @@ -1327,6 +1321,30 @@ mod postgres_tests { ); } + // Version validation precedes the UUID lookup, so a changed version is + // rejected as unsupported rather than as a request conflict. + let changed_version = signed_operator_request( + Arc::clone(&state), + &operator, + "POST", + "/operator/communities/delete", + Some( + serde_json::json!({ + "host": host, + "owner_pubkey": owner.public_key().to_hex(), + "request_id": request_id, + "acknowledgement_version": 2, + }) + .to_string(), + ), + ) + .await; + assert_eq!(changed_version.status(), StatusCode::BAD_REQUEST); + assert_eq!( + read_json(changed_version).await["code"], + "unsupported_acknowledgement_version" + ); + let outsider_response = signed_operator_request( Arc::clone(&state), &outsider, From bb59f0ef27719030d979306687d7f0d6c66ba4a0 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Tue, 29 Sep 2026 18:13:19 -0400 Subject: [PATCH 45/46] docs(relay): note ack-version dependency of resend recovery Signed-off-by: OpenAI Codex --- docs/operator-community-deletion.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index d54093d991e..9ddb5dc21d8 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -157,6 +157,12 @@ current `status` at any stage, including after membership purge, and admits no new work. Clients recover an ambiguous submission by resending it. The same UUID with a different tuple returns `409 deletion_request_conflict`. +The acknowledgement version is validated before the UUID lookup, so resend +recovery only works while the relay still accepts the version the request was +made under. Retiring a version makes resends of pending requests created under +it fail with `400 unsupported_acknowledgement_version`; keep the old version +accepted for replay until no client can hold a pending request made with it. + ## Owner quota The relay enforces two per-owner caps on create and on transfer-in, both as From e21151f47df20f396083702333bdf62179ef8cb4 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Tue, 29 Sep 2026 18:16:18 -0400 Subject: [PATCH 46/46] docs(relay): ack version is a code constant that also gates execution Signed-off-by: OpenAI Codex --- docs/operator-community-deletion.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/operator-community-deletion.md b/docs/operator-community-deletion.md index 9ddb5dc21d8..d4689006d58 100644 --- a/docs/operator-community-deletion.md +++ b/docs/operator-community-deletion.md @@ -157,11 +157,15 @@ current `status` at any stage, including after membership purge, and admits no new work. Clients recover an ambiguous submission by resending it. The same UUID with a different tuple returns `409 deletion_request_conflict`. -The acknowledgement version is validated before the UUID lookup, so resend -recovery only works while the relay still accepts the version the request was -made under. Retiring a version makes resends of pending requests created under -it fail with `400 unsupported_acknowledgement_version`; keep the old version -accepted for replay until no client can hold a pending request made with it. +The acknowledgement version is a compile-time constant +(`OWNER_DELETION_ACKNOWLEDGEMENT_VERSION`), not operator configuration. +Admission validates it before the UUID lookup, and the executor claims and +leases only requests carrying the current version. Raising it therefore makes +resends of pending requests made under the old version fail with +`400 unsupported_acknowledgement_version`, and leaves already-admitted +old-version requests unclaimed in `submitted`. A version bump must ship code +that keeps admitting replays of, and executing, requests at the prior version +until none remain in a non-terminal stage. ## Owner quota