diff --git a/database/migrations/0026_workload_provider_budget.sql b/database/migrations/0026_workload_provider_budget.sql new file mode 100644 index 00000000..5bab2e29 --- /dev/null +++ b/database/migrations/0026_workload_provider_budget.sql @@ -0,0 +1,475 @@ +ALTER TABLE agentos.provider_budget_reservations + ALTER COLUMN binding_id DROP NOT NULL, + ALTER COLUMN subject DROP NOT NULL, + ADD COLUMN workload_principal jsonb, + ADD COLUMN model text, + ADD COLUMN effective_limits jsonb, + ADD COLUMN effective_pricing jsonb, + ADD COLUMN policy_expires_at_millis bigint, + ADD COLUMN reserved_tokens bigint CHECK (reserved_tokens > 0), + ADD COLUMN reserved_spend_micros bigint CHECK (reserved_spend_micros > 0), + ADD COLUMN token_window_started_at_millis bigint, + ADD COLUMN spend_window_started_at_millis bigint, + ADD COLUMN attempted_at_millis bigint CHECK (attempted_at_millis >= 0); + +ALTER TABLE agentos.provider_budget_reservations + DROP CONSTRAINT provider_budget_reservations_subject_check, + ADD CONSTRAINT provider_budget_reservations_subject_check CHECK ( + subject IS NULL OR agentos.valid_access_subject(subject) + ); + +ALTER TABLE agentos.provider_budget_reservations + ADD CONSTRAINT provider_budget_reservation_authority_check CHECK ( + (binding_id IS NOT NULL AND subject IS NOT NULL + AND workload_principal IS NULL AND model IS NULL + AND effective_limits IS NULL AND effective_pricing IS NULL + AND policy_expires_at_millis IS NULL) + OR + (binding_id IS NULL AND subject IS NULL + AND workload_principal IS NOT NULL AND model IS NOT NULL + AND effective_limits IS NOT NULL AND effective_pricing IS NOT NULL + AND policy_expires_at_millis IS NOT NULL) + ); + +CREATE FUNCTION agentos.valid_workload_budget_principal(p_value jsonb) +RETURNS boolean LANGUAGE sql IMMUTABLE SET search_path = agentos, pg_temp AS $$ + SELECT jsonb_typeof(p_value) = 'object' + AND (SELECT count(*) FROM jsonb_object_keys(p_value)) = 9 + AND p_value ->> 'kind' = 'kubernetes_workload' + AND p_value ->> 'namespace' ~ '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + AND length(p_value ->> 'namespace') <= 63 + AND p_value ->> 'serviceAccountName' ~ '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + AND length(p_value ->> 'serviceAccountName') <= 63 + AND length(p_value ->> 'serviceAccountUid') BETWEEN 1 AND 128 + AND p_value ->> 'podName' ~ '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + AND length(p_value ->> 'podName') <= 63 + AND length(p_value ->> 'podUid') BETWEEN 1 AND 128 + AND jsonb_typeof(p_value -> 'policyRevision') = 'number' + AND (p_value ->> 'policyRevision')::bigint > 0 + AND p_value ->> 'policyResourceVersion' ~ '^[1-9][0-9]*$' + AND p_value ->> 'hermesProfile' ~ '^[a-z][a-z0-9._-]*$' + AND length(p_value ->> 'hermesProfile') <= 96 +$$; + +CREATE FUNCTION agentos.valid_workload_budget_limits(p_value jsonb) +RETURNS boolean LANGUAGE sql IMMUTABLE SET search_path = agentos, pg_temp AS $$ + SELECT jsonb_typeof(p_value) = 'object' + AND (SELECT count(*) FROM jsonb_object_keys(p_value)) = 7 + AND p_value ?& ARRAY[ + 'requestWindowMillis', 'maximumRequests', 'maximumConcurrent', + 'tokenWindowMillis', 'maximumTokens', 'spendWindowMillis', + 'maximumSpendMicros' + ] + AND NOT EXISTS ( + SELECT 1 FROM jsonb_each(p_value) AS item + WHERE jsonb_typeof(item.value) <> 'number' + OR item.value::text !~ '^[1-9][0-9]*$' + ) +$$; + +CREATE FUNCTION agentos.reserve_workload_provider_budget( + p_decision_ref text, p_budget_key text, p_correlation_id text, + p_principal jsonb, p_provider text, p_credential_domain text, + p_capability text, p_resource jsonb, p_environment text, p_model text, + p_rate_class text, p_limits jsonb, p_pricing jsonb, + p_policy_expires_at_millis bigint, + p_requested_tokens bigint, p_requested_spend_micros bigint, p_now_millis bigint +) +RETURNS TABLE ( + "outcome" text, "effectiveRateClass" text, "retryAtMillis" double precision, + "requestWindowEndsAtMillis" double precision, + "tokenWindowEndsAtMillis" double precision, + "spendWindowEndsAtMillis" double precision, + "leaseExpiresAtMillis" double precision +) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = agentos, pg_temp AS $$ +DECLARE + v_existing agentos.provider_budget_reservations%ROWTYPE; + v_request_start bigint; v_token_start bigint; v_spend_start bigint; + v_request_end bigint; v_token_end bigint; v_spend_end bigint; + v_request_consumed bigint; v_token_consumed bigint; v_spend_consumed bigint; + v_token_reserved bigint; v_spend_reserved bigint; + v_active_concurrent bigint; v_concurrency_retry bigint; v_retry bigint; + v_lease_expires bigint; +BEGIN + IF p_decision_ref !~ '^decision_[0-9a-f]{32}$' + OR p_budget_key !~ '^budget_[0-9a-f]{64}$' + OR p_correlation_id !~ '^corr_[0-9a-f]{32}$' + OR NOT agentos.valid_workload_budget_principal(p_principal) + OR p_provider <> 'openai' OR p_credential_domain <> 'openai-responses' + OR p_capability NOT IN ('openai.responses.create', 'openai.responses.compact') + OR p_resource <> jsonb_build_object('kind', 'provider_service', + 'provider', 'openai', 'service', 'responses') + OR p_environment IS NULL + OR p_environment !~ '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + OR length(p_environment) > 63 + OR p_model !~ '^[a-z0-9][a-z0-9._:-]*$' OR length(p_model) > 128 + OR p_rate_class NOT IN ('low', 'standard', 'high') + OR NOT agentos.valid_workload_budget_limits(p_limits) + OR jsonb_typeof(p_pricing) <> 'object' + OR (SELECT count(*) FROM jsonb_object_keys(p_pricing)) <> 3 + OR NOT p_pricing ?& ARRAY['version','inputMicrosPerMillionTokens','outputMicrosPerMillionTokens'] + OR EXISTS (SELECT 1 FROM jsonb_each(p_pricing) item + WHERE jsonb_typeof(item.value) <> 'number' + OR item.value::text !~ '^[1-9][0-9]*$') + OR p_requested_tokens IS NULL OR p_requested_tokens <= 0 + OR p_requested_tokens > (p_limits ->> 'maximumTokens')::bigint + OR p_requested_spend_micros IS NULL OR p_requested_spend_micros <= 0 + OR p_requested_spend_micros > (p_limits ->> 'maximumSpendMicros')::bigint + OR p_policy_expires_at_millis IS NULL OR p_now_millis IS NULL + OR p_now_millis < 0 OR p_policy_expires_at_millis <= p_now_millis THEN + RAISE EXCEPTION 'invalid workload provider budget reservation'; + END IF; + + SELECT reservation.* INTO v_existing + FROM agentos.provider_budget_reservations AS reservation + WHERE reservation.decision_ref = p_decision_ref FOR UPDATE; + IF FOUND THEN + IF v_existing.budget_key <> p_budget_key + OR v_existing.correlation_id <> p_correlation_id + OR v_existing.workload_principal <> p_principal + OR v_existing.provider <> p_provider + OR v_existing.credential_domain <> p_credential_domain + OR v_existing.capability <> p_capability + OR v_existing.resource <> p_resource + OR v_existing.environment IS DISTINCT FROM p_environment + OR v_existing.model <> p_model OR v_existing.rate_class <> p_rate_class + OR v_existing.effective_limits <> p_limits + OR v_existing.effective_pricing <> p_pricing + OR v_existing.reserved_tokens <> p_requested_tokens + OR v_existing.reserved_spend_micros <> p_requested_spend_micros + OR v_existing.policy_expires_at_millis <> p_policy_expires_at_millis THEN + RAISE EXCEPTION 'workload provider budget decision reference conflicts'; + END IF; + IF v_existing.state <> 'active' OR v_existing.attempted_at_millis IS NOT NULL + OR v_existing.lease_expires_at_millis <= p_now_millis + OR v_existing.policy_expires_at_millis <= p_now_millis THEN + RAISE EXCEPTION 'workload provider budget reservation is not active'; + END IF; + RETURN QUERY SELECT 'reserved'::text, v_existing.rate_class, + NULL::double precision, + (v_existing.reserved_at_millis - mod(v_existing.reserved_at_millis, + (p_limits ->> 'requestWindowMillis')::bigint) + + (p_limits ->> 'requestWindowMillis')::bigint)::double precision, + (v_existing.reserved_at_millis - mod(v_existing.reserved_at_millis, + (p_limits ->> 'tokenWindowMillis')::bigint) + + (p_limits ->> 'tokenWindowMillis')::bigint)::double precision, + (v_existing.reserved_at_millis - mod(v_existing.reserved_at_millis, + (p_limits ->> 'spendWindowMillis')::bigint) + + (p_limits ->> 'spendWindowMillis')::bigint)::double precision, + v_existing.lease_expires_at_millis::double precision; + RETURN; + END IF; + + v_request_start := p_now_millis - mod(p_now_millis, + (p_limits ->> 'requestWindowMillis')::bigint); + v_token_start := p_now_millis - mod(p_now_millis, + (p_limits ->> 'tokenWindowMillis')::bigint); + v_spend_start := p_now_millis - mod(p_now_millis, + (p_limits ->> 'spendWindowMillis')::bigint); + v_request_end := v_request_start + (p_limits ->> 'requestWindowMillis')::bigint; + v_token_end := v_token_start + (p_limits ->> 'tokenWindowMillis')::bigint; + v_spend_end := v_spend_start + (p_limits ->> 'spendWindowMillis')::bigint; + v_lease_expires := least(p_policy_expires_at_millis, p_now_millis + 900000); + + INSERT INTO agentos.provider_budget_counters + (budget_key, dimension, window_started_at_millis, window_ends_at_millis, + consumed, rate_class) + VALUES + (p_budget_key, 'request', v_request_start, v_request_end, 0, p_rate_class), + (p_budget_key, 'token', v_token_start, v_token_end, 0, p_rate_class), + (p_budget_key, 'spend', v_spend_start, v_spend_end, 0, p_rate_class) + ON CONFLICT (budget_key, dimension, window_started_at_millis) DO NOTHING; + SELECT consumed INTO v_request_consumed FROM agentos.provider_budget_counters + WHERE budget_key = p_budget_key AND dimension = 'request' + AND window_started_at_millis = v_request_start FOR UPDATE; + SELECT consumed INTO v_token_consumed FROM agentos.provider_budget_counters + WHERE budget_key = p_budget_key AND dimension = 'token' + AND window_started_at_millis = v_token_start FOR UPDATE; + SELECT consumed INTO v_spend_consumed FROM agentos.provider_budget_counters + WHERE budget_key = p_budget_key AND dimension = 'spend' + AND window_started_at_millis = v_spend_start FOR UPDATE; + + SELECT coalesce(sum(reserved_tokens), 0) INTO v_token_reserved + FROM agentos.provider_budget_reservations + WHERE budget_key = p_budget_key AND state = 'active' + AND (attempted_at_millis IS NOT NULL OR lease_expires_at_millis > p_now_millis) + AND token_window_started_at_millis = v_token_start; + SELECT coalesce(sum(reserved_spend_micros), 0) INTO v_spend_reserved + FROM agentos.provider_budget_reservations + WHERE budget_key = p_budget_key AND state = 'active' + AND (attempted_at_millis IS NOT NULL OR lease_expires_at_millis > p_now_millis) + AND spend_window_started_at_millis = v_spend_start; + + IF v_token_consumed + v_token_reserved + p_requested_tokens > + (p_limits ->> 'maximumTokens')::bigint + OR v_spend_consumed + v_spend_reserved + p_requested_spend_micros > + (p_limits ->> 'maximumSpendMicros')::bigint THEN + v_retry := greatest( + CASE WHEN v_token_consumed + v_token_reserved > 0 + THEN v_token_end ELSE 0 END, + CASE WHEN v_spend_consumed + v_spend_reserved > 0 + THEN v_spend_end ELSE 0 END); + RETURN QUERY SELECT 'budget_exhausted'::text, p_rate_class, + v_retry::double precision, v_request_end::double precision, + v_token_end::double precision, v_spend_end::double precision, + NULL::double precision; + RETURN; + END IF; + SELECT count(*)::bigint, min(lease_expires_at_millis) + INTO v_active_concurrent, v_concurrency_retry + FROM agentos.provider_budget_reservations + WHERE budget_key = p_budget_key AND state = 'active' + AND (attempted_at_millis IS NOT NULL OR lease_expires_at_millis > p_now_millis); + IF v_request_consumed >= (p_limits ->> 'maximumRequests')::bigint + OR v_active_concurrent >= (p_limits ->> 'maximumConcurrent')::bigint THEN + v_retry := greatest( + CASE WHEN v_request_consumed >= (p_limits ->> 'maximumRequests')::bigint + THEN v_request_end ELSE p_now_millis END, + CASE WHEN v_active_concurrent >= (p_limits ->> 'maximumConcurrent')::bigint + THEN coalesce(v_concurrency_retry, p_now_millis) ELSE p_now_millis END); + RETURN QUERY SELECT 'rate_limited'::text, p_rate_class, + v_retry::double precision, v_request_end::double precision, + v_token_end::double precision, v_spend_end::double precision, + NULL::double precision; + RETURN; + END IF; + + UPDATE agentos.provider_budget_counters SET consumed = consumed + 1, + updated_at = transaction_timestamp() + WHERE budget_key = p_budget_key AND dimension = 'request' + AND window_started_at_millis = v_request_start; + INSERT INTO agentos.provider_budget_reservations ( + decision_ref, budget_key, workload_principal, provider, credential_domain, + capability, resource, environment, model, rate_class, effective_limits, + effective_pricing, + policy_expires_at_millis, correlation_id, reserved_at_millis, + lease_expires_at_millis, reserved_tokens, reserved_spend_micros, + token_window_started_at_millis, spend_window_started_at_millis + ) VALUES ( + p_decision_ref, p_budget_key, p_principal, p_provider, p_credential_domain, + p_capability, p_resource, p_environment, p_model, p_rate_class, p_limits, + p_pricing, + p_policy_expires_at_millis, p_correlation_id, p_now_millis, v_lease_expires, + p_requested_tokens, p_requested_spend_micros, v_token_start, v_spend_start + ); + RETURN QUERY SELECT 'reserved'::text, p_rate_class, NULL::double precision, + v_request_end::double precision, v_token_end::double precision, + v_spend_end::double precision, v_lease_expires::double precision; +END; +$$; + +REVOKE ALL ON FUNCTION agentos.valid_workload_budget_principal(jsonb) FROM PUBLIC; +REVOKE ALL ON FUNCTION agentos.valid_workload_budget_limits(jsonb) FROM PUBLIC; +REVOKE ALL ON FUNCTION agentos.reserve_workload_provider_budget( + text,text,text,jsonb,text,text,text,jsonb,text,text,text,jsonb,jsonb,bigint,bigint,bigint,bigint +) FROM PUBLIC; + +CREATE FUNCTION agentos.validate_workload_provider_budget( + p_decision_ref text, p_correlation_id text, p_principal jsonb, + p_provider text, p_credential_domain text, p_capability text, + p_resource jsonb, p_model text, p_rate_class text, p_limits jsonb, + p_pricing jsonb, p_requested_tokens bigint, p_requested_spend_micros bigint, + p_expires_at_millis bigint, p_now_millis bigint +) +RETURNS TABLE ("outcome" text) +LANGUAGE plpgsql SECURITY DEFINER +SET search_path = agentos, pg_temp AS $$ +DECLARE v_count integer; +BEGIN + IF p_decision_ref !~ '^decision_[0-9a-f]{32}$' + OR p_correlation_id !~ '^corr_[0-9a-f]{32}$' + OR jsonb_typeof(p_principal) <> 'object' + OR (SELECT count(*) FROM jsonb_object_keys(p_principal)) <> 6 + OR p_provider <> 'openai' OR p_credential_domain <> 'openai-responses' + OR p_capability NOT IN ('openai.responses.create', 'openai.responses.compact') + OR NOT agentos.valid_workload_budget_limits(p_limits) + OR jsonb_typeof(p_pricing) <> 'object' + OR p_requested_tokens <= 0 OR p_requested_spend_micros <= 0 + OR p_now_millis IS NULL OR p_now_millis < 0 THEN + RAISE EXCEPTION 'invalid workload provider budget validation'; + END IF; + SELECT count(*)::integer INTO v_count + FROM agentos.provider_budget_reservations AS reservation + WHERE reservation.decision_ref = p_decision_ref + AND reservation.correlation_id = p_correlation_id + AND reservation.workload_principal + - 'serviceAccountUid' - 'podName' - 'podUid' = p_principal + AND reservation.provider = p_provider + AND reservation.credential_domain = p_credential_domain + AND reservation.capability = p_capability + AND reservation.resource = p_resource + AND reservation.model = p_model + AND reservation.rate_class = p_rate_class + AND reservation.effective_limits = p_limits + AND reservation.effective_pricing = p_pricing + AND reservation.reserved_tokens = p_requested_tokens + AND reservation.reserved_spend_micros = p_requested_spend_micros + AND reservation.policy_expires_at_millis = p_expires_at_millis + AND reservation.state = 'active' + AND reservation.attempted_at_millis IS NULL + AND reservation.lease_expires_at_millis > p_now_millis + AND reservation.policy_expires_at_millis > p_now_millis; + IF v_count <> 1 THEN + RAISE EXCEPTION 'workload provider budget reservation unavailable'; + END IF; + UPDATE agentos.provider_budget_reservations + SET attempted_at_millis = p_now_millis, updated_at = transaction_timestamp() + WHERE decision_ref = p_decision_ref AND state = 'active' + AND attempted_at_millis IS NULL; + GET DIAGNOSTICS v_count = ROW_COUNT; + IF v_count <> 1 THEN + RAISE EXCEPTION 'workload provider budget attempt already claimed'; + END IF; + RETURN QUERY SELECT 'attempted'::text; +END; +$$; + +REVOKE ALL ON FUNCTION agentos.validate_workload_provider_budget( + text,text,jsonb,text,text,text,jsonb,text,text,jsonb,jsonb,bigint,bigint,bigint,bigint +) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION agentos.configure_egress_authorizer_privileges( + p_database_role name +) +RETURNS void LANGUAGE plpgsql SECURITY DEFINER +SET search_path = agentos, pg_temp AS $$ +DECLARE v_role oid; v_dangerous boolean; +BEGIN + IF session_user <> current_user THEN + RAISE EXCEPTION 'egress authorizer privileges require the schema owner'; + END IF; + SELECT oid, rolsuper OR rolcreaterole OR rolcreatedb OR rolreplication + OR rolbypassrls INTO v_role, v_dangerous + FROM pg_catalog.pg_roles WHERE rolname = p_database_role; + IF v_role IS NULL OR v_dangerous THEN + RAISE EXCEPTION 'egress authorizer database role is unavailable or privileged'; + END IF; + IF EXISTS (SELECT 1 FROM pg_catalog.pg_auth_members WHERE member = v_role) THEN + RAISE EXCEPTION 'egress authorizer database role must not inherit roles'; + END IF; + EXECUTE format('REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA agentos FROM %I', p_database_role); + EXECUTE format('REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA agentos FROM %I', p_database_role); + EXECUTE format('REVOKE ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA agentos FROM %I', p_database_role); + EXECUTE format('GRANT USAGE ON SCHEMA agentos TO %I', p_database_role); + EXECUTE format( + 'GRANT EXECUTE ON FUNCTION agentos.read_egress_workload_agents(text,text), agentos.read_egress_assignments(uuid), agentos.read_egress_policy_snapshots(jsonb), agentos.reserve_workload_provider_budget(text,text,text,jsonb,text,text,text,jsonb,text,text,text,jsonb,jsonb,bigint,bigint,bigint,bigint), agentos.validate_workload_provider_budget(text,text,jsonb,text,text,text,jsonb,text,text,jsonb,jsonb,bigint,bigint,bigint,bigint), agentos.settle_provider_budget_for_provider(text,text,text,text,bigint,bigint,bigint,bigint,bigint) TO %I', + p_database_role); +END; +$$; + +CREATE OR REPLACE FUNCTION agentos.settle_provider_budget_for_provider( + p_decision_ref text, p_provider text, p_credential_domain text, + p_forward_outcome text, p_input_tokens bigint, p_output_tokens bigint, + p_cached_input_tokens bigint, p_spend_micros bigint, + p_settled_at_millis bigint +) +RETURNS TABLE ( + "outcome" text, "forwardOutcome" text, "inputTokens" double precision, + "outputTokens" double precision, "cachedInputTokens" double precision, + "spendMicros" double precision, "settledAtMillis" double precision +) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = agentos, pg_temp AS $$ +DECLARE + v_reservation agentos.provider_budget_reservations%ROWTYPE; + v_token_start bigint; v_spend_start bigint; + v_calculated_spend bigint; +BEGIN + IF p_decision_ref !~ '^decision_[0-9a-f]{32}$' + OR p_provider NOT IN ('github', 'openai') + OR p_credential_domain !~ '^[a-z0-9]([-a-z0-9]*[a-z0-9])?$' + OR length(p_credential_domain) > 63 + OR p_forward_outcome NOT IN + ('completed', 'cancelled', 'provider_rejected', 'transport_failed') + OR p_input_tokens IS NULL OR p_input_tokens < 0 + OR p_output_tokens IS NULL OR p_output_tokens < 0 + OR p_cached_input_tokens IS NULL OR p_cached_input_tokens < 0 + OR p_cached_input_tokens > p_input_tokens + OR p_spend_micros IS NULL OR p_spend_micros < 0 + OR p_settled_at_millis IS NULL OR p_settled_at_millis < 0 THEN + RAISE EXCEPTION 'invalid provider budget provider settlement'; + END IF; + SELECT reservation.* INTO v_reservation + FROM agentos.provider_budget_reservations AS reservation + WHERE reservation.decision_ref = p_decision_ref FOR UPDATE; + IF NOT FOUND OR v_reservation.provider <> p_provider + OR v_reservation.credential_domain <> p_credential_domain THEN + RAISE EXCEPTION 'provider budget provider settlement is unauthorized'; + END IF; + IF v_reservation.workload_principal IS NULL THEN + RETURN QUERY SELECT * FROM agentos.settle_provider_budget( + p_decision_ref, v_reservation.subject, p_forward_outcome, + p_input_tokens, p_output_tokens, p_cached_input_tokens, p_spend_micros, + p_settled_at_millis); + RETURN; + END IF; + v_calculated_spend := ceil(( + p_input_tokens::numeric * (v_reservation.effective_pricing ->> 'inputMicrosPerMillionTokens')::numeric + + p_output_tokens::numeric * (v_reservation.effective_pricing ->> 'outputMicrosPerMillionTokens')::numeric + ) / 1000000)::bigint; + IF p_spend_micros <> v_calculated_spend THEN + RAISE EXCEPTION 'provider budget settlement pricing conflicts'; + END IF; + IF v_reservation.state = 'settled' THEN + IF v_reservation.forward_outcome <> p_forward_outcome + OR v_reservation.input_tokens <> p_input_tokens + OR v_reservation.output_tokens <> p_output_tokens + OR v_reservation.cached_input_tokens <> p_cached_input_tokens + OR v_reservation.spend_micros <> p_spend_micros + OR v_reservation.settled_at_millis <> p_settled_at_millis THEN + RAISE EXCEPTION 'provider budget settlement conflicts'; + END IF; + RETURN QUERY SELECT 'settled'::text, v_reservation.forward_outcome, + v_reservation.input_tokens::double precision, + v_reservation.output_tokens::double precision, + v_reservation.cached_input_tokens::double precision, + v_reservation.spend_micros::double precision, + v_reservation.settled_at_millis::double precision; + RETURN; + END IF; + IF v_reservation.state <> 'active' + OR v_reservation.attempted_at_millis IS NULL THEN + RAISE EXCEPTION 'provider budget reservation has no claimed attempt'; + END IF; + IF p_settled_at_millis < v_reservation.reserved_at_millis THEN + RAISE EXCEPTION 'provider budget settlement predates reservation'; + END IF; + IF p_input_tokens + p_output_tokens > v_reservation.reserved_tokens + OR p_spend_micros > v_reservation.reserved_spend_micros THEN + RAISE EXCEPTION 'provider budget settlement exceeds reservation'; + END IF; + v_token_start := v_reservation.token_window_started_at_millis; + v_spend_start := v_reservation.spend_window_started_at_millis; + INSERT INTO agentos.provider_budget_counters + (budget_key, dimension, window_started_at_millis, window_ends_at_millis, + consumed, rate_class) + VALUES + (v_reservation.budget_key, 'token', v_token_start, + v_token_start + (v_reservation.effective_limits ->> 'tokenWindowMillis')::bigint, + 0, v_reservation.rate_class), + (v_reservation.budget_key, 'spend', v_spend_start, + v_spend_start + (v_reservation.effective_limits ->> 'spendWindowMillis')::bigint, + 0, v_reservation.rate_class) + ON CONFLICT (budget_key, dimension, window_started_at_millis) DO NOTHING; + UPDATE agentos.provider_budget_counters + SET consumed = consumed + p_input_tokens + p_output_tokens, + updated_at = transaction_timestamp() + WHERE budget_key = v_reservation.budget_key AND dimension = 'token' + AND window_started_at_millis = v_token_start; + UPDATE agentos.provider_budget_counters + SET consumed = consumed + p_spend_micros, + updated_at = transaction_timestamp() + WHERE budget_key = v_reservation.budget_key AND dimension = 'spend' + AND window_started_at_millis = v_spend_start; + UPDATE agentos.provider_budget_reservations SET state = 'settled', + forward_outcome = p_forward_outcome, input_tokens = p_input_tokens, + output_tokens = p_output_tokens, cached_input_tokens = p_cached_input_tokens, + spend_micros = p_spend_micros, settled_at_millis = p_settled_at_millis, + updated_at = transaction_timestamp() + WHERE decision_ref = p_decision_ref; + RETURN QUERY SELECT 'settled'::text, p_forward_outcome, + p_input_tokens::double precision, p_output_tokens::double precision, + p_cached_input_tokens::double precision, p_spend_micros::double precision, + p_settled_at_millis::double precision; +END; +$$; diff --git a/database/migrations/0027_provider_attempt_recovery.sql b/database/migrations/0027_provider_attempt_recovery.sql new file mode 100644 index 00000000..e3b92a89 --- /dev/null +++ b/database/migrations/0027_provider_attempt_recovery.sql @@ -0,0 +1,204 @@ +ALTER TABLE agentos.provider_budget_reservations + ADD COLUMN attempt_lease_expires_at_millis bigint + CHECK (attempt_lease_expires_at_millis IS NULL OR attempt_lease_expires_at_millis >= 0); + +CREATE OR REPLACE FUNCTION agentos.validate_workload_provider_budget( + p_decision_ref text, p_correlation_id text, p_principal jsonb, + p_provider text, p_credential_domain text, p_capability text, + p_resource jsonb, p_model text, p_rate_class text, p_limits jsonb, + p_pricing jsonb, p_requested_tokens bigint, p_requested_spend_micros bigint, + p_expires_at_millis bigint, p_now_millis bigint +) +RETURNS TABLE ("outcome" text) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = agentos, pg_temp AS $$ +DECLARE v_reservation agentos.provider_budget_reservations%ROWTYPE; +BEGIN + SELECT reservation.* INTO v_reservation + FROM agentos.provider_budget_reservations reservation + WHERE reservation.decision_ref = p_decision_ref FOR UPDATE; + IF NOT FOUND OR v_reservation.correlation_id <> p_correlation_id + OR v_reservation.workload_principal - 'serviceAccountUid' - 'podName' - 'podUid' <> p_principal + OR v_reservation.provider <> p_provider + OR v_reservation.credential_domain <> p_credential_domain + OR v_reservation.capability <> p_capability OR v_reservation.resource <> p_resource + OR v_reservation.model <> p_model OR v_reservation.rate_class <> p_rate_class + OR v_reservation.effective_limits <> p_limits OR v_reservation.effective_pricing <> p_pricing + OR v_reservation.reserved_tokens <> p_requested_tokens + OR v_reservation.reserved_spend_micros <> p_requested_spend_micros + OR v_reservation.policy_expires_at_millis <> p_expires_at_millis + OR v_reservation.state <> 'active' OR v_reservation.attempted_at_millis IS NOT NULL + OR v_reservation.lease_expires_at_millis <= p_now_millis + OR v_reservation.policy_expires_at_millis <= p_now_millis THEN + RAISE EXCEPTION 'workload provider budget reservation unavailable'; + END IF; + UPDATE agentos.provider_budget_reservations SET attempted_at_millis = p_now_millis, + attempt_lease_expires_at_millis = p_now_millis + 900000, + updated_at = transaction_timestamp() + WHERE decision_ref = p_decision_ref; + RETURN QUERY SELECT 'attempted'::text; +END; +$$; + +ALTER FUNCTION agentos.settle_provider_budget_for_provider( + text,text,text,text,bigint,bigint,bigint,bigint,bigint +) RENAME TO settle_provider_budget_for_provider_timestamped; + +CREATE FUNCTION agentos.settle_provider_budget_for_provider( + p_decision_ref text, p_provider text, p_credential_domain text, + p_forward_outcome text, p_input_tokens bigint, p_output_tokens bigint, + p_cached_input_tokens bigint, p_spend_micros bigint, p_settled_at_millis bigint +) +RETURNS TABLE ( + "outcome" text, "forwardOutcome" text, "inputTokens" double precision, + "outputTokens" double precision, "cachedInputTokens" double precision, + "spendMicros" double precision, "settledAtMillis" double precision +) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = agentos, pg_temp AS $$ +DECLARE v_row agentos.provider_budget_reservations%ROWTYPE; +BEGIN + SELECT reservation.* INTO v_row FROM agentos.provider_budget_reservations reservation + WHERE decision_ref = p_decision_ref FOR UPDATE; + IF FOUND AND v_row.state = 'settled' THEN + IF v_row.provider <> p_provider OR v_row.credential_domain <> p_credential_domain + OR v_row.forward_outcome <> p_forward_outcome + OR v_row.input_tokens <> p_input_tokens OR v_row.output_tokens <> p_output_tokens + OR v_row.cached_input_tokens <> p_cached_input_tokens + OR v_row.spend_micros <> p_spend_micros THEN + RAISE EXCEPTION 'provider budget settlement conflicts'; + END IF; + RETURN QUERY SELECT 'settled'::text, v_row.forward_outcome, + v_row.input_tokens::double precision, v_row.output_tokens::double precision, + v_row.cached_input_tokens::double precision, v_row.spend_micros::double precision, + v_row.settled_at_millis::double precision; + RETURN; + END IF; + RETURN QUERY SELECT * FROM agentos.settle_provider_budget_for_provider_timestamped( + p_decision_ref,p_provider,p_credential_domain,p_forward_outcome,p_input_tokens, + p_output_tokens,p_cached_input_tokens,p_spend_micros,p_settled_at_millis); +END; +$$; + +REVOKE ALL ON FUNCTION agentos.settle_provider_budget_for_provider( + text,text,text,text,bigint,bigint,bigint,bigint,bigint +) FROM PUBLIC; + +ALTER FUNCTION agentos.reserve_workload_provider_budget( + text,text,text,jsonb,text,text,text,jsonb,text,text,text,jsonb,jsonb, + bigint,bigint,bigint,bigint +) RENAME TO reserve_workload_provider_budget_unreconciled; + +CREATE FUNCTION agentos.reserve_workload_provider_budget( + p_decision_ref text, p_budget_key text, p_correlation_id text, + p_principal jsonb, p_provider text, p_credential_domain text, + p_capability text, p_resource jsonb, p_environment text, p_model text, + p_rate_class text, p_limits jsonb, p_pricing jsonb, + p_policy_expires_at_millis bigint, + p_requested_tokens bigint, p_requested_spend_micros bigint, p_now_millis bigint +) +RETURNS TABLE ( + "outcome" text, "effectiveRateClass" text, "retryAtMillis" double precision, + "requestWindowEndsAtMillis" double precision, + "tokenWindowEndsAtMillis" double precision, + "spendWindowEndsAtMillis" double precision, + "leaseExpiresAtMillis" double precision +) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = agentos, pg_temp AS $$ +BEGIN + PERFORM * FROM agentos.recover_expired_workload_provider_attempts(p_now_millis, 100); + RETURN QUERY SELECT * FROM agentos.reserve_workload_provider_budget_unreconciled( + p_decision_ref,p_budget_key,p_correlation_id,p_principal,p_provider, + p_credential_domain,p_capability,p_resource,p_environment,p_model,p_rate_class, + p_limits,p_pricing,p_policy_expires_at_millis,p_requested_tokens, + p_requested_spend_micros,p_now_millis); +END; +$$; + +REVOKE ALL ON FUNCTION agentos.reserve_workload_provider_budget( + text,text,text,jsonb,text,text,text,jsonb,text,text,text,jsonb,jsonb, + bigint,bigint,bigint,bigint +) FROM PUBLIC; + +CREATE FUNCTION agentos.renew_workload_provider_attempt( + p_decision_ref text, p_provider text, p_credential_domain text, + p_now_millis bigint +) +RETURNS TABLE ("outcome" text, "leaseExpiresAtMillis" double precision) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = agentos, pg_temp AS $$ +DECLARE v_attempted bigint; v_lease bigint; +BEGIN + SELECT attempted_at_millis, attempt_lease_expires_at_millis + INTO v_attempted, v_lease FROM agentos.provider_budget_reservations + WHERE decision_ref = p_decision_ref AND state = 'active' + AND provider = p_provider AND credential_domain = p_credential_domain + FOR UPDATE; + IF NOT FOUND OR v_attempted IS NULL OR v_lease <= p_now_millis + OR p_now_millis >= v_attempted + 900000 THEN + RAISE EXCEPTION 'provider attempt lease unavailable'; + END IF; + v_lease := least(v_attempted + 900000, p_now_millis + 60000); + UPDATE agentos.provider_budget_reservations + SET attempt_lease_expires_at_millis = v_lease, updated_at = transaction_timestamp() + WHERE decision_ref = p_decision_ref; + RETURN QUERY SELECT 'renewed'::text, v_lease::double precision; +END; +$$; + +CREATE FUNCTION agentos.recover_expired_workload_provider_attempts( + p_now_millis bigint, p_limit integer +) +RETURNS TABLE ("outcome" text) +LANGUAGE plpgsql SECURITY DEFINER SET search_path = agentos, pg_temp AS $$ +DECLARE v_row agentos.provider_budget_reservations%ROWTYPE; v_count integer := 0; +BEGIN + IF p_now_millis < 0 OR p_limit < 1 OR p_limit > 1000 THEN + RAISE EXCEPTION 'invalid provider attempt recovery'; + END IF; + FOR v_row IN SELECT reservation.* FROM agentos.provider_budget_reservations reservation + WHERE state = 'active' AND attempted_at_millis IS NOT NULL + AND (attempt_lease_expires_at_millis IS NULL + OR attempt_lease_expires_at_millis <= p_now_millis) + ORDER BY attempt_lease_expires_at_millis NULLS FIRST, decision_ref + LIMIT p_limit FOR UPDATE SKIP LOCKED + LOOP + UPDATE agentos.provider_budget_counters SET consumed = consumed + v_row.reserved_tokens, + updated_at = transaction_timestamp() + WHERE budget_key = v_row.budget_key AND dimension = 'token' + AND window_started_at_millis = v_row.token_window_started_at_millis; + UPDATE agentos.provider_budget_counters SET consumed = consumed + v_row.reserved_spend_micros, + updated_at = transaction_timestamp() + WHERE budget_key = v_row.budget_key AND dimension = 'spend' + AND window_started_at_millis = v_row.spend_window_started_at_millis; + UPDATE agentos.provider_budget_reservations SET state = 'settled', + forward_outcome = 'transport_failed', input_tokens = reserved_tokens, + output_tokens = 0, cached_input_tokens = 0, + spend_micros = reserved_spend_micros, settled_at_millis = p_now_millis, + updated_at = transaction_timestamp() + WHERE decision_ref = v_row.decision_ref AND state = 'active'; + v_count := v_count + 1; + END LOOP; + IF v_count = 0 THEN RETURN QUERY SELECT 'unchanged'::text; + ELSE RETURN QUERY SELECT 'recovered'::text; END IF; +END; +$$; + +REVOKE ALL ON FUNCTION agentos.renew_workload_provider_attempt(text,text,text,bigint) FROM PUBLIC; +REVOKE ALL ON FUNCTION agentos.recover_expired_workload_provider_attempts(bigint,integer) FROM PUBLIC; + +CREATE OR REPLACE FUNCTION agentos.configure_egress_authorizer_privileges(p_database_role name) +RETURNS void LANGUAGE plpgsql SECURITY DEFINER SET search_path = agentos, pg_temp AS $$ +DECLARE v_role oid; v_dangerous boolean; +BEGIN + IF session_user <> current_user THEN RAISE EXCEPTION 'egress authorizer privileges require the schema owner'; END IF; + SELECT oid, rolsuper OR rolcreaterole OR rolcreatedb OR rolreplication OR rolbypassrls + INTO v_role, v_dangerous FROM pg_catalog.pg_roles WHERE rolname = p_database_role; + IF v_role IS NULL OR v_dangerous OR EXISTS + (SELECT 1 FROM pg_catalog.pg_auth_members WHERE member = v_role) THEN + RAISE EXCEPTION 'egress authorizer database role is unavailable or privileged'; + END IF; + EXECUTE format('REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA agentos FROM %I', p_database_role); + EXECUTE format('REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA agentos FROM %I', p_database_role); + EXECUTE format('REVOKE ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA agentos FROM %I', p_database_role); + EXECUTE format('GRANT USAGE ON SCHEMA agentos TO %I', p_database_role); + EXECUTE format('GRANT EXECUTE ON FUNCTION agentos.read_egress_workload_agents(text,text), agentos.read_egress_assignments(uuid), agentos.read_egress_policy_snapshots(jsonb), agentos.reserve_workload_provider_budget(text,text,text,jsonb,text,text,text,jsonb,text,text,text,jsonb,jsonb,bigint,bigint,bigint,bigint), agentos.validate_workload_provider_budget(text,text,jsonb,text,text,text,jsonb,text,text,jsonb,jsonb,bigint,bigint,bigint,bigint), agentos.settle_provider_budget_for_provider(text,text,text,text,bigint,bigint,bigint,bigint,bigint), agentos.renew_workload_provider_attempt(text,text,text,bigint), agentos.recover_expired_workload_provider_attempts(bigint,integer) TO %I', p_database_role); +END; +$$; diff --git a/database/migrations/meta/0026_snapshot.json b/database/migrations/meta/0026_snapshot.json new file mode 100644 index 00000000..bdf07cc7 --- /dev/null +++ b/database/migrations/meta/0026_snapshot.json @@ -0,0 +1,18 @@ +{ + "id": "320c7619-ae09-4b7c-b37e-5bbfdc14a8d9", + "prevId": "cec0b2c9-c4d9-46c8-be64-6bfd3f1e377b", + "version": "7", + "dialect": "postgresql", + "tables": {}, + "enums": {}, + "schemas": {}, + "views": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/database/migrations/meta/_journal.json b/database/migrations/meta/_journal.json index 0740621b..b7cdee60 100644 --- a/database/migrations/meta/_journal.json +++ b/database/migrations/meta/_journal.json @@ -183,6 +183,20 @@ "when": 1785747600000, "tag": "0025_assignment_execution_epochs", "breakpoints": true + }, + { + "idx": 26, + "version": "7", + "when": 1786464018520, + "tag": "0026_workload_provider_budget", + "breakpoints": true + }, + { + "idx": 27, + "version": "7", + "when": 1786525200000, + "tag": "0027_provider_attempt_recovery", + "breakpoints": true } ] } diff --git a/database/tests/provider-budget-enforcement.effect.test.ts b/database/tests/provider-budget-enforcement.effect.test.ts index 98935afc..d91f88b0 100644 --- a/database/tests/provider-budget-enforcement.effect.test.ts +++ b/database/tests/provider-budget-enforcement.effect.test.ts @@ -2,7 +2,7 @@ import * as BunFileSystem from "@effect/platform-bun/BunFileSystem"; import * as BunPath from "@effect/platform-bun/BunPath"; import { PGlite } from "@electric-sql/pglite"; import { assert, layer } from "@effect/vitest"; -import { Context, Effect, FileSystem, Layer, Path } from "effect"; +import { Context, Effect, Exit, FileSystem, Layer, Path } from "effect"; import { fileURLToPath } from "node:url"; const migrationsDirectory = fileURLToPath( @@ -232,6 +232,11 @@ const settleProvider = Effect.fn("test.providerBudget.settleProvider")( readonly atMillis?: number; }) { const database = yield* TestDatabase; + yield* database.exec(` + UPDATE agentos.provider_budget_reservations SET attempted_at_millis = ${input.atMillis ?? now} + WHERE decision_ref = '${input.decision}' + AND workload_principal IS NOT NULL AND state = 'active' + `); return yield* database.query<{ readonly outcome: string; readonly forwardOutcome: string; @@ -253,7 +258,503 @@ const settleProvider = Effect.fn("test.providerBudget.settleProvider")( }, ); +const renewProviderAttempt = Effect.fn("test.providerBudget.renewProviderAttempt")( + function*(decision: string, provider: string, credentialDomain: string, atMillis: number) { + const database = yield* TestDatabase; + return yield* database.query<{ + readonly outcome: string; + readonly leaseExpiresAtMillis: number; + }>(` + SELECT * FROM agentos.renew_workload_provider_attempt( + '${decision}', '${provider}', '${credentialDomain}', ${atMillis} + ) + `); + }, +); + +const workloadPrincipal = { + kind: "kubernetes_workload", + namespace: "hermes-workers", + serviceAccountName: "hermes-codex", + serviceAccountUid: "11111111-1111-4111-8111-111111111111", + podName: "hermes-0", + podUid: "22222222-2222-4222-8222-222222222222", + policyRevision: 7, + policyResourceVersion: "18422", + hermesProfile: "fleet-codex", +}; +const workloadLimits = { + requestWindowMillis: 60_000, + maximumRequests: 2, + maximumConcurrent: 1, + tokenWindowMillis: 60_000, + maximumTokens: 1_000, + spendWindowMillis: 3_600_000, + maximumSpendMicros: 100_000, +}; +const workloadPricing = { + version: 1, + inputMicrosPerMillionTokens: 1_000_000, + outputMicrosPerMillionTokens: 1_000_000, +}; +const reserveWorkload = Effect.fn("test.providerBudget.reserveWorkload")( + function*(input: { + readonly decision: string; + readonly budgetKey?: string; + readonly principal?: object; + readonly limits?: object; + readonly rateClass?: string; + readonly requestedTokens?: number; + readonly requestedSpendMicros?: number; + readonly atMillis?: number; + }) { + const database = yield* TestDatabase; + const rows = yield* database.query(` + SELECT * FROM agentos.reserve_workload_provider_budget( + '${input.decision}', '${input.budgetKey ?? `budget_${"9".repeat(64)}`}', + 'corr_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + '${JSON.stringify(input.principal ?? workloadPrincipal)}'::jsonb, + 'openai', 'openai-responses', 'openai.responses.create', + '{"kind":"provider_service","provider":"openai","service":"responses"}'::jsonb, + 'production', 'gpt-5.6-sol', '${input.rateClass ?? "low"}', + '${JSON.stringify(input.limits ?? workloadLimits)}'::jsonb, + '${JSON.stringify(workloadPricing)}'::jsonb, + ${(input.atMillis ?? now) + 15_000}, + ${input.requestedTokens ?? 1_000}, ${input.requestedSpendMicros ?? 100_000}, + ${input.atMillis ?? now} + ) + `); + return rows[0]!; + }, +); + +const claimWorkload = Effect.fn("test.providerBudget.claimWorkload")( + function*(input: { readonly decision: string; readonly atMillis?: number; readonly requestedTokens?: number; readonly requestedSpendMicros?: number }) { + const database = yield* TestDatabase; + const principal = { + kind: workloadPrincipal.kind, + namespace: workloadPrincipal.namespace, + serviceAccountName: workloadPrincipal.serviceAccountName, + policyRevision: workloadPrincipal.policyRevision, + policyResourceVersion: workloadPrincipal.policyResourceVersion, + hermesProfile: workloadPrincipal.hermesProfile, + }; + return yield* database.query<{ readonly outcome: string }>(` + SELECT * FROM agentos.validate_workload_provider_budget( + '${input.decision}', 'corr_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + '${JSON.stringify(principal)}'::jsonb, 'openai', 'openai-responses', + 'openai.responses.create', + '{"kind":"provider_service","provider":"openai","service":"responses"}'::jsonb, + 'gpt-5.6-sol', 'low', '${JSON.stringify(workloadLimits)}'::jsonb, + '${JSON.stringify(workloadPricing)}'::jsonb, ${input.requestedTokens ?? 1000}, ${input.requestedSpendMicros ?? 100000}, + ${(input.atMillis ?? now) + 15_000}, ${input.atMillis ?? now} + ) + `).pipe(Effect.map((rows) => rows[0]!)); + }, +); + layer(databaseLayer)("durable provider budgets", (it) => { + it.effect("renews only an active attempted reservation bound to the provider domain", () => + Effect.gen(function*() { + const decision = `decision_${"e1".repeat(16)}`; + yield* reserveWorkload({ + decision, + budgetKey: `budget_${"e1".repeat(32)}`, + atMillis: now, + }); + yield* claimWorkload({ decision, atMillis: now }); + const renewed = yield* renewProviderAttempt( + decision, + "openai", + "openai-responses", + now + 1_000, + ); + assert.strictEqual(renewed[0]?.outcome, "renewed"); + assert.strictEqual(renewed[0]?.leaseExpiresAtMillis, now + 61_000); + const mismatch = yield* Effect.flip( + renewProviderAttempt(decision, "openai", "other-domain", now + 2_000), + ); + assert.match(mismatch.message, /attempt lease unavailable/); + })); + + it.effect("claims one provider attempt and rejects replay while retaining expired attempted liability", () => + Effect.gen(function*() { + const budgetKey = `budget_${"a1".repeat(32)}`; + const decision = `decision_${"a1".repeat(16)}`; + assert.strictEqual((yield* reserveWorkload({ decision, budgetKey })).outcome, "reserved"); + assert.strictEqual((yield* claimWorkload({ decision })).outcome, "attempted"); + const replay = yield* Effect.flip(claimWorkload({ decision })); + assert.match(replay.message, /reservation unavailable/); + const denied = yield* reserveWorkload({ + decision: `decision_${"a2".repeat(16)}`, + budgetKey, + atMillis: now + 15_001, + requestedTokens: 1, + requestedSpendMicros: 1, + }); + assert.strictEqual(denied.outcome, "budget_exhausted"); + })); + + it.effect("charges settlement to the reservation windows across a boundary", () => + Effect.gen(function*() { + const database = yield* TestDatabase; + const budgetKey = `budget_${"b1".repeat(32)}`; + const oldDecision = `decision_${"b1".repeat(16)}`; + const newDecision = `decision_${"b2".repeat(16)}`; + const limits = { ...workloadLimits, maximumConcurrent: 2 }; + yield* reserveWorkload({ decision: oldDecision, budgetKey, limits, atMillis: now + 59_000, requestedTokens: 50, requestedSpendMicros: 50 }); + yield* reserveWorkload({ decision: newDecision, budgetKey, limits, atMillis: now + 61_000, requestedTokens: 50, requestedSpendMicros: 50 }); + yield* settleProvider({ decision: oldDecision, provider: "openai", credentialDomain: "openai-responses", inputTokens: 50, spendMicros: 50, atMillis: now + 62_000 }); + const counters = yield* database.query<{ readonly window: number; readonly consumed: number }>(` + SELECT window_started_at_millis::double precision AS window, + consumed::double precision AS consumed + FROM agentos.provider_budget_counters + WHERE budget_key = '${budgetKey}' AND dimension = 'token' + ORDER BY window_started_at_millis + `); + assert.deepStrictEqual(counters.map(({ consumed }) => consumed), [50, 0]); + })); + + it.effect("rejects zero and over-maximum exact reservation quantities", () => + Effect.gen(function*() { + for (const [requestedTokens, requestedSpendMicros] of [[0, 1], [1, 0], [1_001, 1], [1, 100_001]]) { + const failure = yield* Effect.flip(reserveWorkload({ + decision: `decision_${String(requestedTokens).padStart(16, "0")}${String(requestedSpendMicros).padStart(16, "0")}`, + budgetKey: `budget_${"c1".repeat(32)}`, + requestedTokens, + requestedSpendMicros, + })); + assert.match(failure.message, /invalid workload provider budget reservation/); + } + })); + + it.effect("reserves only an exact modern workload decision and reclaims expired concurrency", () => + Effect.gen(function*() { + const decision = `decision_${"ab".repeat(16)}`; + assert.strictEqual((yield* reserveWorkload({ decision })).outcome, "reserved"); + assert.strictEqual((yield* reserveWorkload({ decision })).outcome, "reserved"); + const conflict = yield* Effect.flip(reserveWorkload({ + decision, + principal: { ...workloadPrincipal, podUid: "different-pod-uid" }, + })); + assert.match(conflict.message, /decision reference conflicts/); + assert.strictEqual((yield* reserveWorkload({ + decision: `decision_${"bc".repeat(16)}`, + })).outcome, "budget_exhausted"); + assert.strictEqual((yield* reserveWorkload({ + decision: `decision_${"cd".repeat(16)}`, + atMillis: now + 15_001, + })).outcome, "reserved"); + })); + + it.effect("settles modern usage exactly once and rejects a conflict", () => + Effect.gen(function*() { + const decision = `decision_${"de".repeat(16)}`; + yield* reserveWorkload({ decision, budgetKey: `budget_${"8".repeat(64)}` }); + const first = yield* settleProvider({ + decision, + provider: "openai", + credentialDomain: "openai-responses", + inputTokens: 80, + outputTokens: 20, + cachedInputTokens: 10, + spendMicros: 100, + }); + assert.deepStrictEqual( + [first.outcome, first.inputTokens, first.outputTokens, first.spendMicros], + ["settled", 80, 20, 100], + ); + assert.strictEqual((yield* settleProvider({ + decision, + provider: "openai", + credentialDomain: "openai-responses", + inputTokens: 80, + outputTokens: 20, + cachedInputTokens: 10, + spendMicros: 100, + })).outcome, "settled"); + const conflict = yield* Effect.flip(settleProvider({ + decision, + provider: "openai", + credentialDomain: "openai-responses", + inputTokens: 81, + outputTokens: 20, + spendMicros: 101, + })); + assert.match(conflict.message, /settlement conflicts/); + })); + + it.effect("returns the canonical committed settlement when an HTTP retry changes only transport time", () => + Effect.gen(function*() { + const decision = `decision_${"d0".repeat(16)}`; + yield* reserveWorkload({ decision, budgetKey: `budget_${"d0".repeat(32)}`, requestedTokens: 100, requestedSpendMicros: 100 }); + yield* claimWorkload({ decision, requestedTokens: 100, requestedSpendMicros: 100 }); + const first = yield* settleProvider({ + decision, + provider: "openai", + credentialDomain: "openai-responses", + inputTokens: 50, + outputTokens: 50, + spendMicros: 100, + atMillis: now + 1_000, + }); + const replay = yield* settleProvider({ + decision, + provider: "openai", + credentialDomain: "openai-responses", + inputTokens: 50, + outputTokens: 50, + spendMicros: 100, + atMillis: now + 2_000, + }); + assert.deepStrictEqual(replay, first); + })); + + it.effect("recovers an expired claimed attempt conservatively and releases later-window concurrency", () => + Effect.gen(function*() { + const database = yield* TestDatabase; + const budgetKey = `budget_${"d1".repeat(32)}`; + const decision = `decision_${"d1".repeat(16)}`; + yield* reserveWorkload({ decision, budgetKey, requestedTokens: 100, requestedSpendMicros: 100 }); + yield* claimWorkload({ decision, requestedTokens: 100, requestedSpendMicros: 100 }); + const recovered = yield* database.query<{ readonly outcome: string }>(` + SELECT * FROM agentos.recover_expired_workload_provider_attempts(${now + 1_000_000}, 100) + `); + assert.deepStrictEqual(recovered, [{ outcome: "recovered" }]); + const row = yield* database.query<{ readonly state: string; readonly tokens: number; readonly spend: number }>(` + SELECT state, (input_tokens + output_tokens)::double precision AS tokens, + spend_micros::double precision AS spend + FROM agentos.provider_budget_reservations WHERE decision_ref = '${decision}' + `); + assert.deepStrictEqual(row, [{ state: "settled", tokens: 100, spend: 100 }]); + assert.deepStrictEqual( + yield* database.query<{ readonly dimension: string; readonly consumed: number }>(` + SELECT dimension, consumed::double precision AS consumed + FROM agentos.provider_budget_counters + WHERE budget_key = '${budgetKey}' AND dimension IN ('token', 'spend') + ORDER BY dimension + `), + [{ dimension: "spend", consumed: 100 }, { dimension: "token", consumed: 100 }], + ); + assert.deepStrictEqual( + yield* database.query<{ readonly outcome: string }>(` + SELECT * FROM agentos.recover_expired_workload_provider_attempts(${now + 1_000_001}, 100) + `), + [{ outcome: "unchanged" }], + ); + const later = yield* reserveWorkload({ + decision: `decision_${"d2".repeat(16)}`, + budgetKey, + requestedTokens: 1, + requestedSpendMicros: 1, + atMillis: now + 1_000_000, + }); + assert.strictEqual(later.outcome, "reserved"); + })); + + it.effect("conservatively recovers a legacy attempted row with a null attempt lease", () => + Effect.gen(function*() { + const database = yield* TestDatabase; + const decision = `decision_${"d3".repeat(16)}`; + yield* reserveWorkload({ + decision, + budgetKey: `budget_${"d3".repeat(32)}`, + requestedTokens: 100, + requestedSpendMicros: 100, + }); + yield* claimWorkload({ decision, requestedTokens: 100, requestedSpendMicros: 100 }); + yield* database.query(` + UPDATE agentos.provider_budget_reservations + SET attempt_lease_expires_at_millis = NULL + WHERE decision_ref = '${decision}' + `); + assert.deepStrictEqual( + yield* database.query<{ readonly outcome: string }>(` + SELECT * FROM agentos.recover_expired_workload_provider_attempts(${now + 1_000_000}, 100) + `), + [{ outcome: "recovered" }], + ); + assert.deepStrictEqual( + yield* database.query<{ readonly state: string; readonly tokens: number; readonly spend: number }>(` + SELECT state, (input_tokens + output_tokens)::double precision AS tokens, + spend_micros::double precision AS spend + FROM agentos.provider_budget_reservations WHERE decision_ref = '${decision}' + `), + [{ state: "settled", tokens: 100, spend: 100 }], + ); + })); + + it.effect("linearizes concurrent attempt renewal and expiry recovery without double charge", () => + Effect.gen(function*() { + const database = yield* TestDatabase; + let renewWins = 0; + let recoveryWins = 0; + for (let repetition = 0; repetition < 12; repetition += 1) { + const suffix = `f${repetition.toString(16).padStart(3, "0")}`; + const decision = `decision_${suffix.repeat(8)}`; + const budgetKey = `budget_${suffix.repeat(16)}`; + const expiryAt = now + 899_999; + const renewAt = expiryAt - 1; + yield* reserveWorkload({ + decision, + budgetKey, + requestedTokens: 100, + requestedSpendMicros: 100, + }); + yield* claimWorkload({ + decision, + requestedTokens: 100, + requestedSpendMicros: 100, + }); + yield* database.query(` + UPDATE agentos.provider_budget_reservations + SET attempt_lease_expires_at_millis = ${expiryAt} + WHERE decision_ref = '${decision}' + `); + const renew = Effect.exit(renewProviderAttempt( + decision, + "openai", + "openai-responses", + renewAt, + )); + const recover = Effect.exit(database.query<{ readonly outcome: string }>(` + SELECT * FROM agentos.recover_expired_workload_provider_attempts(${expiryAt}, 100) + `)); + const [first, second] = yield* Effect.all( + repetition % 2 === 0 ? [renew, recover] : [recover, renew], + { concurrency: "unbounded" }, + ); + const [renewResult, recoverResult] = repetition % 2 === 0 + ? [first, second] + : [second, first]; + const row = (yield* database.query<{ + readonly state: string; + readonly lease: number | null; + readonly tokenWindow: number; + readonly spendWindow: number; + readonly tokens: number; + readonly spend: number; + }>(` + SELECT state, attempt_lease_expires_at_millis::double precision AS lease, + token_window_started_at_millis::double precision AS "tokenWindow", + spend_window_started_at_millis::double precision AS "spendWindow", + (input_tokens + output_tokens)::double precision AS tokens, + spend_micros::double precision AS spend + FROM agentos.provider_budget_reservations + WHERE decision_ref = '${decision}' + `))[0]; + assert.isDefined(row); + const renewed = Exit.isSuccess(renewResult); + assert.strictEqual(row?.state, renewed ? "active" : "settled"); + assert.deepStrictEqual( + { tokens: row?.tokens, spend: row?.spend }, + renewed ? { tokens: null, spend: null } : { tokens: 100, spend: 100 }, + ); + if (renewed) { + renewWins += 1; + assert.strictEqual(renewResult.value[0]?.outcome, "renewed"); + assert.strictEqual(row?.lease, now + 900_000); + assert.isAbove(row?.lease ?? 0, expiryAt); + assert.isTrue(Exit.isSuccess(recoverResult)); + if (Exit.isSuccess(recoverResult)) { + assert.strictEqual(recoverResult.value[0]?.outcome, "unchanged"); + } + } else { + recoveryWins += 1; + assert.isTrue(Exit.isSuccess(recoverResult)); + if (Exit.isSuccess(recoverResult)) { + assert.strictEqual(recoverResult.value[0]?.outcome, "recovered"); + } + const laterRenew = yield* Effect.exit(renewProviderAttempt( + decision, + "openai", + "openai-responses", + renewAt, + )); + assert.isTrue(Exit.isFailure(laterRenew)); + } + const counters = yield* database.query<{ + readonly dimension: string; + readonly consumed: number; + readonly window: number; + }>(` + SELECT dimension, consumed::double precision AS consumed, + window_started_at_millis::double precision AS window + FROM agentos.provider_budget_counters + WHERE budget_key = '${budgetKey}' AND dimension IN ('token', 'spend') + ORDER BY dimension + `); + assert.deepStrictEqual( + counters.map(({ dimension, consumed }) => ({ dimension, consumed })), + renewed + ? [{ dimension: "spend", consumed: 0 }, { dimension: "token", consumed: 0 }] + : [{ dimension: "spend", consumed: 100 }, { dimension: "token", consumed: 100 }], + ); + assert.isTrue(counters.every(({ window, dimension }) => + window === (dimension === "token" ? row?.tokenWindow : row?.spendWindow) + )); + assert.deepStrictEqual( + yield* database.query<{ readonly outcome: string }>(` + SELECT * FROM agentos.recover_expired_workload_provider_attempts(${expiryAt}, 100) + `), + [{ outcome: "unchanged" }], + ); + if (renewed) { + const blocked = yield* reserveWorkload({ + decision: `decision_${suffix.repeat(7)}eeee`, + budgetKey, + requestedTokens: 1, + requestedSpendMicros: 1, + atMillis: expiryAt, + }); + assert.strictEqual(blocked.outcome, "rate_limited"); + assert.deepStrictEqual( + yield* database.query<{ readonly outcome: string }>(` + SELECT * FROM agentos.recover_expired_workload_provider_attempts(${now + 900_000}, 100) + `), + [{ outcome: "recovered" }], + ); + } + const later = yield* reserveWorkload({ + decision: `decision_${suffix.repeat(7)}ffff`, + budgetKey, + requestedTokens: 1, + requestedSpendMicros: 1, + atMillis: now + 1_000_000, + }); + assert.strictEqual(later.outcome, "reserved"); + } + assert.strictEqual(renewWins, 6); + assert.strictEqual(recoveryWins, 6); + })); + + it.effect("atomically fits exact concurrent reservations and releases only unused capacity", () => + Effect.gen(function*() { + const budgetKey = `budget_${"7".repeat(64)}`; + const limits = { ...workloadLimits, maximumRequests: 4, maximumConcurrent: 2, maximumTokens: 100, maximumSpendMicros: 100 }; + const firstDecision = `decision_${"71".repeat(16)}`; + const secondDecision = `decision_${"72".repeat(16)}`; + assert.strictEqual((yield* reserveWorkload({ decision: firstDecision, budgetKey, limits, requestedTokens: 60, requestedSpendMicros: 60 })).outcome, "reserved"); + assert.strictEqual((yield* reserveWorkload({ decision: secondDecision, budgetKey, limits, requestedTokens: 40, requestedSpendMicros: 40 })).outcome, "reserved"); + assert.strictEqual((yield* reserveWorkload({ decision: `decision_${"73".repeat(16)}`, budgetKey, limits, requestedTokens: 1, requestedSpendMicros: 1 })).outcome, "budget_exhausted"); + yield* settleProvider({ decision: firstDecision, provider: "openai", credentialDomain: "openai-responses", inputTokens: 10, spendMicros: 10 }); + assert.strictEqual((yield* reserveWorkload({ decision: `decision_${"74".repeat(16)}`, budgetKey, limits, requestedTokens: 50, requestedSpendMicros: 50 })).outcome, "reserved"); + const replay = yield* Effect.flip(reserveWorkload({ decision: firstDecision, budgetKey, limits, requestedTokens: 60, requestedSpendMicros: 60 })); + assert.match(replay.message, /reservation is not active/); + })); + + it.effect("rejects settlement larger than the exact token or spend reservation", () => + Effect.gen(function*() { + const budgetKey = `budget_${"6".repeat(64)}`; + const limits = { ...workloadLimits, maximumTokens: 100, maximumSpendMicros: 100 }; + const decision = `decision_${"61".repeat(16)}`; + yield* reserveWorkload({ decision, budgetKey, limits, requestedTokens: 100, requestedSpendMicros: 100 }); + const tokenOverage = yield* Effect.flip(settleProvider({ decision, provider: "openai", credentialDomain: "openai-responses", inputTokens: 101, spendMicros: 101 })); + assert.match(tokenOverage.message, /exceeds reservation/); + const spendOverage = yield* Effect.flip(settleProvider({ decision, provider: "openai", credentialDomain: "openai-responses", outputTokens: 101, spendMicros: 101 })); + assert.match(spendOverage.message, /exceeds reservation/); + })); + it.effect("isolates request and concurrency capacity by durable subject", () => Effect.gen(function*() { const first = yield* reserve({ @@ -469,7 +970,9 @@ layer(databaseLayer)("durable provider budgets", (it) => { const database = yield* TestDatabase; yield* database.exec(` CREATE ROLE provider_budget_egress LOGIN; + CREATE ROLE provider_budget_unprivileged LOGIN; GRANT USAGE ON SCHEMA agentos TO provider_budget_egress; + GRANT USAGE ON SCHEMA agentos TO provider_budget_unprivileged; GRANT SELECT ON ALL TABLES IN SCHEMA agentos TO provider_budget_egress; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA agentos TO provider_budget_egress; @@ -478,6 +981,8 @@ layer(databaseLayer)("durable provider budgets", (it) => { ); `); const privileges = yield* database.query<{ + readonly unprivilegedReserveExecute: boolean; + readonly unprivilegedProviderSettleExecute: boolean; readonly countersSelect: boolean; readonly reserveExecute: boolean; readonly providerSettleExecute: boolean; @@ -485,13 +990,23 @@ layer(databaseLayer)("durable provider budgets", (it) => { readonly overrideExecute: boolean; }>(` SELECT + has_function_privilege( + 'provider_budget_unprivileged', + 'agentos.reserve_workload_provider_budget(text,text,text,jsonb,text,text,text,jsonb,text,text,text,jsonb,jsonb,bigint,bigint,bigint,bigint)', + 'EXECUTE' + ) AS "unprivilegedReserveExecute", + has_function_privilege( + 'provider_budget_unprivileged', + 'agentos.settle_provider_budget_for_provider(text,text,text,text,bigint,bigint,bigint,bigint,bigint)', + 'EXECUTE' + ) AS "unprivilegedProviderSettleExecute", has_table_privilege( 'provider_budget_egress', 'agentos.provider_budget_counters', 'SELECT' ) AS "countersSelect", has_function_privilege( 'provider_budget_egress', - 'agentos.reserve_provider_budget(text,text,text,jsonb,text,text,text,jsonb,text,text,text,bigint)', + 'agentos.reserve_workload_provider_budget(text,text,text,jsonb,text,text,text,jsonb,text,text,text,jsonb,jsonb,bigint,bigint,bigint,bigint)', 'EXECUTE' ) AS "reserveExecute", has_function_privilege( @@ -510,13 +1025,50 @@ layer(databaseLayer)("durable provider budgets", (it) => { 'EXECUTE' ) AS "overrideExecute" `); + yield* database.exec("SET ROLE provider_budget_unprivileged"); + const unprivilegedReserve = yield* Effect.exit(database.exec(` + SELECT * FROM agentos.reserve_workload_provider_budget( + 'decision_${"ef".repeat(16)}', 'budget_${"ef".repeat(32)}', + 'corr_efefefefefefefefefefefefefefefef', + '${JSON.stringify(workloadPrincipal)}'::jsonb, + 'openai', 'openai-responses', 'openai.responses.create', + '{"kind":"provider_service","provider":"openai","service":"responses"}'::jsonb, + 'production', 'gpt-5.6-sol', 'low', + '${JSON.stringify(workloadLimits)}'::jsonb, + '${JSON.stringify(workloadPricing)}'::jsonb, + ${now + 15_000}, 1, 1, ${now} + ) + `)); + const unprivilegedSettlement = yield* Effect.exit(database.exec(` + SELECT * FROM agentos.settle_provider_budget_for_provider( + 'decision_${"f".repeat(32)}', 'openai', 'openai-responses', + 'transport_failed', 0, 0, 0, 0, ${now} + ) + `)); + yield* database.exec("RESET ROLE"); + assert.isTrue(Exit.isFailure(unprivilegedReserve)); assert.deepStrictEqual(privileges, [{ + unprivilegedReserveExecute: false, + unprivilegedProviderSettleExecute: false, countersSelect: false, reserveExecute: true, providerSettleExecute: true, subjectSettleExecute: false, overrideExecute: false, }]); + if (Exit.isFailure(unprivilegedReserve)) { + assert.include( + String(unprivilegedReserve.cause), + "permission denied for function reserve_workload_provider_budget", + ); + } + assert.isTrue(Exit.isFailure(unprivilegedSettlement)); + if (Exit.isFailure(unprivilegedSettlement)) { + assert.include( + String(unprivilegedSettlement.cause), + "permission denied for function settle_provider_budget_for_provider", + ); + } })); it.effect("authorizes the explicit null environment as its own route", () => diff --git a/docs/integrations/hermes-ai-gateway.md b/docs/integrations/hermes-ai-gateway.md index 313f3de7..e04f47cb 100644 --- a/docs/integrations/hermes-ai-gateway.md +++ b/docs/integrations/hermes-ai-gateway.md @@ -194,6 +194,5 @@ followed. ## Operator workflow Use `$agentos-ai-gateway` for the approval, migration, rollout, verification, -rollback, and retirement workflow, including migration from the live v0.1.24 -shared-token topology. This page owns only the Hermes client contract and Pod -wiring. +rollback, and retirement workflow. This page owns only the Hermes client +contract and Pod wiring. diff --git a/docs/security/provider-budget-enforcement.md b/docs/security/provider-budget-enforcement.md index 2c804377..13b587b2 100644 --- a/docs/security/provider-budget-enforcement.md +++ b/docs/security/provider-budget-enforcement.md @@ -1,10 +1,11 @@ # Provider budget enforcement -Status: durable enforcement foundation for AgentOS issue #107. +Status: LC2 workload-principal reservation and broker-validation boundary. -AgentOS reserves governed provider capacity after current PostgreSQL policy and -higher-consistency OpenFGA authorization succeed, but before it returns an -authorization grant. PostgreSQL is authoritative; metrics, traces, gateway +AgentOS reserves governed provider capacity after verified projected Kubernetes +identity and the current versioned Hermes workload policy succeed, but before +the egress authorizer returns an authorization grant. PostgreSQL is +authoritative; metrics, traces, gateway process memory, Pod identity and process lifetime are not counters or policy. An unavailable budget store therefore fails the governed route closed even when telemetry is unavailable. @@ -19,23 +20,28 @@ The implementation is Effect-native end to end: - [`provider-budget-settlement-http.ts`](../../packages/agentos/src/access/provider-budget-settlement-http.ts) rereads a rotating audience-bound Pod token for each closed settlement report and uses only Effect FileSystem, HTTP, timeout and stream boundaries; -- [`policy-decision.ts`](../../packages/agentos/src/access/policy-decision.ts) - reserves capacity only after exact live policy and OpenFGA checks; and +- [`http-authorizer.ts`](../../packages/agentos/src/access/http-authorizer.ts) + reserves modern Hermes workload capacity before emitting allow headers; - [`0020_provider_budget_enforcement.sql`](../../database/migrations/0020_provider_budget_enforcement.sql) atomically revalidates the binding and policy, locks deterministic windows, records reservations and settlements, and applies First-Mate overrides; and - [`0021_provider_budget_provider_settlement.sql`](../../database/migrations/0021_provider_budget_provider_settlement.sql) removes subject-bearing settlement authority from the egress-authorizer - role and exposes only exact provider/credential-domain settlement. + role and exposes only exact provider/credential-domain settlement; and +- [`0026_workload_provider_budget.sql`](../../database/migrations/0026_workload_provider_budget.sql) + evolves that same counter and reservation authority for the full verified + Kubernetes workload principal, exact model, route, policy version, finite + limits, broker validation and workload settlement. ## Stable isolation key -The budget key is a one-way digest over the contract version, canonical Mate -or Assignment subject, provider, credential domain, capability, canonical -resource and environment. It deliberately excludes decision reference, -correlation ID, Pod name, Pod UID and ServiceAccount token. A Pod restart or -identity recreation therefore cannot reset capacity, while two Mates or a Mate -and Assignment cannot consume each other's counters. +The supported forwarding key is a one-way digest over the canonical workload +principal, provider, credential domain, capability, canonical resource and +environment. The durable reservation additionally records ServiceAccount and +Pod names and UIDs, policy revision/resource version, Hermes profile, model, +versioned input/output pricing, exact request-specific token/spend ceilings and +the exact effective finite limits. Historical Mate/Assignment rows remain +interpretable, but they are not a callable fallback for modern forwarding. Dynamic subjects, decisions, correlations and budget keys are persisted only where needed for enforcement and bounded audit. They are not metric labels. @@ -57,13 +63,17 @@ ceiling. | `high` | 300 | 32 | 10,000,000 | 100,000,000 micros | 15 minutes | Request, token and spend windows use epoch-aligned boundaries, so reset time is -deterministic across replicas and restarts. A reservation consumes one request -and one concurrency lease. Settlement releases concurrency and records input, -output, cached-input and spend values exactly once. Cached input is a subset of -input tokens, not additional token consumption. Exact retries are idempotent; -conflicting reuse of a decision or operation identity fails closed. - -An expired lease prevents an abandoned call from holding concurrency forever. +deterministic across replicas and restarts. A reservation consumes one request, +one concurrency lease and only the conservative request-specific token/spend +ceiling derived from the exact streamed Responses body. Settlement charges the +windows recorded by that reservation, releases only unused capacity and records +input, output, cached-input and policy-priced spend exactly once. Cached input +is a subset of input tokens, not additional token consumption. Exact retries +are idempotent; conflicting reuse of a decision or operation identity fails +closed. + +An expired unattempted lease releases concurrency. A reservation claimed for a +provider attempt remains conservatively liable until terminal settlement. Provider components must settle every terminal result (`completed`, `cancelled`, `provider_rejected`, or `transport_failed`) so token and spend usage becomes authoritative promptly. `agentos-egress-authz` exposes a private @@ -71,11 +81,15 @@ usage becomes authoritative promptly. `agentos-egress-authz` exposes a private domain: a dedicated Kubernetes TokenReview audience binds the live Pod and ServiceAccount, and a finite registry derives those two authority fields. -The GitHub broker settles after a streamed response terminates, distinguishes -native provider rejection, downstream cancellation and transport failure, and -uses zero token/spend values. A settlement dependency failure never replaces -the provider response; the still-active 15-minute lease remains the fail-closed -fallback. The OpenAI adapter remains required before #107 can close. +The AI Gateway atomically claims an exact active reservation with its own +projected broker identity before selecting or injecting a credential. The +modern OpenAI path requires literal `stream:true`, positive +`max_output_tokens`, an exact model and a pricing-bound request ceiling at both +boundaries. It settles streamed terminal usage from that pricing, while native +provider rejection, downstream cancellation and transport failure settle +known zero usage. Missing terminal usage, a second attempt claim or exhausted +settlement retries fail the client stream closed and retain conservative +database liability for deterministic recovery. ## Surgical kill switches diff --git a/packages/agentos/resources/crewmates/default/kubernetes/base/kustomization.yaml b/packages/agentos/resources/crewmates/default/kubernetes/base/kustomization.yaml index 28612f4a..c01b124c 100644 --- a/packages/agentos/resources/crewmates/default/kubernetes/base/kustomization.yaml +++ b/packages/agentos/resources/crewmates/default/kubernetes/base/kustomization.yaml @@ -6,7 +6,6 @@ resources: - ../../../../../runtime/kubernetes/base components: - ../../../../../runtime/kubernetes/observability - - ../../../../../runtime/kubernetes/github-provider patches: - path: statefulset.patch.yaml target: diff --git a/packages/agentos/resources/crewmates/default/kubernetes/base/statefulset.patch.yaml b/packages/agentos/resources/crewmates/default/kubernetes/base/statefulset.patch.yaml index 981a4798..72e2583f 100644 --- a/packages/agentos/resources/crewmates/default/kubernetes/base/statefulset.patch.yaml +++ b/packages/agentos/resources/crewmates/default/kubernetes/base/statefulset.patch.yaml @@ -104,9 +104,11 @@ spec: limits: cpu: "2" memory: 2Gi + ephemeral-storage: 1Gi requests: cpu: 250m memory: 512Mi + ephemeral-storage: 128Mi - name: prepare-home image: agentos:dev imagePullPolicy: Never @@ -123,13 +125,17 @@ spec: capabilities: drop: - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true resources: limits: cpu: "2" memory: 2Gi + ephemeral-storage: 1Gi requests: cpu: 250m memory: 512Mi + ephemeral-storage: 128Mi volumeMounts: - name: home mountPath: /home/agent @@ -168,9 +174,11 @@ spec: limits: cpu: "2" memory: 4Gi + ephemeral-storage: 1Gi requests: cpu: 250m memory: 512Mi + ephemeral-storage: 128Mi volumes: - name: database-credentials secret: diff --git a/packages/agentos/resources/crewmates/default/kubernetes/patches/ai-gateway-client.yaml b/packages/agentos/resources/crewmates/default/kubernetes/patches/ai-gateway-client.yaml index 33a55aa9..51611183 100644 --- a/packages/agentos/resources/crewmates/default/kubernetes/patches/ai-gateway-client.yaml +++ b/packages/agentos/resources/crewmates/default/kubernetes/patches/ai-gateway-client.yaml @@ -4,9 +4,6 @@ metadata: name: agentos-crewmate spec: template: - metadata: - labels: - agentos.akua.dev/agentgateway-client: "true" spec: initContainers: - name: install-tools @@ -14,18 +11,14 @@ spec: env: - name: AGENTOS_CODEX_PROVIDER_MODE value: ai-gateway - - name: AGENTOS_EGRESS_TOKEN_FILE - value: /var/run/secrets/agentos-egress/token - name: AI_GATEWAY_URL - value: http://agentgateway-openai.agentos.svc.cluster.local:8788 + value: http://127.0.0.1:8790 containers: - name: crewmate env: - name: AGENTOS_CODEX_PROVIDER_MODE value: ai-gateway - - name: AGENTOS_EGRESS_TOKEN_FILE - value: /var/run/secrets/agentos-egress/token - name: AGENTOS_PROVIDER_CREDENTIAL_KIND value: ai_gateway - name: AI_GATEWAY_URL - value: http://agentgateway-openai.agentos.svc.cluster.local:8788 + value: http://127.0.0.1:8790 diff --git a/packages/agentos/resources/crewmates/default/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml b/packages/agentos/resources/crewmates/default/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml index 490c9b01..8c9ae11c 100644 --- a/packages/agentos/resources/crewmates/default/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml +++ b/packages/agentos/resources/crewmates/default/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml @@ -3,6 +3,8 @@ kind: Kustomization namespace: agentos resources: - ../../../base +components: + - ../../../../../../../../../services/ai-gateway/kubernetes/workload-proxy/component patches: - path: ../../../patches/ai-gateway-client.yaml target: diff --git a/packages/agentos/resources/crewmates/default/kubernetes/tests/manifest.effect.test.ts b/packages/agentos/resources/crewmates/default/kubernetes/tests/manifest.effect.test.ts index 0b070f91..cf5538fd 100644 --- a/packages/agentos/resources/crewmates/default/kubernetes/tests/manifest.effect.test.ts +++ b/packages/agentos/resources/crewmates/default/kubernetes/tests/manifest.effect.test.ts @@ -17,7 +17,8 @@ const EnvironmentEntry = Schema.Struct({ }); type EnvironmentEntry = typeof EnvironmentEntry.Type; const ExecProbe = Schema.Struct({ - exec: Schema.Struct({ command: Schema.Array(Schema.String) }), + exec: Schema.optional(Schema.Struct({ command: Schema.Array(Schema.String) })), + httpGet: Schema.optional(Schema.Unknown), }); const Container = Schema.Struct({ name: Schema.String, @@ -32,6 +33,13 @@ const Container = Schema.Struct({ securityContext: Schema.optional(Schema.Unknown), volumeMounts: Schema.optional(Schema.Unknown), }); +const Volume = Schema.Struct({ + name: Schema.String, + projected: Schema.optional(Schema.Unknown), + secret: Schema.optional(Schema.Unknown), + configMap: Schema.optional(Schema.Unknown), + emptyDir: Schema.optional(Schema.Unknown), +}); const Metadata = Schema.Struct({ name: Schema.String, namespace: Schema.optional(Schema.String), @@ -69,7 +77,7 @@ const StatefulSet = Schema.Struct({ securityContext: Schema.Unknown, initContainers: Schema.Array(Container), containers: Schema.Array(Container), - volumes: Schema.Unknown, + volumes: Schema.Array(Volume), }), }), }), @@ -159,7 +167,6 @@ describe("Crewmate Kubernetes base", () => { "agentos.akua.dev/ai-runtime-version": "0.144.5", "agentos.akua.dev/assignment-id": "00000000-0000-4000-8000-000000000005", - "agentos.akua.dev/github-client": "true", "agentos.akua.dev/otel-client": "true", "agentos.akua.dev/owner-agent-id": "00000000-0000-4000-8000-000000000002", @@ -181,33 +188,29 @@ describe("Crewmate Kubernetes base", () => { runAsUser: 1000, seccompProfile: { type: "RuntimeDefault" }, }); - assert.lengthOf(pod.initContainers, 3); + assert.lengthOf(pod.initContainers, 2); assert.lengthOf(pod.containers, 1); const install = yield* required(pod.initContainers[0], "Missing installer"); const prepare = yield* required(pod.initContainers[1], "Missing prepare"); const container = yield* required(pod.containers[0], "Missing Crewmate"); assert.deepStrictEqual( [...pod.initContainers, container].map(({ image }) => image), - ["agentos:dev", "agentos:dev", "agentos:dev", "agentos:dev"], + ["agentos:dev", "agentos:dev", "agentos:dev"], ); assert.deepStrictEqual( [...pod.initContainers, container].map(({ resources }) => resources), [ { - limits: { cpu: "2", memory: "2Gi" }, - requests: { cpu: "250m", memory: "512Mi" }, - }, - { - limits: { cpu: "2", memory: "2Gi" }, - requests: { cpu: "250m", memory: "512Mi" }, + limits: { cpu: "2", memory: "2Gi", "ephemeral-storage": "1Gi" }, + requests: { cpu: "250m", memory: "512Mi", "ephemeral-storage": "128Mi" }, }, { - limits: { cpu: "250m", memory: "128Mi" }, - requests: { cpu: "25m", memory: "64Mi" }, + limits: { cpu: "2", memory: "2Gi", "ephemeral-storage": "1Gi" }, + requests: { cpu: "250m", memory: "512Mi", "ephemeral-storage": "128Mi" }, }, { - limits: { cpu: "2", memory: "4Gi" }, - requests: { cpu: "250m", memory: "512Mi" }, + limits: { cpu: "2", memory: "4Gi", "ephemeral-storage": "1Gi" }, + requests: { cpu: "250m", memory: "512Mi", "ephemeral-storage": "128Mi" }, }, ], ); @@ -216,7 +219,6 @@ describe("Crewmate Kubernetes base", () => { [ "/opt/agentos/packages/agentos/resources/crewmates/default", "/opt/agentos/packages/agentos/resources/crewmates/default", - undefined, "/opt/agentos/packages/agentos/resources/crewmates/default", ], ); @@ -284,6 +286,24 @@ describe("Crewmate Kubernetes base", () => { ); } assert.isUndefined(variables.PI_CODING_AGENT_DIR); + assert.isUndefined(variables.AGENTOS_EGRESS_TOKEN_FILE); + for (const name of Object.keys(variables)) { + assert.isFalse(name.startsWith("AGENTOS_GITHUB_")); + } + for (const initContainer of pod.initContainers) { + const initVariables = environment(initContainer.env ?? []); + assert.isUndefined(initVariables.AGENTOS_EGRESS_TOKEN_FILE); + for (const name of Object.keys(initVariables)) { + assert.isFalse(name.startsWith("AGENTOS_GITHUB_")); + } + } + assert.notProperty( + statefulSet.spec.template.metadata.labels, + "agentos.akua.dev/github-client", + ); + assert.isUndefined( + pod.initContainers.find(({ name }) => name === "prepare-github-provider"), + ); assert.deepStrictEqual(container.command, ["herdr"]); assert.deepStrictEqual(container.args, [ "server", @@ -313,19 +333,12 @@ describe("Crewmate Kubernetes base", () => { assert.deepStrictEqual(container.securityContext, { allowPrivilegeEscalation: false, capabilities: { drop: ["ALL"] }, + readOnlyRootFilesystem: true, + runAsNonRoot: true, }); assert.deepStrictEqual(container.volumeMounts, [ { mountPath: "/home/agent", name: "home" }, - { - mountPath: "/var/run/secrets/agentos-egress", - name: "agentos-egress-identity", - readOnly: true, - }, - { - mountPath: "/var/run/config/agentos-github", - name: "agentos-github-ca", - readOnly: true, - }, + { mountPath: "/tmp", name: "tmp" }, ]); assert.deepStrictEqual(prepare.volumeMounts, [ { mountPath: "/home/agent", name: "home" }, @@ -340,27 +353,7 @@ describe("Crewmate Kubernetes base", () => { name: "database-credentials", secret: { defaultMode: 288, secretName: "agentos-crewmate-postgres" }, }, - { - name: "agentos-egress-identity", - projected: { - defaultMode: 288, - sources: [{ - serviceAccountToken: { - audience: AGENTOS_EGRESS_TOKEN_AUDIENCE, - expirationSeconds: AGENTOS_EGRESS_TOKEN_EXPIRATION_SECONDS, - path: "token", - }, - }], - }, - }, - { - name: "agentos-github-ca", - configMap: { - defaultMode: 292, - items: [{ key: "ca.pem", path: "ca.pem" }], - name: "agentos-github-ca", - }, - }, + { name: "tmp", emptyDir: { sizeLimit: "256Mi" } }, ]); }).pipe(Effect.provide(BunServices.layer))); @@ -393,26 +386,81 @@ describe("Crewmate Kubernetes base", () => { }); assert.strictEqual( variables.AI_GATEWAY_URL, - "http://agentgateway-openai.agentos.svc.cluster.local:8788", + "http://127.0.0.1:8790", ); assert.isUndefined(variables.AI_GATEWAY_TOKEN); assert.strictEqual(variables.AGENTOS_CODEX_PROVIDER_MODE, "ai-gateway"); - assert.strictEqual( - variables.AGENTOS_EGRESS_TOKEN_FILE, - "/var/run/secrets/agentos-egress/token", - ); + assert.isUndefined(variables.AGENTOS_EGRESS_TOKEN_FILE); assert.deepInclude(prepareVariables, { AGENTOS_ASSIGNMENT_ID: "00000000-0000-4000-8000-000000000005", AGENTOS_CODEX_PROVIDER_MODE: "ai-gateway", - AGENTOS_EGRESS_TOKEN_FILE: "/var/run/secrets/agentos-egress/token", - AI_GATEWAY_URL: - "http://agentgateway-openai.agentos.svc.cluster.local:8788", + AI_GATEWAY_URL: "http://127.0.0.1:8790", }); assert.strictEqual( variables.AGENTOS_PROVIDER_CREDENTIAL_KIND, "ai_gateway", ); assert.strictEqual(pod.serviceAccountName, "agentos-crewmate"); + assert.lengthOf(pod.containers, 2); + const proxy = yield* required( + pod.containers.find(({ name }) => name === "ai-gateway-workload-proxy"), + "Missing workload proxy", + ); + assert.deepStrictEqual(proxy.volumeMounts, [{ + mountPath: "/var/run/secrets/agentos-egress", + name: "agentos-egress-identity", + readOnly: true, + }]); + assert.deepStrictEqual(environment(proxy.env ?? []), { + AGENTOS_EGRESS_TOKEN_FILE: "/var/run/secrets/agentos-egress/token", + AI_GATEWAY_GRACEFUL_SHUTDOWN_MILLIS: "20000", + AI_GATEWAY_IDLE_TIMEOUT_SECONDS: "255", + AI_GATEWAY_URL: "http://agentgateway-openai.agentos.svc.cluster.local:8788", + }); + assert.deepStrictEqual(proxy.securityContext, { + allowPrivilegeEscalation: false, + capabilities: { drop: ["ALL"] }, + readOnlyRootFilesystem: true, + runAsGroup: 1000, + runAsNonRoot: true, + runAsUser: 1000, + seccompProfile: { type: "RuntimeDefault" }, + }); + assert.deepStrictEqual(proxy.resources, { + limits: { cpu: "250m", memory: "256Mi", "ephemeral-storage": "128Mi" }, + requests: { cpu: "25m", memory: "64Mi", "ephemeral-storage": "32Mi" }, + }); + for (const workloadContainer of pod.containers.filter( + ({ name }) => name !== "ai-gateway-workload-proxy", + )) { + const mounts = yield* Schema.decodeUnknownEffect( + Schema.Array(Schema.Struct({ name: Schema.String })), + )(workloadContainer.volumeMounts ?? []); + assert.isUndefined( + mounts.find(({ name }) => name === "agentos-egress-identity"), + ); + } + for (const initContainer of pod.initContainers) { + const mounts = yield* Schema.decodeUnknownEffect( + Schema.Array(Schema.Struct({ name: Schema.String })), + )(initContainer.volumeMounts ?? []); + assert.isUndefined( + mounts.find(({ name }) => name === "agentos-egress-identity"), + ); + } + const identityVolumes = pod.volumes + .filter(({ name }) => name === "agentos-egress-identity"); + assert.deepStrictEqual(identityVolumes, [{ + name: "agentos-egress-identity", + projected: { + defaultMode: 288, + sources: [{ serviceAccountToken: { + audience: AGENTOS_EGRESS_TOKEN_AUDIENCE, + expirationSeconds: AGENTOS_EGRESS_TOKEN_EXPIRATION_SECONDS, + path: "token", + } }], + }, + }]); assert.strictEqual(statefulSet.spec.volumeClaimTemplates[0]?.metadata.name, "home"); }).pipe(Effect.provide(BunServices.layer))); diff --git a/packages/agentos/resources/roles/firstmate/kubernetes/base/kustomization.yaml b/packages/agentos/resources/roles/firstmate/kubernetes/base/kustomization.yaml index e7cfc04d..2fb4622f 100644 --- a/packages/agentos/resources/roles/firstmate/kubernetes/base/kustomization.yaml +++ b/packages/agentos/resources/roles/firstmate/kubernetes/base/kustomization.yaml @@ -9,7 +9,6 @@ resources: - ../../../../../runtime/kubernetes/mate components: - ../../../../../runtime/kubernetes/observability - - ../../../../../runtime/kubernetes/github-provider patches: - path: statefulset.patch.yaml target: diff --git a/packages/agentos/resources/roles/firstmate/kubernetes/base/serviceaccount.yaml b/packages/agentos/resources/roles/firstmate/kubernetes/base/serviceaccount.yaml index 0b259f45..39ca1dbf 100644 --- a/packages/agentos/resources/roles/firstmate/kubernetes/base/serviceaccount.yaml +++ b/packages/agentos/resources/roles/firstmate/kubernetes/base/serviceaccount.yaml @@ -5,3 +5,4 @@ metadata: labels: app.kubernetes.io/name: agentos-firstmate app.kubernetes.io/part-of: agentos +automountServiceAccountToken: false diff --git a/packages/agentos/resources/roles/firstmate/kubernetes/patches/ai-gateway-client.yaml b/packages/agentos/resources/roles/firstmate/kubernetes/patches/ai-gateway-client.yaml index 1e179acc..cf0aacec 100644 --- a/packages/agentos/resources/roles/firstmate/kubernetes/patches/ai-gateway-client.yaml +++ b/packages/agentos/resources/roles/firstmate/kubernetes/patches/ai-gateway-client.yaml @@ -4,9 +4,6 @@ metadata: name: agentos-firstmate spec: template: - metadata: - labels: - agentos.akua.dev/agentgateway-client: "true" spec: initContainers: - name: install-tools @@ -17,7 +14,7 @@ spec: - name: AGENTOS_PROVIDER_CREDENTIAL_KIND value: ai_gateway - name: AI_GATEWAY_URL - value: http://agentgateway-openai.agentos.svc.cluster.local:8788 + value: http://127.0.0.1:8790 containers: - name: agentos env: @@ -26,6 +23,4 @@ spec: - name: AGENTOS_PROVIDER_CREDENTIAL_KIND value: ai_gateway - name: AI_GATEWAY_URL - value: http://agentgateway-openai.agentos.svc.cluster.local:8788 - - name: AGENTOS_EGRESS_TOKEN_FILE - value: /var/run/secrets/agentos-egress/token + value: http://127.0.0.1:8790 diff --git a/packages/agentos/resources/roles/firstmate/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml b/packages/agentos/resources/roles/firstmate/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml index 9ba0c064..5b3363f0 100644 --- a/packages/agentos/resources/roles/firstmate/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml +++ b/packages/agentos/resources/roles/firstmate/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml @@ -2,6 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../../../base +components: + - ../../../../../../../../../services/ai-gateway/kubernetes/workload-proxy/component patches: - path: ../../../patches/ai-gateway-client.yaml target: diff --git a/packages/agentos/resources/roles/firstmate/kubernetes/tests/manifests.effect.test.ts b/packages/agentos/resources/roles/firstmate/kubernetes/tests/manifests.effect.test.ts index fd0569b4..a71d22f8 100644 --- a/packages/agentos/resources/roles/firstmate/kubernetes/tests/manifests.effect.test.ts +++ b/packages/agentos/resources/roles/firstmate/kubernetes/tests/manifests.effect.test.ts @@ -24,6 +24,10 @@ const VolumeMount = Schema.Struct({ name: Schema.String, readOnly: Schema.optional(Schema.Boolean), }); +const Probe = Schema.Struct({ + exec: Schema.optional(Schema.Struct({ command: Schema.Array(Schema.String) })), + httpGet: Schema.optional(Schema.Unknown), +}); const Container = Schema.Struct({ name: Schema.String, image: Schema.String, @@ -33,12 +37,8 @@ const Container = Schema.Struct({ args: Schema.optional(Schema.Array(Schema.String)), env: Schema.optional(Schema.Array(EnvironmentEntry)), volumeMounts: Schema.optional(Schema.Array(VolumeMount)), - livenessProbe: Schema.optional(Schema.Struct({ - exec: Schema.Struct({ command: Schema.Array(Schema.String) }), - })), - readinessProbe: Schema.optional(Schema.Struct({ - exec: Schema.Struct({ command: Schema.Array(Schema.String) }), - })), + livenessProbe: Schema.optional(Probe), + readinessProbe: Schema.optional(Probe), securityContext: Schema.optional(Schema.Unknown), resources: Schema.optional(Schema.Unknown), }); @@ -49,6 +49,7 @@ const Metadata = Schema.Struct({ labels: Schema.optional(Schema.Record(Schema.String, Schema.String)), }); const Resource = Schema.Struct({ + automountServiceAccountToken: Schema.optional(Schema.Boolean), apiVersion: Schema.String, kind: Schema.String, metadata: Metadata, @@ -79,6 +80,7 @@ const StatefulSet = Schema.Struct({ labels: Schema.Record(Schema.String, Schema.String), }), spec: Schema.Struct({ + automountServiceAccountToken: Schema.Boolean, serviceAccountName: Schema.String, securityContext: Schema.Unknown, initContainers: Schema.Array(Container), @@ -209,7 +211,6 @@ describe("First Mate Kubernetes resources", () => { }, labels: { "agentos.akua.dev/agent": "firstmate", - "agentos.akua.dev/github-client": "true", "agentos.akua.dev/otel-client": "true", "app.kubernetes.io/name": "agentos-firstmate", "app.kubernetes.io/part-of": "agentos", @@ -223,7 +224,10 @@ describe("First Mate Kubernetes resources", () => { runAsUser: 1000, seccompProfile: { type: "RuntimeDefault" }, }); - assert.lengthOf(pod.initContainers, 3); + assert.isFalse(pod.automountServiceAccountToken); + assert.isFalse((yield* resource(resources, "ServiceAccount", "agentos-firstmate")) + .automountServiceAccountToken ?? true); + assert.lengthOf(pod.initContainers, 2); assert.lengthOf(pod.containers, 1); const install = yield* required(pod.initContainers[0], "Missing installer"); const prepare = yield* required(pod.initContainers[1], "Missing prepare"); @@ -241,20 +245,13 @@ describe("First Mate Kubernetes resources", () => { ); assert.deepStrictEqual(install.volumeMounts, [ { mountPath: "/home/agent", name: "home" }, + { mountPath: "/tmp", name: "tmp" }, + ]); + assert.deepStrictEqual(prepare.volumeMounts, [ + { mountPath: "/home/agent", name: "home" }, ]); - assert.deepStrictEqual(prepare.volumeMounts, install.volumeMounts); assert.deepStrictEqual(firstMate.volumeMounts, [ ...(install.volumeMounts ?? []), - { - mountPath: "/var/run/secrets/agentos-egress", - name: "agentos-egress-identity", - readOnly: true, - }, - { - mountPath: "/var/run/config/agentos-github", - name: "agentos-github-ca", - readOnly: true, - }, ]); assert.deepStrictEqual(install.command, ["mise"]); assert.deepStrictEqual(install.args, [ @@ -296,20 +293,11 @@ describe("First Mate Kubernetes resources", () => { assert.deepStrictEqual(firstMate.securityContext, { allowPrivilegeEscalation: false, capabilities: { drop: ["ALL"] }, + readOnlyRootFilesystem: true, + runAsNonRoot: true, }); - assert.deepInclude(pod.volumes, { - name: "agentos-egress-identity", - projected: { - defaultMode: 288, - sources: [{ - serviceAccountToken: { - audience: AGENTOS_EGRESS_TOKEN_AUDIENCE, - expirationSeconds: AGENTOS_EGRESS_TOKEN_EXPIRATION_SECONDS, - path: "token", - }, - }], - }, - }); + assert.isFalse(JSON.stringify(pod).includes("agentos-egress-identity")); + assert.isFalse(JSON.stringify(pod).includes("AGENTOS_GITHUB_")); assert.deepStrictEqual( resources.filter(({ kind }) => ["Ingress", "LoadBalancer", "NodePort", "ClusterRoleBinding"].includes(kind) @@ -478,7 +466,7 @@ describe("First Mate Kubernetes resources", () => { ).pipe(Effect.flatMap(statefulSet)); assert.deepStrictEqual( workload.spec.template.spec.initContainers.map(({ name }) => name), - ["install-tools", "prepare-home", "prepare-github-provider"], + ["install-tools", "prepare-home"], ); }))); @@ -506,33 +494,80 @@ describe("First Mate Kubernetes resources", () => { assert.deepStrictEqual(pod.initContainers.map(({ name }) => name), [ "install-tools", "prepare-home", - "prepare-github-provider", ]); assert.deepInclude(spec.template.metadata.labels, { "agentos.akua.dev/agentgateway-client": "true", }); assert.strictEqual( variables.AI_GATEWAY_URL, - "http://agentgateway-openai.agentos.svc.cluster.local:8788", + "http://127.0.0.1:8790", ); assert.isUndefined(variables.AI_GATEWAY_TOKEN); - assert.strictEqual( - variables.AGENTOS_EGRESS_TOKEN_FILE, - "/var/run/secrets/agentos-egress/token", - ); + assert.isUndefined(variables.AGENTOS_EGRESS_TOKEN_FILE); assert.strictEqual(variables.AGENTOS_PI_PROVIDER_MODE, "ai-gateway"); assert.strictEqual(variables.AGENTOS_PROVIDER_CREDENTIAL_KIND, "ai_gateway"); assert.deepInclude(prepareVariables, { AGENTOS_PI_PROVIDER_MODE: "ai-gateway", - AI_GATEWAY_URL: - "http://agentgateway-openai.agentos.svc.cluster.local:8788", + AI_GATEWAY_URL: "http://127.0.0.1:8790", }); assert.isUndefined(prepareVariables.AI_GATEWAY_TOKEN); + assert.isUndefined(prepareVariables.AGENTOS_EGRESS_TOKEN_FILE); assert.isUndefined(variables.AGENTOS_MODEL); assert.isUndefined(variables.AGENTOS_THINKING); assert.isUndefined(prepareVariables.AGENTOS_MODEL); assert.isUndefined(prepareVariables.AGENTOS_THINKING); assert.strictEqual(pod.serviceAccountName, "agentos-firstmate"); + const proxy = yield* required( + pod.containers.find(({ name }) => name === "ai-gateway-workload-proxy"), + "Missing workload proxy", + ); + assert.deepStrictEqual(proxy.volumeMounts, [{ + mountPath: "/var/run/secrets/agentos-egress", + name: "agentos-egress-identity", + readOnly: true, + }]); + assert.deepStrictEqual(environment(proxy.env ?? []), { + AGENTOS_EGRESS_TOKEN_FILE: "/var/run/secrets/agentos-egress/token", + AI_GATEWAY_GRACEFUL_SHUTDOWN_MILLIS: "20000", + AI_GATEWAY_IDLE_TIMEOUT_SECONDS: "255", + AI_GATEWAY_URL: "http://agentgateway-openai.agentos.svc.cluster.local:8788", + }); + assert.deepStrictEqual(proxy.livenessProbe, { + httpGet: { host: "127.0.0.1", path: "/livez", port: "workload-proxy" }, + }); + assert.deepStrictEqual(proxy.readinessProbe, { + httpGet: { host: "127.0.0.1", path: "/readyz", port: "workload-proxy" }, + }); + assert.deepStrictEqual(proxy.securityContext, { + allowPrivilegeEscalation: false, + capabilities: { drop: ["ALL"] }, + readOnlyRootFilesystem: true, + runAsGroup: 1000, + runAsNonRoot: true, + runAsUser: 1000, + seccompProfile: { type: "RuntimeDefault" }, + }); + assert.deepStrictEqual(proxy.resources, { + limits: { cpu: "250m", memory: "256Mi", "ephemeral-storage": "128Mi" }, + requests: { cpu: "25m", memory: "64Mi", "ephemeral-storage": "32Mi" }, + }); + assert.deepStrictEqual(pod.volumes.filter((volume) => + JSON.stringify(volume).includes("agentos-egress-identity") + ), [{ + name: "agentos-egress-identity", + projected: { + defaultMode: 288, + sources: [{ serviceAccountToken: { + audience: AGENTOS_EGRESS_TOKEN_AUDIENCE, + expirationSeconds: AGENTOS_EGRESS_TOKEN_EXPIRATION_SECONDS, + path: "token", + } }], + }, + }]); + for (const candidate of [...pod.initContainers, firstMate]) { + assert.isFalse(JSON.stringify(candidate).includes("agentos-egress-identity")); + assert.isFalse(JSON.stringify(candidate).includes("AGENTOS_EGRESS_TOKEN_FILE")); + } assert.strictEqual(spec.volumeClaimTemplates[0]?.metadata.name, "home"); }))); @@ -559,7 +594,6 @@ describe("First Mate Kubernetes resources", () => { assert.deepStrictEqual(pod.initContainers.map(({ name }) => name), [ "install-tools", "prepare-home", - "prepare-github-provider", ]); assert.strictEqual(prepareVariables.AGENTOS_PI_PROVIDER_MODE, "direct"); assert.isUndefined(prepareVariables.AI_GATEWAY_URL); diff --git a/packages/agentos/resources/roles/secondmate/kubernetes/AGENTS.md b/packages/agentos/resources/roles/secondmate/kubernetes/AGENTS.md index a76d9b8c..f0096fa3 100644 --- a/packages/agentos/resources/roles/secondmate/kubernetes/AGENTS.md +++ b/packages/agentos/resources/roles/secondmate/kubernetes/AGENTS.md @@ -11,10 +11,9 @@ and its reviewed managed-domain composition. working directory, tasks and credentials here. - Require a distinct ServiceAccount, retained home PVC, database identity and Herdr session for each Second Mate. -- Explicitly mount the kubelet-rotated projected ServiceAccount identity in - every persistent Second-Mate Pod. Native in-cluster `kubectl` must use that - identity; never substitute a separately minted bearer token as steady-state - supervision authentication. +- Disable default ServiceAccount token automount for every persistent + Second-Mate Pod and ServiceAccount. Hermes is the control plane; do not + preserve a Kubernetes API token in the Mate workload as a supervision path. - Keep child authority out of `base/`. The `domain/` composition binds the Second Mate to namespaced Crewmate workload operations while withholding Namespace, Secret, RBAC, quota, LimitRange, NetworkPolicy and cluster-scoped diff --git a/packages/agentos/resources/roles/secondmate/kubernetes/admission/validatingadmissionpolicies.yaml b/packages/agentos/resources/roles/secondmate/kubernetes/admission/validatingadmissionpolicies.yaml index 41eb539b..8718f8f3 100644 --- a/packages/agentos/resources/roles/secondmate/kubernetes/admission/validatingadmissionpolicies.yaml +++ b/packages/agentos/resources/roles/secondmate/kubernetes/admission/validatingadmissionpolicies.yaml @@ -40,7 +40,7 @@ spec: has(variables.podSpec.volumes) ? variables.podSpec.volumes.filter(volume, has(volume.projected) && has(volume.projected.sources) && volume.projected.sources.exists(source, has(source.serviceAccountToken))) : [] - name: egressIdentityValid expression: >- - size(variables.egressIdentityVolumes) == 1 && variables.egressIdentityVolumes[0].name == 'agentos-egress-identity' && has(variables.egressIdentityVolumes[0].projected.defaultMode) && variables.egressIdentityVolumes[0].projected.defaultMode == 288 && size(variables.egressIdentityVolumes[0].projected.sources) == 1 && has(variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken) && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.audience == 'agentos-egress-authz' && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.expirationSeconds == 600 && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.path == 'token' && variables.podSpec.containers.all(container, has(container.volumeMounts) && container.volumeMounts.exists(mount, mount.name == 'agentos-egress-identity' && mount.mountPath == '/var/run/secrets/agentos-egress' && has(mount.readOnly) && mount.readOnly == true)) && (!has(variables.podSpec.initContainers) || variables.podSpec.initContainers.all(container, !has(container.volumeMounts) || container.volumeMounts.all(mount, mount.name != 'agentos-egress-identity'))) + size(variables.egressIdentityVolumes) == 1 && variables.egressIdentityVolumes[0].name == 'agentos-egress-identity' && has(variables.egressIdentityVolumes[0].projected.defaultMode) && variables.egressIdentityVolumes[0].projected.defaultMode == 288 && size(variables.egressIdentityVolumes[0].projected.sources) == 1 && has(variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken) && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.audience == 'agentos-egress-authz' && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.expirationSeconds == 600 && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.path == 'token' && variables.podSpec.containers.exists_one(container, container.name == 'ai-gateway-workload-proxy' && has(container.volumeMounts) && container.volumeMounts.exists(mount, mount.name == 'agentos-egress-identity' && mount.mountPath == '/var/run/secrets/agentos-egress' && has(mount.readOnly) && mount.readOnly == true)) && variables.podSpec.containers.all(container, container.name == 'ai-gateway-workload-proxy' || !has(container.volumeMounts) || container.volumeMounts.all(mount, mount.name != 'agentos-egress-identity')) && (!has(variables.podSpec.initContainers) || variables.podSpec.initContainers.all(container, !has(container.volumeMounts) || container.volumeMounts.all(mount, mount.name != 'agentos-egress-identity'))) validations: - expression: >- !variables.persistentMate || !request.userInfo.username.startsWith('system:serviceaccount:' + request.namespace + ':') @@ -133,7 +133,7 @@ spec: has(object.spec.volumes) ? object.spec.volumes.filter(volume, has(volume.projected) && has(volume.projected.sources) && volume.projected.sources.exists(source, has(source.serviceAccountToken))) : [] - name: egressIdentityValid expression: >- - size(variables.egressIdentityVolumes) == 1 && variables.egressIdentityVolumes[0].name == 'agentos-egress-identity' && has(variables.egressIdentityVolumes[0].projected.defaultMode) && variables.egressIdentityVolumes[0].projected.defaultMode == 288 && size(variables.egressIdentityVolumes[0].projected.sources) == 1 && has(variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken) && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.audience == 'agentos-egress-authz' && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.expirationSeconds == 600 && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.path == 'token' && object.spec.containers.all(container, has(container.volumeMounts) && container.volumeMounts.exists(mount, mount.name == 'agentos-egress-identity' && mount.mountPath == '/var/run/secrets/agentos-egress' && has(mount.readOnly) && mount.readOnly == true)) && (!has(object.spec.initContainers) || object.spec.initContainers.all(container, !has(container.volumeMounts) || container.volumeMounts.all(mount, mount.name != 'agentos-egress-identity'))) + size(variables.egressIdentityVolumes) == 1 && variables.egressIdentityVolumes[0].name == 'agentos-egress-identity' && has(variables.egressIdentityVolumes[0].projected.defaultMode) && variables.egressIdentityVolumes[0].projected.defaultMode == 288 && size(variables.egressIdentityVolumes[0].projected.sources) == 1 && has(variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken) && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.audience == 'agentos-egress-authz' && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.expirationSeconds == 600 && variables.egressIdentityVolumes[0].projected.sources[0].serviceAccountToken.path == 'token' && object.spec.containers.exists_one(container, container.name == 'ai-gateway-workload-proxy' && has(container.volumeMounts) && container.volumeMounts.exists(mount, mount.name == 'agentos-egress-identity' && mount.mountPath == '/var/run/secrets/agentos-egress' && has(mount.readOnly) && mount.readOnly == true)) && object.spec.containers.all(container, container.name == 'ai-gateway-workload-proxy' || !has(container.volumeMounts) || container.volumeMounts.all(mount, mount.name != 'agentos-egress-identity')) && (!has(object.spec.initContainers) || object.spec.initContainers.all(container, !has(container.volumeMounts) || container.volumeMounts.all(mount, mount.name != 'agentos-egress-identity'))) validations: - expression: '!variables.crewmate || (variables.identityKeys.all(key, key in object.metadata.labels && object.metadata.labels[key].matches(''^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'')))' message: Crewmate Pods require UUID Agent, owner, Task, and Assignment labels. diff --git a/packages/agentos/resources/roles/secondmate/kubernetes/base/kustomization.yaml b/packages/agentos/resources/roles/secondmate/kubernetes/base/kustomization.yaml index ff553765..a7b027d7 100644 --- a/packages/agentos/resources/roles/secondmate/kubernetes/base/kustomization.yaml +++ b/packages/agentos/resources/roles/secondmate/kubernetes/base/kustomization.yaml @@ -6,7 +6,6 @@ resources: - ../../../../../runtime/kubernetes/mate components: - ../../../../../runtime/kubernetes/observability - - ../../../../../runtime/kubernetes/github-provider patches: - path: statefulset.patch.yaml target: diff --git a/packages/agentos/resources/roles/secondmate/kubernetes/base/serviceaccount.yaml b/packages/agentos/resources/roles/secondmate/kubernetes/base/serviceaccount.yaml index eacd8789..b20d2156 100644 --- a/packages/agentos/resources/roles/secondmate/kubernetes/base/serviceaccount.yaml +++ b/packages/agentos/resources/roles/secondmate/kubernetes/base/serviceaccount.yaml @@ -5,3 +5,4 @@ metadata: labels: app.kubernetes.io/name: agentos-secondmate app.kubernetes.io/part-of: agentos +automountServiceAccountToken: false diff --git a/packages/agentos/resources/roles/secondmate/kubernetes/base/statefulset.patch.yaml b/packages/agentos/resources/roles/secondmate/kubernetes/base/statefulset.patch.yaml index 4edac063..ec44cb18 100644 --- a/packages/agentos/resources/roles/secondmate/kubernetes/base/statefulset.patch.yaml +++ b/packages/agentos/resources/roles/secondmate/kubernetes/base/statefulset.patch.yaml @@ -17,7 +17,7 @@ spec: agentos.akua.dev/agent: secondmate app.kubernetes.io/name: agentos-secondmate spec: - automountServiceAccountToken: true + automountServiceAccountToken: false serviceAccountName: agentos-secondmate initContainers: - name: install-tools diff --git a/packages/agentos/resources/roles/secondmate/kubernetes/patches/ai-gateway-client.yaml b/packages/agentos/resources/roles/secondmate/kubernetes/patches/ai-gateway-client.yaml index 4de53f7a..2daf88ee 100644 --- a/packages/agentos/resources/roles/secondmate/kubernetes/patches/ai-gateway-client.yaml +++ b/packages/agentos/resources/roles/secondmate/kubernetes/patches/ai-gateway-client.yaml @@ -4,9 +4,6 @@ metadata: name: agentos-secondmate spec: template: - metadata: - labels: - agentos.akua.dev/agentgateway-client: "true" spec: initContainers: - name: install-tools @@ -17,7 +14,7 @@ spec: - name: AGENTOS_PROVIDER_CREDENTIAL_KIND value: ai_gateway - name: AI_GATEWAY_URL - value: http://agentgateway-openai.agentos.svc.cluster.local:8788 + value: http://127.0.0.1:8790 containers: - name: agentos env: @@ -26,6 +23,4 @@ spec: - name: AGENTOS_PROVIDER_CREDENTIAL_KIND value: ai_gateway - name: AI_GATEWAY_URL - value: http://agentgateway-openai.agentos.svc.cluster.local:8788 - - name: AGENTOS_EGRESS_TOKEN_FILE - value: /var/run/secrets/agentos-egress/token + value: http://127.0.0.1:8790 diff --git a/packages/agentos/resources/roles/secondmate/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml b/packages/agentos/resources/roles/secondmate/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml index af82cf6e..132e93a9 100644 --- a/packages/agentos/resources/roles/secondmate/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml +++ b/packages/agentos/resources/roles/secondmate/kubernetes/tests/fixtures/ai-gateway-client/kustomization.yaml @@ -3,6 +3,8 @@ kind: Kustomization namespace: agentos resources: - ../../../base +components: + - ../../../../../../../../../services/ai-gateway/kubernetes/workload-proxy/component patches: - path: ../../../patches/ai-gateway-client.yaml target: diff --git a/packages/agentos/resources/roles/secondmate/kubernetes/tests/fixtures/lifecycle-child/statefulset.patch.yaml b/packages/agentos/resources/roles/secondmate/kubernetes/tests/fixtures/lifecycle-child/statefulset.patch.yaml index 6d4704b6..11ca410d 100644 --- a/packages/agentos/resources/roles/secondmate/kubernetes/tests/fixtures/lifecycle-child/statefulset.patch.yaml +++ b/packages/agentos/resources/roles/secondmate/kubernetes/tests/fixtures/lifecycle-child/statefulset.patch.yaml @@ -28,6 +28,23 @@ spec: volumeMounts: - name: home mountPath: /home/agent + - name: ai-gateway-workload-proxy + image: registry.k8s.io/pause@sha256:278fb9dbcca9518083ad1e11276933a2e96f23de604a3a08cc3c80002767d24c + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 250m + memory: 128Mi + requests: + cpu: 25m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: - name: agentos-egress-identity mountPath: /var/run/secrets/agentos-egress readOnly: true diff --git a/packages/agentos/resources/roles/secondmate/kubernetes/tests/manifest.effect.test.ts b/packages/agentos/resources/roles/secondmate/kubernetes/tests/manifest.effect.test.ts index 72fb6266..9ba0c204 100644 --- a/packages/agentos/resources/roles/secondmate/kubernetes/tests/manifest.effect.test.ts +++ b/packages/agentos/resources/roles/secondmate/kubernetes/tests/manifest.effect.test.ts @@ -31,8 +31,17 @@ const Container = Schema.Struct({ limits: Schema.Record(Schema.String, Schema.String), requests: Schema.Record(Schema.String, Schema.String), })), + livenessProbe: Schema.optional(Schema.Unknown), + readinessProbe: Schema.optional(Schema.Unknown), + securityContext: Schema.optional(Schema.Unknown), volumeMounts: Schema.optional(Schema.Array(VolumeMount)), }); +const Volume = Schema.Struct({ + name: Schema.String, + projected: Schema.optional(Schema.Unknown), + secret: Schema.optional(Schema.Unknown), + emptyDir: Schema.optional(Schema.Unknown), +}); const Metadata = Schema.Struct({ labels: Schema.optional(Schema.Record(Schema.String, Schema.String)), name: Schema.String, @@ -54,6 +63,7 @@ const Subject = Schema.Struct({ namespace: Schema.optional(Schema.String), }); const Resource = Schema.Struct({ + automountServiceAccountToken: Schema.optional(Schema.Boolean), kind: Schema.String, metadata: Metadata, roleRef: Schema.optional(RoleReference), @@ -79,7 +89,7 @@ const StatefulSet = Schema.Struct({ serviceAccountName: Schema.String, automountServiceAccountToken: Schema.Boolean, securityContext: Schema.Unknown, - volumes: Schema.Unknown, + volumes: Schema.Array(Volume), initContainers: Schema.Array(Container), containers: Schema.Array(Container), }), @@ -176,7 +186,9 @@ describe("Second Mate Kubernetes base", () => { ); const pod = workload.spec.template.spec; assert.strictEqual(pod.serviceAccountName, "agentos-secondmate"); - assert.isTrue(pod.automountServiceAccountToken); + assert.isFalse(pod.automountServiceAccountToken); + assert.isFalse((yield* namedResource(resources, "ServiceAccount", "agentos-secondmate")) + .automountServiceAccountToken ?? true); assert.deepStrictEqual(pod.securityContext, { fsGroup: 1000, fsGroupChangePolicy: "OnRootMismatch", @@ -193,29 +205,9 @@ describe("Second Mate Kubernetes base", () => { secretName: "agentos-secondmate-postgres", }, }, - { - name: "agentos-egress-identity", - projected: { - defaultMode: 288, - sources: [{ - serviceAccountToken: { - audience: AGENTOS_EGRESS_TOKEN_AUDIENCE, - expirationSeconds: AGENTOS_EGRESS_TOKEN_EXPIRATION_SECONDS, - path: "token", - }, - }], - }, - }, - { - name: "agentos-github-ca", - configMap: { - defaultMode: 292, - items: [{ key: "ca.pem", path: "ca.pem" }], - name: "agentos-github-ca", - }, - }, + { name: "tmp", emptyDir: { sizeLimit: "256Mi" } }, ]); - assert.lengthOf(pod.initContainers, 3); + assert.lengthOf(pod.initContainers, 2); assert.lengthOf(pod.containers, 1); const container = yield* required( pod.containers[0], @@ -226,24 +218,12 @@ describe("Second Mate Kubernetes base", () => { "agentos:dev", "agentos:dev", "agentos:dev", - "agentos:dev", ]); - const githubProvider = yield* required( - pod.initContainers.find(({ name }) => - name === "prepare-github-provider" - ), - "Missing GitHub provider preparation container", - ); - assert.deepStrictEqual(githubProvider.resources, { - limits: { cpu: "250m", memory: "128Mi" }, - requests: { cpu: "25m", memory: "64Mi" }, - }); assert.deepStrictEqual( allContainers.map(({ workingDir }) => workingDir), [ "/opt/agentos/packages/agentos/resources/roles/secondmate", "/opt/agentos/packages/agentos/resources/roles/secondmate", - undefined, "/opt/agentos/packages/agentos/resources/roles/secondmate", ], ); @@ -271,11 +251,10 @@ describe("Second Mate Kubernetes base", () => { ); assert.isUndefined(variables.AGENTOS_MODEL); assert.isUndefined(variables.AGENTOS_THINKING); - assert.deepInclude(container.volumeMounts ?? [], { - mountPath: "/var/run/secrets/agentos-egress", - name: "agentos-egress-identity", - readOnly: true, - }); + assert.isFalse((container.volumeMounts ?? []).some(({ name }) => + name === "agentos-egress-identity" + )); + assert.isFalse(JSON.stringify(pod).includes("AGENTOS_GITHUB_")); for (const initContainer of pod.initContainers) { assert.isFalse( (initContainer.volumeMounts ?? []).some(({ name }) => @@ -314,33 +293,77 @@ describe("Second Mate Kubernetes base", () => { assert.deepStrictEqual(pod.initContainers.map(({ name }) => name), [ "install-tools", "prepare-home", - "prepare-github-provider", ]); assert.deepInclude(spec.template.metadata.labels, { "agentos.akua.dev/agentgateway-client": "true", }); assert.strictEqual( variables.AI_GATEWAY_URL, - "http://agentgateway-openai.agentos.svc.cluster.local:8788", + "http://127.0.0.1:8790", ); assert.isUndefined(variables.AI_GATEWAY_TOKEN); - assert.strictEqual( - variables.AGENTOS_EGRESS_TOKEN_FILE, - "/var/run/secrets/agentos-egress/token", - ); + assert.isUndefined(variables.AGENTOS_EGRESS_TOKEN_FILE); assert.strictEqual(variables.AGENTOS_PI_PROVIDER_MODE, "ai-gateway"); assert.strictEqual(variables.AGENTOS_PROVIDER_CREDENTIAL_KIND, "ai_gateway"); assert.deepInclude(prepareVariables, { AGENTOS_PI_PROVIDER_MODE: "ai-gateway", - AI_GATEWAY_URL: - "http://agentgateway-openai.agentos.svc.cluster.local:8788", + AI_GATEWAY_URL: "http://127.0.0.1:8790", }); assert.isUndefined(prepareVariables.AI_GATEWAY_TOKEN); + assert.isUndefined(prepareVariables.AGENTOS_EGRESS_TOKEN_FILE); assert.isUndefined(variables.AGENTOS_MODEL); assert.isUndefined(variables.AGENTOS_THINKING); assert.isUndefined(prepareVariables.AGENTOS_MODEL); assert.isUndefined(prepareVariables.AGENTOS_THINKING); assert.strictEqual(pod.serviceAccountName, "agentos-secondmate"); + const proxy = yield* required( + pod.containers.find(({ name }) => name === "ai-gateway-workload-proxy"), + "Missing workload proxy", + ); + assert.deepStrictEqual(proxy.volumeMounts, [{ + mountPath: "/var/run/secrets/agentos-egress", + name: "agentos-egress-identity", + readOnly: true, + }]); + assert.deepStrictEqual(environment(proxy.env ?? []), { + AGENTOS_EGRESS_TOKEN_FILE: "/var/run/secrets/agentos-egress/token", + AI_GATEWAY_GRACEFUL_SHUTDOWN_MILLIS: "20000", + AI_GATEWAY_IDLE_TIMEOUT_SECONDS: "255", + AI_GATEWAY_URL: "http://agentgateway-openai.agentos.svc.cluster.local:8788", + }); + assert.deepStrictEqual(proxy.securityContext, { + allowPrivilegeEscalation: false, + capabilities: { drop: ["ALL"] }, + readOnlyRootFilesystem: true, + runAsGroup: 1000, + runAsNonRoot: true, + runAsUser: 1000, + seccompProfile: { type: "RuntimeDefault" }, + }); + assert.deepStrictEqual(proxy.resources, { + limits: { cpu: "250m", memory: "256Mi", "ephemeral-storage": "128Mi" }, + requests: { cpu: "25m", memory: "64Mi", "ephemeral-storage": "32Mi" }, + }); + assert.deepStrictEqual( + pod.volumes.filter(({ name }) => + name === "agentos-egress-identity" + ), + [{ + name: "agentos-egress-identity", + projected: { + defaultMode: 288, + sources: [{ serviceAccountToken: { + audience: AGENTOS_EGRESS_TOKEN_AUDIENCE, + expirationSeconds: AGENTOS_EGRESS_TOKEN_EXPIRATION_SECONDS, + path: "token", + } }], + }, + }], + ); + for (const candidate of [...pod.initContainers, container]) { + assert.isFalse(JSON.stringify(candidate).includes("agentos-egress-identity")); + assert.isFalse(JSON.stringify(candidate).includes("AGENTOS_EGRESS_TOKEN_FILE")); + } assert.strictEqual(spec.volumeClaimTemplates[0]?.metadata.name, "home"); }))); @@ -367,7 +390,6 @@ describe("Second Mate Kubernetes base", () => { assert.deepStrictEqual(pod.initContainers.map(({ name }) => name), [ "install-tools", "prepare-home", - "prepare-github-provider", ]); assert.strictEqual(prepareVariables.AGENTOS_PI_PROVIDER_MODE, "direct"); assert.isUndefined(prepareVariables.AI_GATEWAY_URL); diff --git a/packages/agentos/runtime/codex-provider.ts b/packages/agentos/runtime/codex-provider.ts index dc2a23b7..68228538 100644 --- a/packages/agentos/runtime/codex-provider.ts +++ b/packages/agentos/runtime/codex-provider.ts @@ -15,25 +15,16 @@ import { export const CODEX_GATEWAY_PROVIDER_ID = "agentos-gateway"; const providerId = CODEX_GATEWAY_PROVIDER_ID; const markerVersion = 1; -const defaultTokenFile = "/var/run/secrets/agentos-egress/token"; const JsonObject = Schema.Record(Schema.String, Schema.Unknown); const GatewayProviderEntry = Schema.Struct({ name: Schema.Literal("AgentOS workload gateway"), base_url: Schema.String, wire_api: Schema.Literal("responses"), + requires_openai_auth: Schema.Literal(false), supports_websockets: Schema.Literal(false), request_max_retries: Schema.Literal(0), stream_max_retries: Schema.Literal(0), - env_http_headers: Schema.Struct({ - "X-AgentOS-Assignment-Id": Schema.Literal("AGENTOS_ASSIGNMENT_ID"), - }), - auth: Schema.Struct({ - command: Schema.String, - args: Schema.Array(Schema.String), - timeout_ms: Schema.Literal(5_000), - refresh_interval_ms: Schema.Literal(60_000), - }), }); class ActiveMarker extends Schema.TaggedClass()("Active", { @@ -178,57 +169,14 @@ export const codexGatewayProviderEntry = Effect.fn( if (rawUrl === undefined) { return yield* configurationError("AI_GATEWAY_URL must be configured"); } - const assignmentId = yield* optionalEnvironment( - environment, - "AGENTOS_ASSIGNMENT_ID", - ); - if ( - assignmentId === undefined || - !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( - assignmentId, - ) - ) { - return yield* configurationError( - "AGENTOS_ASSIGNMENT_ID must be a UUID v4 in Gateway mode", - ); - } - const home = yield* requiredAbsolutePath(environment, paths, "HOME"); - const releaseRoot = yield* requiredAbsolutePath( - environment, - paths, - "AGENTOS_RELEASE_ROOT", - ); - const tokenFile = yield* requiredAbsolutePath( - environment, - paths, - "AGENTOS_EGRESS_TOKEN_FILE", - defaultTokenFile, - ); const entry = { name: "AgentOS workload gateway", base_url: yield* normalizedGatewayUrl(rawUrl), wire_api: "responses", + requires_openai_auth: false, supports_websockets: false, request_max_retries: 0, stream_max_retries: 0, - env_http_headers: { - "X-AgentOS-Assignment-Id": "AGENTOS_ASSIGNMENT_ID", - }, - auth: { - command: paths.join(home, ".local", "share", "mise", "shims", "bun"), - args: [ - paths.join( - releaseRoot, - "packages", - "agentos", - "runtime", - "codex-token.ts", - ), - tokenFile, - ], - timeout_ms: 5_000, - refresh_interval_ms: 60_000, - }, }; return yield* Schema.decodeUnknownEffect(GatewayProviderEntry)(entry).pipe( Effect.mapError(() => configurationError("Invalid workload Gateway provider")), @@ -398,16 +346,10 @@ function renderProvider(entry: GatewayProviderEntry): string { `name = ${JSON.stringify(entry.name)}`, `base_url = ${JSON.stringify(entry.base_url)}`, `wire_api = ${JSON.stringify(entry.wire_api)}`, + `requires_openai_auth = ${entry.requires_openai_auth}`, `supports_websockets = ${entry.supports_websockets}`, `request_max_retries = ${entry.request_max_retries}`, `stream_max_retries = ${entry.stream_max_retries}`, - `env_http_headers = { "X-AgentOS-Assignment-Id" = "AGENTOS_ASSIGNMENT_ID" }`, - "", - `[model_providers.${providerId}.auth]`, - `command = ${JSON.stringify(entry.auth.command)}`, - `args = ${JSON.stringify(entry.auth.args)}`, - `timeout_ms = ${entry.auth.timeout_ms}`, - `refresh_interval_ms = ${entry.auth.refresh_interval_ms}`, ].join("\n"); } diff --git a/packages/agentos/runtime/kubernetes/base/statefulset.yaml b/packages/agentos/runtime/kubernetes/base/statefulset.yaml index 04aeb7e0..c764ee32 100644 --- a/packages/agentos/runtime/kubernetes/base/statefulset.yaml +++ b/packages/agentos/runtime/kubernetes/base/statefulset.yaml @@ -24,6 +24,7 @@ spec: app.kubernetes.io/name: agentos-agent app.kubernetes.io/part-of: agentos spec: + automountServiceAccountToken: false serviceAccountName: agentos-agent terminationGracePeriodSeconds: 30 securityContext: @@ -77,9 +78,22 @@ spec: capabilities: drop: - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + resources: &runtime-resources + requests: + cpu: 100m + memory: 256Mi + ephemeral-storage: 128Mi + limits: + cpu: "2" + memory: 2Gi + ephemeral-storage: 1Gi volumeMounts: &home-volume - name: home mountPath: /home/agent + - name: tmp + mountPath: /tmp containers: - name: agentos image: agentos:dev @@ -93,12 +107,12 @@ spec: - agentos-agent env: *runtime-environment securityContext: *container-security + resources: *runtime-resources volumeMounts: - name: home mountPath: /home/agent - - name: agentos-egress-identity - mountPath: /var/run/secrets/agentos-egress - readOnly: true + - name: tmp + mountPath: /tmp livenessProbe: exec: command: @@ -124,14 +138,9 @@ spec: timeoutSeconds: 3 failureThreshold: 3 volumes: - - name: agentos-egress-identity - projected: - defaultMode: 0440 - sources: - - serviceAccountToken: - audience: agentos-egress-authz - expirationSeconds: 600 - path: token + - name: tmp + emptyDir: + sizeLimit: 256Mi volumeClaimTemplates: - metadata: name: home diff --git a/packages/agentos/runtime/kubernetes/github-provider/kustomization.yaml b/packages/agentos/runtime/kubernetes/github-provider/kustomization.yaml index d1cc9a61..7f08cf91 100644 --- a/packages/agentos/runtime/kubernetes/github-provider/kustomization.yaml +++ b/packages/agentos/runtime/kubernetes/github-provider/kustomization.yaml @@ -49,13 +49,17 @@ patches: capabilities: drop: - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true resources: requests: cpu: 25m memory: 64Mi + ephemeral-storage: 32Mi limits: cpu: 250m memory: 128Mi + ephemeral-storage: 128Mi volumeMounts: - name: home mountPath: /home/agent diff --git a/packages/agentos/runtime/readiness.ts b/packages/agentos/runtime/readiness.ts index 45c394d5..c3f0ac43 100644 --- a/packages/agentos/runtime/readiness.ts +++ b/packages/agentos/runtime/readiness.ts @@ -443,12 +443,19 @@ const verifyCredential = Effect.fn("agentos.readiness.verifyCredential")( : "pi_auth"); let available: boolean; if (kind === "ai_gateway") { - const tokenPath = - requiredEnvironment(environment, "AGENTOS_EGRESS_TOKEN_FILE") ?? - defaultEgressTokenPath; - available = - gatewayMetadataValid(environment) && - isSecureProjectedToken(yield* probeRuntime.metadata(tokenPath)); + if ( + requiredEnvironment(environment, "AGENTOS_CODEX_PROVIDER_MODE") === + "ai-gateway" + ) { + available = gatewayMetadataValid(environment); + } else { + const tokenPath = + requiredEnvironment(environment, "AGENTOS_EGRESS_TOKEN_FILE") ?? + defaultEgressTokenPath; + available = + gatewayMetadataValid(environment) && + isSecureProjectedToken(yield* probeRuntime.metadata(tokenPath)); + } } else if (kind === "pi_auth") { const directory = requiredEnvironment(environment, "PI_CODING_AGENT_DIR") ?? diff --git a/packages/agentos/runtime/tests/codex-provider.effect.test.ts b/packages/agentos/runtime/tests/codex-provider.effect.test.ts index d0b85225..0503a006 100644 --- a/packages/agentos/runtime/tests/codex-provider.effect.test.ts +++ b/packages/agentos/runtime/tests/codex-provider.effect.test.ts @@ -24,10 +24,9 @@ const platform = Layer.merge( const gatewayEnvironment = { AGENTOS_ASSIGNMENT_ID: "20000000-0000-4000-8000-000000000001", AGENTOS_CODEX_PROVIDER_MODE: "ai-gateway", - AGENTOS_EGRESS_TOKEN_FILE: "/var/run/secrets/agentos-egress/token", AGENTOS_RELEASE_ROOT: "/opt/agentos", AI_GATEWAY_URL: - "http://agentgateway-openai.agentos.svc.cluster.local:8788/", + "http://127.0.0.1:8790/", HOME: "/home/agent", }; @@ -123,18 +122,12 @@ describe("Effect Codex workload-authenticated provider", () => { assert.include(source, 'model_provider = "agentos-gateway"'); assert.include(source, '[projects."/workspace"]'); assert.include(source, 'trust_level = "trusted"'); + assert.include(source, "requires_openai_auth = false"); assert.include( source, - '[model_providers.agentos-gateway.auth]', - ); - assert.include( - source, - 'base_url = "http://agentgateway-openai.agentos.svc.cluster.local:8788"', - ); - assert.include( - source, - 'command = "/home/agent/.local/share/mise/shims/bun"', + 'base_url = "http://127.0.0.1:8790"', ); + assert.notInclude(source, "[model_providers.agentos-gateway.auth]"); assert.notInclude(source, "header.payload.signature"); assert.notInclude(source, "AI_GATEWAY_TOKEN"); assert.strictEqual( @@ -182,7 +175,7 @@ describe("Effect Codex workload-authenticated provider", () => { ); it.effect( - "fails closed on ownership collisions and malformed identity inputs", + "fails closed on ownership collisions and malformed gateway inputs", () => Effect.scoped(Effect.gen(function*() { const fileSystem = yield* FileSystem.FileSystem; @@ -203,9 +196,9 @@ describe("Effect Codex workload-authenticated provider", () => { assert.include( (yield* reconcile(malformed, { ...gatewayEnvironment, - AGENTOS_ASSIGNMENT_ID: "not-an-assignment", + AI_GATEWAY_URL: "file:///tmp/not-http", }).pipe(Effect.flip)).message, - "AGENTOS_ASSIGNMENT_ID", + "AI_GATEWAY_URL", ); assert.strictEqual( yield* fileSystem.readFileString(malformed.configPath), diff --git a/packages/agentos/runtime/tests/kubernetes.effect.test.ts b/packages/agentos/runtime/tests/kubernetes.effect.test.ts index 8f0b2a94..ee01b74b 100644 --- a/packages/agentos/runtime/tests/kubernetes.effect.test.ts +++ b/packages/agentos/runtime/tests/kubernetes.effect.test.ts @@ -4,10 +4,6 @@ import { Effect, Schema } from "effect"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { - AGENTOS_EGRESS_TOKEN_AUDIENCE, - AGENTOS_EGRESS_TOKEN_EXPIRATION_SECONDS, -} from "../../src/access/identity.ts"; import { renderKustomize } from "../../../../tooling/testing/kubernetes.ts"; const EnvironmentEntry = Schema.Struct({ @@ -126,31 +122,20 @@ describe("persistent Agent Kubernetes runtime", () => { assert.deepStrictEqual(agent.securityContext, { allowPrivilegeEscalation: false, capabilities: { drop: ["ALL"] }, + readOnlyRootFilesystem: true, + runAsNonRoot: true, }); assert.deepStrictEqual(agent.volumeMounts, [ { mountPath: "/home/agent", name: "home" }, - { - mountPath: "/var/run/secrets/agentos-egress", - name: "agentos-egress-identity", - readOnly: true, - }, + { mountPath: "/tmp", name: "tmp" }, ]); assert.deepStrictEqual(install.volumeMounts, [ { mountPath: "/home/agent", name: "home" }, + { mountPath: "/tmp", name: "tmp" }, + ]); + assert.deepStrictEqual(pod.volumes, [ + { name: "tmp", emptyDir: { sizeLimit: "256Mi" } }, ]); - assert.deepStrictEqual(pod.volumes, [{ - name: "agentos-egress-identity", - projected: { - defaultMode: 288, - sources: [{ - serviceAccountToken: { - audience: AGENTOS_EGRESS_TOKEN_AUDIENCE, - expirationSeconds: AGENTOS_EGRESS_TOKEN_EXPIRATION_SECONDS, - path: "token", - }, - }], - }, - }]); }).pipe(Effect.provide(BunServices.layer))); it.effect("adds the persistent Pi lifecycle only in the Mate layer", () => diff --git a/packages/agentos/runtime/tests/prepare-home.effect.test.ts b/packages/agentos/runtime/tests/prepare-home.effect.test.ts index e43637e4..bf2dbeb1 100644 --- a/packages/agentos/runtime/tests/prepare-home.effect.test.ts +++ b/packages/agentos/runtime/tests/prepare-home.effect.test.ts @@ -51,7 +51,7 @@ const CodexConfig = Schema.Struct({ model_providers: Schema.Record(Schema.String, Schema.Struct({ base_url: Schema.String, wire_api: Schema.String, - auth: Schema.Struct({ refresh_interval_ms: Schema.Number }), + requires_openai_auth: Schema.Boolean, })), otel: Schema.Struct({ log_user_prompt: Schema.Boolean, @@ -190,10 +190,9 @@ layer(platform)("Mate home preparation", (it) => { AGENTOS_ASSIGNMENT_ID: "20000000-0000-4000-8000-000000000001", AGENTOS_AGENT_ROLE: "crewmate", AGENTOS_CODEX_PROVIDER_MODE: "ai-gateway", - AGENTOS_EGRESS_TOKEN_FILE: "/var/run/secrets/agentos-egress/token", AGENTOS_RELEASE_ROOT: repository, AI_GATEWAY_URL: - "http://agentgateway-openai.agentos.svc.cluster.local:8788", + "http://127.0.0.1:8790", HOME: home, MISE_SYSTEM_CONFIG_FILE: paths.join(repository, "mise.toml"), OTEL_EXPORTER_OTLP_ENDPOINT: "http://agentos-otel-collector:4318", @@ -214,9 +213,9 @@ layer(platform)("Mate home preparation", (it) => { ); assert.strictEqual(config.model_provider, "agentos-gateway"); assert.deepInclude(config.model_providers["agentos-gateway"], { - base_url: "http://agentgateway-openai.agentos.svc.cluster.local:8788", + base_url: "http://127.0.0.1:8790", wire_api: "responses", - auth: { refresh_interval_ms: 60_000 }, + requires_openai_auth: false, }); assert.isFalse(config.otel.log_user_prompt); assert.strictEqual(config.otel.environment, "test"); diff --git a/packages/agentos/runtime/tests/readiness.effect.test.ts b/packages/agentos/runtime/tests/readiness.effect.test.ts index c98c9746..0a2337a2 100644 --- a/packages/agentos/runtime/tests/readiness.effect.test.ts +++ b/packages/agentos/runtime/tests/readiness.effect.test.ts @@ -617,42 +617,24 @@ describe("semantic Agent readiness", () => { const codexProviderMarker = `${home}/.local/state/agentos/codex-provider.json`; const codexEntry = { name: "AgentOS workload gateway", - base_url: - "http://agentgateway-openai.agentos.svc.cluster.local:8788", + base_url: "http://127.0.0.1:8790", wire_api: "responses", + requires_openai_auth: false, supports_websockets: false, request_max_retries: 0, stream_max_retries: 0, - env_http_headers: { - "X-AgentOS-Assignment-Id": "AGENTOS_ASSIGNMENT_ID", - }, - auth: { - command: "/home/agent/.local/share/mise/shims/bun", - args: [ - "/opt/agentos/packages/agentos/runtime/codex-token.ts", - egressTokenPath, - ], - timeout_ms: 5_000, - refresh_interval_ms: 60_000, - }, }; const codexConfig = [ 'model_provider = "agentos-gateway"', "", "[model_providers.agentos-gateway]", 'name = "AgentOS workload gateway"', - 'base_url = "http://agentgateway-openai.agentos.svc.cluster.local:8788"', + 'base_url = "http://127.0.0.1:8790"', 'wire_api = "responses"', + "requires_openai_auth = false", "supports_websockets = false", "request_max_retries = 0", "stream_max_retries = 0", - 'env_http_headers = { "X-AgentOS-Assignment-Id" = "AGENTOS_ASSIGNMENT_ID" }', - "", - "[model_providers.agentos-gateway.auth]", - 'command = "/home/agent/.local/share/mise/shims/bun"', - `args = ["/opt/agentos/packages/agentos/runtime/codex-token.ts","${egressTokenPath}"]`, - "timeout_ms = 5000", - "refresh_interval_ms = 60000", "", ].join("\n"); const gatewayCrewEnvironment = { @@ -661,7 +643,7 @@ describe("semantic Agent readiness", () => { AGENTOS_RELEASE_ROOT: "/opt/agentos", AGENTOS_PROVIDER_CREDENTIAL_KIND: "ai_gateway", AI_GATEWAY_URL: - "http://agentgateway-openai.agentos.svc.cluster.local:8788", + "http://127.0.0.1:8790", }; const gatewayCrewFiles = { [briefPath]: crewBrief, @@ -699,7 +681,7 @@ describe("semantic Agent readiness", () => { files: { ...gatewayCrewFiles, [codexConfigPath]: codexConfig.replace( - "agentgateway-openai.agentos.svc.cluster.local", + "127.0.0.1", "unreviewed.example", ), }, diff --git a/packages/agentos/skills/agentos-ai-gateway/SKILL.md b/packages/agentos/skills/agentos-ai-gateway/SKILL.md index 632d626c..067dcbee 100644 --- a/packages/agentos/skills/agentos-ai-gateway/SKILL.md +++ b/packages/agentos/skills/agentos-ai-gateway/SKILL.md @@ -230,17 +230,12 @@ update this integration while that gate is unmet; auxiliary requests remain out of scope until the selected revision satisfies the contract's no-retry, no-redirect and no-fallback requirements. -For the live v0.1.24 Hermes topology, the observed pre-migration state uses one -`ai-gateway` replica with a shared `AI_GATEWAY_TOKEN`, while Hermes uses direct -in-Pod provider authentication and has no projected egress identity. Do not -mutate it in place. - 1. Keep Hermes on verified direct OAuth while deploying the reviewed Agentgateway plus `agentos-egress-authz` topology and registering the Hermes workload identity/access profile. 2. Build and publish the reviewed AgentOS revision, then update the Hermes manifest with the exact immutable adapter image digest and projected token - wiring. Do not copy the legacy shared token or any `ai-gateway-client` Secret. + wiring. Do not copy any client credential into the workload. 3. Render and review the StatefulSet and NetworkPolicies. Confirm the proxy is loopback-only, Agentgateway is the only AI Gateway ingress, and ordinary Internet egress is unchanged. @@ -251,10 +246,9 @@ mutate it in place. 5. Keep direct OAuth intact until that evidence is accepted. Roll back by restoring the prior Hermes config/workload revision; do not delete provider auth or retained homes as part of route rollback. -6. Retire the legacy shared-token request path only after every selected client - has either passed the workload-identity route or returned to verified direct - authentication. Secret removal, deployment, restart, and provider login are - separate human approval gates. +6. Keep every selected client either on the workload-identity route or verified + direct authentication. Secret removal, deployment, restart, and provider + login are separate human approval gates. For First and Second Mate, the additive client patch sets `AGENTOS_PI_PROVIDER_MODE=ai-gateway` on `prepare-home`. Before Pi can start, diff --git a/packages/agentos/src/access/hermes-authorizer.ts b/packages/agentos/src/access/hermes-authorizer.ts index ecbea9aa..e9fe1d60 100644 --- a/packages/agentos/src/access/hermes-authorizer.ts +++ b/packages/agentos/src/access/hermes-authorizer.ts @@ -17,6 +17,10 @@ import { const ResponsesRequestModelSchema = Schema.fromJsonString(Schema.Struct({ model: Schema.String, + stream: Schema.Literal(true), + max_output_tokens: Schema.Number.pipe( + Schema.check(Schema.isInt(), Schema.isGreaterThan(0)), + ), })); export class HermesProviderAccessPolicyDependencyUnavailable extends Schema.TaggedErrorClass()( @@ -46,6 +50,13 @@ export interface HermesProviderAuthorizationResult { readonly tokenExpiresAtMillis: number; readonly policyExpiresAtMillis: number | null; readonly grant: HermesProviderAccessGrantV1; + readonly requestedTokens: number; + readonly requestedSpendMicros: number; + readonly workloadIdentity?: { + readonly serviceAccountUid: string; + readonly podName: string; + readonly podUid: string; + }; } export interface HermesProviderAuthorization { @@ -132,11 +143,35 @@ export const createHermesProviderAuthorization = Effect.fn( capability, atMillis: request.atMillis, }); + const requestBytes = new TextEncoder().encode(request.body).byteLength; + const requestedTokens = requestBytes + payload.max_output_tokens; + const requestedSpendMicros = Math.ceil(( + requestBytes * grant.pricing.inputMicrosPerMillionTokens + + payload.max_output_tokens * grant.pricing.outputMicrosPerMillionTokens + ) / 1_000_000); + if ( + !Number.isSafeInteger(requestBytes) || + !Number.isSafeInteger(requestedTokens) || requestedTokens <= 0 || + requestedTokens > grant.limits.maximumTokens || + !Number.isSafeInteger(requestedSpendMicros) || requestedSpendMicros <= 0 || + requestedSpendMicros > grant.limits.maximumSpendMicros + ) { + return yield* HermesProviderAccessPolicyError.make({ + code: "access_denied", + }); + } const authorized: HermesProviderAuthorizationResult = { kind: "authorized", tokenExpiresAtMillis: bound.tokenExpiresAtMillis, policyExpiresAtMillis: bindingResult.success.expiresAtMillis, grant, + requestedTokens, + requestedSpendMicros, + workloadIdentity: { + serviceAccountUid: bound.serviceAccountUid, + podName: bound.kubernetesPod, + podUid: bound.podUid, + }, }; return authorized; }), diff --git a/packages/agentos/src/access/http-authorizer.ts b/packages/agentos/src/access/http-authorizer.ts index 02b35f82..e0d2a2de 100644 --- a/packages/agentos/src/access/http-authorizer.ts +++ b/packages/agentos/src/access/http-authorizer.ts @@ -19,6 +19,11 @@ import { } from "./identity.ts"; import type { HermesProviderAuthorization } from "./hermes-authorizer.ts"; import { + ProviderBudgetEnforcementError, + type ProviderBudgetEnforcer, +} from "./provider-budget.ts"; +import { + HermesProviderAccessGrantV1Schema, HermesProviderLimitsV1Schema, HermesProviderModelIdSchema, normalizeHermesProviderModelId, @@ -75,6 +80,9 @@ export const HermesProviderAuthorizationGrantV1Schema = Schema.Struct({ identity: KubernetesWorkloadPrincipalV1Schema, model: HermesProviderModelIdSchema, limits: HermesProviderLimitsV1Schema, + pricing: HermesProviderAccessGrantV1Schema.fields.pricing, + requestedTokens: EpochMillis, + requestedSpendMicros: EpochMillis, }); export const ProviderAuthorizationGrantV1Schema = Schema.Union([ @@ -238,6 +246,11 @@ export const PROVIDER_AUTHORIZATION_GRANT_HEADERS = Object.freeze([ "x-agentos-authz-maximum-tokens", "x-agentos-authz-spend-window-millis", "x-agentos-authz-maximum-spend-micros", + "x-agentos-authz-pricing-version", + "x-agentos-authz-input-micros-per-million-tokens", + "x-agentos-authz-output-micros-per-million-tokens", + "x-agentos-authz-requested-tokens", + "x-agentos-authz-requested-spend-micros", ]); export function resolveProviderAuthorizationRoute( @@ -313,6 +326,11 @@ export function providerAuthorizationGrantHeaders( headers.set("x-agentos-authz-maximum-tokens", String(grant.limits.maximumTokens)); headers.set("x-agentos-authz-spend-window-millis", String(grant.limits.spendWindowMillis)); headers.set("x-agentos-authz-maximum-spend-micros", String(grant.limits.maximumSpendMicros)); + headers.set("x-agentos-authz-pricing-version", String(grant.pricing.version)); + headers.set("x-agentos-authz-input-micros-per-million-tokens", String(grant.pricing.inputMicrosPerMillionTokens)); + headers.set("x-agentos-authz-output-micros-per-million-tokens", String(grant.pricing.outputMicrosPerMillionTokens)); + headers.set("x-agentos-authz-requested-tokens", String(grant.requestedTokens)); + headers.set("x-agentos-authz-requested-spend-micros", String(grant.requestedSpendMicros)); } else { headers.set("x-agentos-authz-principal-kind", "agentos"); headers.set("x-agentos-authz-agent-id", grant.identity.agentId); @@ -437,6 +455,7 @@ export const createProviderAuthorizationHttpHandler = Effect.fn( readonly id: Effect.Effect; readonly telemetry?: ProviderAccessTelemetry["Service"]; readonly hermes: HermesProviderAuthorization; + readonly budgets?: ProviderBudgetEnforcer["Service"]; }) { const clock = options.clock ?? Clock.currentTimeMillis; const id = options.id; @@ -504,8 +523,38 @@ export const createProviderAuthorizationHttpHandler = Effect.fn( rateClass: hermes.grant.rateClass, model: hermes.grant.model, limits: hermes.grant.limits, + pricing: hermes.grant.pricing, + requestedTokens: hermes.requestedTokens, + requestedSpendMicros: hermes.requestedSpendMicros, }).pipe(Effect.mapError(() => authorizerError("invalid_grant"))); if (grant.expiresAtMillis <= issuedAtMillis) return forbiddenResponse(); + if ( + options.budgets?.reserveWorkload === undefined || + hermes.workloadIdentity === undefined + ) { + return unavailableResponse(); + } + yield* options.budgets.reserveWorkload({ + schemaVersion: 1, + decisionRef: grant.decisionRef, + correlationId: grant.correlationId, + principal: { ...hermes.grant.principal, ...hermes.workloadIdentity }, + provider: "openai", + credentialDomain: "openai-responses", + capability: grant.capability === "openai.responses.create" + ? "openai.responses.create" + : "openai.responses.compact", + resource: grant.resource, + environment: "production", + model: hermes.grant.model, + rateClass: grant.rateClass, + limits: hermes.grant.limits, + pricing: hermes.grant.pricing, + policyExpiresAtMillis: grant.expiresAtMillis, + requestedTokens: hermes.requestedTokens, + requestedSpendMicros: hermes.requestedSpendMicros, + nowMillis: issuedAtMillis, + }); if (telemetry !== undefined) yield* telemetry.correlate(grant); return new Response(null, { status: 200, @@ -1224,6 +1273,22 @@ function authorizationPolicyFieldsFromHeaders(headers: Headers) { "x-agentos-authz-maximum-spend-micros", ), }, + pricing: { + version: integerHeader(headers, "x-agentos-authz-pricing-version"), + inputMicrosPerMillionTokens: integerHeader( + headers, + "x-agentos-authz-input-micros-per-million-tokens", + ), + outputMicrosPerMillionTokens: integerHeader( + headers, + "x-agentos-authz-output-micros-per-million-tokens", + ), + }, + requestedTokens: integerHeader(headers, "x-agentos-authz-requested-tokens"), + requestedSpendMicros: integerHeader( + headers, + "x-agentos-authz-requested-spend-micros", + ), }; } return { @@ -1255,6 +1320,14 @@ function authorizerError(code: ProviderAuthorizationError["code"]) { } function responseForAuthorizationFailure(error: unknown): Response { + if (error instanceof ProviderBudgetEnforcementError) { + if (error.outcome === "rate_limited" || error.outcome === "budget_exhausted") { + return quotaDeniedResponse(error.outcome); + } + return error.outcome === "rate_class_disabled" + ? forbiddenResponse() + : unavailableResponse(); + } if (error instanceof ProviderPolicyDecisionError) { switch (error.outcome) { case "database_unavailable": diff --git a/packages/agentos/src/access/kubernetes-workload-policy.ts b/packages/agentos/src/access/kubernetes-workload-policy.ts index aea36607..7c089664 100644 --- a/packages/agentos/src/access/kubernetes-workload-policy.ts +++ b/packages/agentos/src/access/kubernetes-workload-policy.ts @@ -59,6 +59,11 @@ export const HermesProviderRuleV1Schema = Schema.Struct({ ), rateClass: AccessRateClassIdSchema, limits: HermesProviderLimitsV1Schema, + pricing: Schema.Struct({ + version: PositiveInt, + inputMicrosPerMillionTokens: PositiveInt, + outputMicrosPerMillionTokens: PositiveInt, + }), }); export const HermesProviderAccessBindingDocumentV1Schema = Schema.Struct({ @@ -250,6 +255,7 @@ export const HermesProviderAccessGrantV1Schema = Schema.Struct({ capability: Schema.Literals(["responses.create", "responses.compact"]), rateClass: AccessRateClassIdSchema, limits: HermesProviderLimitsV1Schema, + pricing: HermesProviderRuleV1Schema.fields.pricing, }); export type HermesProviderAccessGrantV1 = typeof HermesProviderAccessGrantV1Schema.Type; @@ -345,6 +351,7 @@ export const matchHermesProviderAccessConfigMapV1 = Effect.fn( capability: target.capability, rateClass: provider.rateClass, limits: provider.limits, + pricing: provider.pricing, } satisfies HermesProviderAccessGrantV1; }); diff --git a/packages/agentos/src/access/provider-budget-postgres.ts b/packages/agentos/src/access/provider-budget-postgres.ts index 550250ac..68c29b8d 100644 --- a/packages/agentos/src/access/provider-budget-postgres.ts +++ b/packages/agentos/src/access/provider-budget-postgres.ts @@ -5,15 +5,18 @@ import { makeProviderBudgetEnforcerLayer, providerBudgetKey, type ProviderBudgetProviderSettlementInputV1, + type ProviderBudgetAttemptRenewalInputV1, type ProviderBudgetReservationInputV1, type ProviderBudgetSettlementInputV1, + type ProviderBudgetWorkloadReservationInputV1, + type ProviderBudgetWorkloadValidationInputV1, type ProviderBudgetStore, } from "./provider-budget.ts"; type ProviderBudgetRow = Readonly>; function normalizeReservation( - input: ProviderBudgetReservationInputV1, + input: Pick, budgetKey: string, row: ProviderBudgetRow | undefined, ): unknown { @@ -82,6 +85,56 @@ export const ProviderBudgetEnforcerPostgresLayer = Layer.unwrap( return normalizeReservation(input, budgetKey, rows[0]); }, ); + const reserveWorkload = Effect.fn( + "ProviderBudgetStorePostgres.reserveWorkload", + )(function*(input: ProviderBudgetWorkloadReservationInputV1) { + const budgetKey = yield* providerBudgetKey({ + subject: input.principal, + provider: input.provider, + credentialDomain: input.credentialDomain, + capability: input.capability, + resource: input.resource, + environment: input.environment, + }).pipe(Effect.provideService(Crypto.Crypto, crypto)); + const rows = yield* sql` + SELECT * FROM agentos.reserve_workload_provider_budget( + ${input.decisionRef}, + ${budgetKey}, + ${input.correlationId}, + ${JSON.stringify(input.principal)}::jsonb, + ${input.provider}, + ${input.credentialDomain}, + ${input.capability}, + ${JSON.stringify(input.resource)}::jsonb, + ${input.environment}, + ${input.model}, + ${input.rateClass}, + ${JSON.stringify(input.limits)}::jsonb, + ${JSON.stringify(input.pricing)}::jsonb, + ${input.policyExpiresAtMillis}, + ${input.requestedTokens}, ${input.requestedSpendMicros}, + ${input.nowMillis} + ) + `; + return normalizeReservation(input, budgetKey, rows[0]); + }); + const validateWorkload = Effect.fn( + "ProviderBudgetStorePostgres.validateWorkload", + )(function*(input: ProviderBudgetWorkloadValidationInputV1) { + const rows = yield* sql` + SELECT * FROM agentos.validate_workload_provider_budget( + ${input.decisionRef}, ${input.correlationId}, + ${JSON.stringify(input.principal)}::jsonb, + ${input.provider}, ${input.credentialDomain}, ${input.capability}, + ${JSON.stringify(input.resource)}::jsonb, ${input.model}, + ${input.rateClass}, ${JSON.stringify(input.limits)}::jsonb, + ${JSON.stringify(input.pricing)}::jsonb, + ${input.requestedTokens}, ${input.requestedSpendMicros}, + ${input.expiresAtMillis}, ${input.nowMillis} + ) + `; + return rows[0]; + }); const settle = Effect.fn("ProviderBudgetStorePostgres.settle")( function*(input: ProviderBudgetSettlementInputV1) { const rows = yield* sql` @@ -117,10 +170,24 @@ export const ProviderBudgetEnforcerPostgresLayer = Layer.unwrap( `; return normalizeSettlement(input, rows[0]); }); + const renewProviderAttempt = Effect.fn( + "ProviderBudgetStorePostgres.renewProviderAttempt", + )(function*(input: ProviderBudgetAttemptRenewalInputV1) { + const rows = yield* sql` + SELECT * FROM agentos.renew_workload_provider_attempt( + ${input.decisionRef}, ${input.provider}, ${input.credentialDomain}, + ${input.renewedAtMillis} + ) + `; + return rows[0]; + }); const store: ProviderBudgetStore = { + reserveWorkload, + validateWorkload, reserve, settle, settleProvider, + renewProviderAttempt, }; return makeProviderBudgetEnforcerLayer(store); }), diff --git a/packages/agentos/src/access/provider-budget-settlement-http.ts b/packages/agentos/src/access/provider-budget-settlement-http.ts index ba3177fc..ca4009df 100644 --- a/packages/agentos/src/access/provider-budget-settlement-http.ts +++ b/packages/agentos/src/access/provider-budget-settlement-http.ts @@ -16,12 +16,32 @@ import { ProviderBudgetSettlementReportV1Schema, type ProviderBudgetSettlementReportV1, } from "./provider-budget.ts"; +import { + ProviderBudgetReservationAcceptanceV1Schema, + ProviderBudgetReservationRequestError, + ProviderBudgetReservationRequestV1Schema, + ProviderBudgetReservationRequester, +} from "./http-authorizer.ts"; const PositiveInteger = Schema.Number.pipe( Schema.check(Schema.isInt(), Schema.isGreaterThan(0)), ); const DecisionRef = ProviderBudgetSettlementReportV1Schema.fields.decisionRef; +export const ProviderBudgetAttemptRenewalRequestV1Schema = Schema.Struct({ + schemaVersion: Schema.Literal(1), + decisionRef: DecisionRef, +}); +export const ProviderBudgetAttemptRenewalReceiptV1Schema = Schema.Struct({ + schemaVersion: Schema.Literal(1), + decisionRef: DecisionRef, + outcome: Schema.Literal("renewed"), +}); +export type ProviderBudgetAttemptRenewalRequestV1 = + typeof ProviderBudgetAttemptRenewalRequestV1Schema.Type; +export type ProviderBudgetAttemptRenewalReceiptV1 = + typeof ProviderBudgetAttemptRenewalReceiptV1Schema.Type; + export const AGENTOS_PROVIDER_BUDGET_SETTLEMENT_BASE_URL = "http://agentos-egress-authz.agentos.svc.cluster.local:9001"; @@ -88,6 +108,15 @@ export class ProviderBudgetSettlementReporter extends Context.Service< } >()("agentos/access/ProviderBudgetSettlementReporter") {} +export class ProviderBudgetAttemptRenewalReporter extends Context.Service< + ProviderBudgetAttemptRenewalReporter, + { + readonly renew: ( + request: ProviderBudgetAttemptRenewalRequestV1, + ) => Effect.Effect; + } +>()("agentos/access/ProviderBudgetAttemptRenewalReporter") {} + export class ProviderBudgetSettlementReadiness extends Context.Service< ProviderBudgetSettlementReadiness, { @@ -163,6 +192,96 @@ export function makeProviderBudgetSettlementHttpLayer( }, ); + const validate = Effect.fn("agentos.providerBudgetReservation.validate")( + function*(untrusted: unknown) { + const body = yield* Schema.decodeUnknownEffect( + ProviderBudgetReservationRequestV1Schema, + { onExcessProperty: "error" }, + )(untrusted).pipe(Effect.mapError(() => + ProviderBudgetReservationRequestError.make({ code: "rejected" }) + )); + const token = yield* fileSystem.readFileString(options.tokenPath).pipe( + Effect.flatMap(validateProjectedToken), + Effect.mapError(() => + ProviderBudgetReservationRequestError.make({ code: "unavailable" }) + ), + ); + let request = HttpClientRequest.post(endpoints.validation).pipe( + HttpClientRequest.acceptJson, + HttpClientRequest.setHeader("authorization", `Bearer ${token}`), + ); + request = yield* HttpClientRequest.bodyJson(request, body).pipe( + Effect.mapError(() => + ProviderBudgetReservationRequestError.make({ code: "rejected" }) + ), + ); + return yield* client.execute(request).pipe( + Effect.flatMap((response) => + response.status >= 200 && response.status < 300 + ? readBoundedReservationAcceptance( + response, + options.maximumResponseBytes, + ) + : Effect.fail(ProviderBudgetReservationRequestError.make({ + code: response.status >= 500 ? "unavailable" : "rejected", + })) + ), + Effect.timeoutOrElse({ + duration: options.timeoutMillis, + orElse: () => ProviderBudgetReservationRequestError.make({ + code: "unavailable", + }), + }), + Effect.mapError((error) => + error instanceof ProviderBudgetReservationRequestError + ? error + : ProviderBudgetReservationRequestError.make({ + code: "invalid_response", + }) + ), + Effect.scoped, + ); + }, + ); + + const renew = Effect.fn("agentos.providerBudgetAttempt.renew")( + function*(untrusted: ProviderBudgetAttemptRenewalRequestV1) { + const body = yield* Schema.decodeUnknownEffect( + ProviderBudgetAttemptRenewalRequestV1Schema, + { onExcessProperty: "error" }, + )(untrusted).pipe(Effect.mapError(() => settlementHttpError("invalid_report"))); + const token = yield* fileSystem.readFileString(options.tokenPath).pipe( + Effect.mapError(() => settlementHttpError("credential_unavailable")), + Effect.flatMap(validateProjectedToken), + ); + let request = HttpClientRequest.post(endpoints.renewal).pipe( + HttpClientRequest.acceptJson, + HttpClientRequest.setHeader("authorization", `Bearer ${token}`), + ); + request = yield* HttpClientRequest.bodyJson(request, body).pipe( + Effect.mapError(() => settlementHttpError("invalid_report")), + ); + const receipt = yield* client.execute(request).pipe( + Effect.flatMap((response) => response.status >= 200 && response.status < 300 + ? readBoundedResponseSource(response, options.maximumResponseBytes).pipe( + Effect.flatMap(Schema.decodeUnknownEffect( + Schema.fromJsonString(ProviderBudgetAttemptRenewalReceiptV1Schema), + { onExcessProperty: "error" }, + )), + Effect.mapError(() => settlementHttpError("invalid_response", response.status)), + ) + : settlementStatusError(response.status)), + Effect.mapError((error) => error instanceof ProviderBudgetSettlementHttpError + ? error + : settlementHttpError("request_failed")), + Effect.timeoutOrElse({ duration: options.timeoutMillis, orElse: () => settlementHttpError("timeout") }), + Effect.scoped, + ); + if (receipt.decisionRef !== body.decisionRef) return yield* settlementHttpError("invalid_response"); + return receipt; + }, + ); + const check = Effect.fn("agentos.providerBudgetSettlement.readiness")( function*() { const token = yield* fileSystem.readFileString(options.tokenPath).pipe( @@ -199,10 +318,15 @@ export function makeProviderBudgetSettlementHttpLayer( ProviderBudgetSettlementReporter, ProviderBudgetSettlementReporter.of({ report }), ).pipe( + Context.add(ProviderBudgetAttemptRenewalReporter, ProviderBudgetAttemptRenewalReporter.of({ renew })), Context.add( ProviderBudgetSettlementReadiness, ProviderBudgetSettlementReadiness.of({ check }), ), + Context.add( + ProviderBudgetReservationRequester, + ProviderBudgetReservationRequester.of({ request: validate }), + ), ); }), ); @@ -226,6 +350,8 @@ function settlementEndpoints(baseUrl: string) { } return { settlement: new URL("/settle", base), + renewal: new URL("/renew", base), + validation: new URL("/validate", base), readiness: new URL("/readyz/settlement", base), }; }); @@ -324,6 +450,23 @@ function readBoundedSettlementReadiness( ); } +function readBoundedReservationAcceptance( + response: HttpClientResponse.HttpClientResponse, + maximumResponseBytes: number, +) { + return readBoundedResponseSource(response, maximumResponseBytes).pipe( + Effect.flatMap((source) => + Schema.decodeUnknownEffect( + Schema.fromJsonString(ProviderBudgetReservationAcceptanceV1Schema), + { onExcessProperty: "error" }, + )(source) + ), + Effect.mapError(() => ProviderBudgetReservationRequestError.make({ + code: "invalid_response", + })), + ); +} + function readBoundedResponseSource( response: HttpClientResponse.HttpClientResponse, maximumResponseBytes: number, diff --git a/packages/agentos/src/access/provider-budget.ts b/packages/agentos/src/access/provider-budget.ts index 5ed5b9c5..6b6b8c4b 100644 --- a/packages/agentos/src/access/provider-budget.ts +++ b/packages/agentos/src/access/provider-budget.ts @@ -13,6 +13,7 @@ import { AccessProviderIdSchema, AccessRateClassIdSchema, AuthorizationResourceV1Schema, + KubernetesWorkloadPrincipalV1Schema, authorizationResourceName, ProviderBudgetSubjectV1Schema, providerBudgetSubjectName, @@ -49,6 +50,26 @@ const Environment = Schema.NullOr(Schema.String.pipe( Schema.isPattern(/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/), ), )); +const KubernetesUid = Schema.String.pipe( + Schema.check(Schema.isMinLength(1), Schema.isMaxLength(128)), +); +const KubernetesName = Schema.String.pipe( + Schema.check( + Schema.isMaxLength(63), + Schema.isPattern(/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/), + ), +); +const ProviderModel = Schema.String.pipe( + Schema.check( + Schema.isMaxLength(128), + Schema.isPattern(/^[a-z0-9][a-z0-9._:-]*$/), + ), +); +const ProviderPricing = Schema.Struct({ + version: SafePositiveInteger, + inputMicrosPerMillionTokens: SafePositiveInteger, + outputMicrosPerMillionTokens: SafePositiveInteger, +}); export const ProviderBudgetRateClassV1Schema = Schema.Struct({ schemaVersion: Schema.Literal(1), @@ -137,6 +158,74 @@ export const ProviderBudgetReservationInputV1Schema = Schema.Struct({ nowMillis: EpochMillis, }); +export const ProviderBudgetWorkloadPrincipalV1Schema = Schema.Struct({ + ...KubernetesWorkloadPrincipalV1Schema.fields, + serviceAccountUid: KubernetesUid, + podName: KubernetesName, + podUid: KubernetesUid, +}); + +export const ProviderBudgetWorkloadReservationInputV1Schema = Schema.Struct({ + schemaVersion: Schema.Literal(1), + decisionRef: DecisionRef, + correlationId: CorrelationId, + principal: ProviderBudgetWorkloadPrincipalV1Schema, + provider: Schema.Literal("openai"), + credentialDomain: Schema.Literal("openai-responses"), + capability: Schema.Literals([ + "openai.responses.create", + "openai.responses.compact", + ]), + resource: AuthorizationResourceV1Schema, + environment: Environment, + model: ProviderModel, + rateClass: AccessRateClassIdSchema, + limits: Schema.Struct({ + requestWindowMillis: SafePositiveInteger, + maximumRequests: SafePositiveInteger, + maximumConcurrent: SafePositiveInteger, + tokenWindowMillis: SafePositiveInteger, + maximumTokens: SafePositiveInteger, + spendWindowMillis: SafePositiveInteger, + maximumSpendMicros: SafePositiveInteger, + }), + pricing: ProviderPricing, + policyExpiresAtMillis: EpochMillis, + requestedTokens: SafePositiveInteger, + requestedSpendMicros: SafePositiveInteger, + nowMillis: EpochMillis, +}); + +export const ProviderBudgetWorkloadValidationInputV1Schema = Schema.Struct({ + schemaVersion: Schema.Literal(1), + decisionRef: DecisionRef, + correlationId: CorrelationId, + principal: KubernetesWorkloadPrincipalV1Schema, + provider: Schema.Literal("openai"), + credentialDomain: Schema.Literal("openai-responses"), + capability: Schema.Literals([ + "openai.responses.create", + "openai.responses.compact", + ]), + resource: AuthorizationResourceV1Schema, + model: ProviderModel, + rateClass: AccessRateClassIdSchema, + limits: Schema.Struct({ + requestWindowMillis: SafePositiveInteger, + maximumRequests: SafePositiveInteger, + maximumConcurrent: SafePositiveInteger, + tokenWindowMillis: SafePositiveInteger, + maximumTokens: SafePositiveInteger, + spendWindowMillis: SafePositiveInteger, + maximumSpendMicros: SafePositiveInteger, + }), + pricing: ProviderPricing, + requestedTokens: SafePositiveInteger, + requestedSpendMicros: SafePositiveInteger, + expiresAtMillis: EpochMillis, + nowMillis: EpochMillis, +}); + export const ProviderBudgetKeyInputV1Schema = Schema.Struct({ subject: ProviderBudgetSubjectV1Schema, provider: AccessProviderIdSchema, @@ -222,8 +311,27 @@ export const ProviderBudgetSettlementV1Schema = Schema.Struct({ settledAtMillis: EpochMillis, }); +export const ProviderBudgetAttemptRenewalInputV1Schema = Schema.Struct({ + schemaVersion: Schema.Literal(1), + decisionRef: DecisionRef, + provider: AccessProviderIdSchema, + credentialDomain: CredentialDomain, + renewedAtMillis: EpochMillis, +}); + +export const ProviderBudgetAttemptRenewalV1Schema = Schema.Struct({ + schemaVersion: Schema.Literal(1), + decisionRef: DecisionRef, + outcome: Schema.Literal("renewed"), + leaseExpiresAtMillis: EpochMillis, +}); + export type ProviderBudgetReservationInputV1 = typeof ProviderBudgetReservationInputV1Schema.Type; +export type ProviderBudgetWorkloadReservationInputV1 = + typeof ProviderBudgetWorkloadReservationInputV1Schema.Type; +export type ProviderBudgetWorkloadValidationInputV1 = + typeof ProviderBudgetWorkloadValidationInputV1Schema.Type; export type ProviderBudgetKeyInputV1 = typeof ProviderBudgetKeyInputV1Schema.Type; export type ProviderBudgetReservationV1 = typeof ProviderBudgetReservationV1Schema.Type; @@ -235,6 +343,10 @@ export type ProviderBudgetProviderSettlementInputV1 = typeof ProviderBudgetProviderSettlementInputV1Schema.Type; export type ProviderBudgetSettlementV1 = typeof ProviderBudgetSettlementV1Schema.Type; +export type ProviderBudgetAttemptRenewalInputV1 = + typeof ProviderBudgetAttemptRenewalInputV1Schema.Type; +export type ProviderBudgetAttemptRenewalV1 = + typeof ProviderBudgetAttemptRenewalV1Schema.Type; const ProviderBudgetStoreReservationResultSchema = Schema.Union([ ProviderBudgetReservationV1Schema, @@ -253,6 +365,12 @@ export type ProviderBudgetStoreReservationResult = typeof ProviderBudgetStoreReservationResultSchema.Type; export interface ProviderBudgetStore { + readonly reserveWorkload?: ( + input: ProviderBudgetWorkloadReservationInputV1, + ) => Effect.Effect; + readonly validateWorkload?: ( + input: ProviderBudgetWorkloadValidationInputV1, + ) => Effect.Effect; readonly reserve: ( input: ProviderBudgetReservationInputV1, ) => Effect.Effect; @@ -262,6 +380,9 @@ export interface ProviderBudgetStore { readonly settleProvider: ( input: ProviderBudgetProviderSettlementInputV1, ) => Effect.Effect; + readonly renewProviderAttempt?: ( + input: ProviderBudgetAttemptRenewalInputV1, + ) => Effect.Effect; } const ProviderBudgetEnforcementOutcome = Schema.Literals([ @@ -286,6 +407,12 @@ export class ProviderBudgetEnforcementError extends Schema.TaggedErrorClass Effect.Effect; + readonly validateWorkload: ( + input: ProviderBudgetWorkloadValidationInputV1, + ) => Effect.Effect; readonly reserve: ( input: ProviderBudgetReservationInputV1, ) => Effect.Effect< @@ -304,6 +431,9 @@ export class ProviderBudgetEnforcer extends Context.Service< ProviderBudgetSettlementV1, ProviderBudgetEnforcementError >; + readonly renewProviderAttempt: ( + input: ProviderBudgetAttemptRenewalInputV1, + ) => Effect.Effect; } >()("agentos/access/ProviderBudgetEnforcer") {} @@ -320,6 +450,58 @@ function enforcementError( } export function makeProviderBudgetEnforcerLayer(store: ProviderBudgetStore) { + const validateWorkload = Effect.fn("agentos.providerBudget.validateWorkload")(function*( + untrusted: unknown, + ) { + const input = yield* Schema.decodeUnknownEffect( + ProviderBudgetWorkloadValidationInputV1Schema, + { onExcessProperty: "error" }, + )(untrusted).pipe( + Effect.mapError(() => enforcementError("invalid_reservation", false)), + ); + if (store.validateWorkload === undefined) { + return yield* enforcementError("database_unavailable", true); + } + const raw = yield* store.validateWorkload(input).pipe( + Effect.mapError(() => enforcementError("database_unavailable", true)), + ); + const decoded = yield* Schema.decodeUnknownEffect( + Schema.Struct({ outcome: Schema.Literal("attempted") }), + { onExcessProperty: "error" }, + )(raw).pipe(Effect.mapError(() => enforcementError("policy_stale", false))); + return void decoded; + }); + const reserveWorkload = Effect.fn("agentos.providerBudget.reserveWorkload")(function*( + untrusted: unknown, + ) { + const input = yield* Schema.decodeUnknownEffect( + ProviderBudgetWorkloadReservationInputV1Schema, + { onExcessProperty: "error" }, + )(untrusted).pipe( + Effect.mapError(() => enforcementError("invalid_reservation", false)), + ); + if (input.policyExpiresAtMillis <= input.nowMillis || input.rateClass === "disabled") { + return yield* enforcementError("policy_stale", false); + } + if (store.reserveWorkload === undefined) { + return yield* enforcementError("database_unavailable", true); + } + const raw = yield* store.reserveWorkload(input).pipe( + Effect.mapError(() => enforcementError("database_unavailable", true)), + ); + const result = yield* Schema.decodeUnknownEffect( + ProviderBudgetStoreReservationResultSchema, + { onExcessProperty: "error" }, + )(raw).pipe(Effect.mapError(() => enforcementError("policy_stale", true))); + if (result.outcome !== "reserved") { + return yield* enforcementError( + result.outcome, + result.outcome !== "rate_class_disabled", + result.retryAtMillis, + ); + } + return result; + }); const reserve = Effect.fn("agentos.providerBudget.reserve")(function*( untrusted: unknown, ) { @@ -388,10 +570,45 @@ export function makeProviderBudgetEnforcerLayer(store: ProviderBudgetStore) { ); }); + const renewProviderAttempt = Effect.fn( + "agentos.providerBudget.renewProviderAttempt", + )(function*(untrusted: unknown) { + const input = yield* Schema.decodeUnknownEffect( + ProviderBudgetAttemptRenewalInputV1Schema, + { onExcessProperty: "error" }, + )(untrusted).pipe( + Effect.mapError(() => enforcementError("invalid_settlement", false)), + ); + if (store.renewProviderAttempt === undefined) { + return yield* enforcementError("database_unavailable", true); + } + const raw = yield* store.renewProviderAttempt(input).pipe( + Effect.mapError(() => enforcementError("database_unavailable", true)), + ); + const result = yield* Schema.decodeUnknownEffect( + Schema.Struct({ + outcome: Schema.Literal("renewed"), + leaseExpiresAtMillis: EpochMillis, + }), + { onExcessProperty: "error" }, + )(raw).pipe( + Effect.mapError(() => enforcementError("policy_stale", false)), + ); + return ProviderBudgetAttemptRenewalV1Schema.make({ + schemaVersion: 1, + decisionRef: input.decisionRef, + outcome: result.outcome, + leaseExpiresAtMillis: result.leaseExpiresAtMillis, + }); + }); + return Layer.succeed(ProviderBudgetEnforcer, { + validateWorkload, + reserveWorkload, reserve, settle, settleProvider, + renewProviderAttempt, }); } diff --git a/packages/agentos/src/access/tests/hermes-http-authorizer.effect.test.ts b/packages/agentos/src/access/tests/hermes-http-authorizer.effect.test.ts index 7f5f956b..9781b235 100644 --- a/packages/agentos/src/access/tests/hermes-http-authorizer.effect.test.ts +++ b/packages/agentos/src/access/tests/hermes-http-authorizer.effect.test.ts @@ -22,6 +22,7 @@ import { createProviderAuthorizationHttpHandler, decodeProviderAuthorizationGrantHeaders, } from "../http-authorizer.ts"; +import type { ProviderBudgetEnforcer } from "../provider-budget.ts"; const now = 1_785_586_000_000; const tokenExpiresAtMillis = now + 600_000; @@ -34,6 +35,29 @@ const limits = { spendWindowMillis: 3_600_000, maximumSpendMicros: 2_000_000, }; +const pricing = { + version: 1, + inputMicrosPerMillionTokens: 2_000_000, + outputMicrosPerMillionTokens: 8_000_000, +}; +const budgets: ProviderBudgetEnforcer["Service"] = { + validateWorkload: () => Effect.void, + reserveWorkload: (input) => Effect.succeed({ + schemaVersion: 1, + decisionRef: input.decisionRef, + budgetKey: `budget_${"7".repeat(64)}`, + outcome: "reserved", + effectiveRateClass: input.rateClass, + requestWindowEndsAtMillis: input.nowMillis + 60_000, + tokenWindowEndsAtMillis: input.nowMillis + 60_000, + spendWindowEndsAtMillis: input.nowMillis + 3_600_000, + leaseExpiresAtMillis: input.policyExpiresAtMillis, + }), + reserve: () => Effect.die("legacy reservation is not expected"), + settle: () => Effect.die("settlement is not expected"), + settleProvider: () => Effect.die("provider settlement is not expected"), + renewProviderAttempt: () => Effect.die("provider renewal is not expected"), +}; const boundIdentity: KubernetesBoundServiceAccountIdentityV1 = { schemaVersion: 1, @@ -104,6 +128,7 @@ function policy( capabilities: ["responses.create", "responses.compact"], rateClass: "standard", limits, + pricing, }], expiresAtMillis, disabled, @@ -113,8 +138,13 @@ function policy( }; } -function request(model = "gpt-5.6-sol") { - const body = JSON.stringify({ model, input: "not inspected for authority" }); +function request(model = "gpt-5.6-sol", stream = true) { + const body = JSON.stringify({ + model, + stream, + max_output_tokens: 100, + input: "not inspected for authority", + }); return new Request("http://authorizer.test/authorize", { method: "POST", headers: { @@ -199,6 +229,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("44444444444444444444444444444444"), hermes, + budgets, }); return yield* handler(request()); }).pipe(Effect.provide(layer)); @@ -250,6 +281,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("44444444444444444444444444444444"), hermes, + budgets, }); const nonHermesRequest = request(); nonHermesRequest.headers.set("x-agentos-hermes-profile", "default"); @@ -275,6 +307,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("44444444444444444444444444444444"), hermes, + budgets, }); const response = yield* handler(request()); @@ -286,7 +319,11 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { method: "POST", path: "/v1/responses", nowMillis: now, - body: JSON.stringify({ model: "gpt-5.6-sol" }), + body: JSON.stringify({ + model: "gpt-5.6-sol", + max_output_tokens: 100, + input: "not inspected for authority", + }), }, ); assert.deepStrictEqual(grant.identity, { @@ -315,6 +352,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("44444444444444444444444444444444"), hermes, + budgets, }); const response = yield* handler(request()); assert.strictEqual( @@ -334,6 +372,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("55555555555555555555555555555555"), hermes, + budgets, }); const forged = request("gpt-4.1"); forged.headers.set("x-agentos-hermes-profile", "default"); @@ -344,6 +383,25 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { }).pipe(Effect.provide(dependencies(identityStoreCalls, currentPolicy))); })); + it.effect("denies stream false before issuing or reserving a workload grant", () => + Effect.gen(function*() { + const identityStoreCalls = yield* Ref.make(0); + const currentPolicy = yield* Ref.make(policy()); + yield* Effect.gen(function*() { + const hermes = yield* createHermesProviderAuthorization(); + const handler = yield* createProviderAuthorizationHttpHandler({ + clock: Effect.succeed(now), + id: Effect.succeed("56565656565656565656565656565656"), + hermes, + budgets: { + ...budgets, + reserveWorkload: () => Effect.die("stream false must not reserve"), + }, + }); + assert.strictEqual((yield* handler(request("gpt-5.6-sol", false))).status, 403); + }).pipe(Effect.provide(dependencies(identityStoreCalls, currentPolicy))); + })); + it.effect("denies a namespace mismatch for an otherwise policy-targeted Hermes ServiceAccount", () => Effect.gen(function*() { const identityStoreCalls = yield* Ref.make(0); @@ -354,6 +412,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), hermes, + budgets, }); assert.strictEqual((yield* handler(request())).status, 403); assert.strictEqual(yield* Ref.get(identityStoreCalls), 0); @@ -374,6 +433,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), hermes, + budgets, }); assert.strictEqual((yield* handler(request())).status, 403); assert.strictEqual(yield* Ref.get(identityStoreCalls), 0); @@ -394,6 +454,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("66666666666666666666666666666666"), hermes, + budgets, }); assert.strictEqual((yield* handler(request())).status, 200); yield* Ref.set(currentPolicy, policy("18423", null, true)); @@ -416,8 +477,9 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { const handler = yield* createProviderAuthorizationHttpHandler({ clock: Effect.succeed(now), id: Effect.succeed("77777777777777777777777777777777"), - hermes, - }); + hermes, + budgets, + }); assert.strictEqual((yield* handler(request())).status, 401); assert.strictEqual(yield* Ref.get(identityStoreCalls), 0); }).pipe(Effect.provide(dependencies( @@ -464,6 +526,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("77777777777777777777777777777777"), hermes, + budgets, }); return yield* handler(request()); }).pipe(Effect.provide(layer)); @@ -483,6 +546,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("88888888888888888888888888888888"), hermes, + budgets, }); assert.strictEqual((yield* handler(request())).status, 403); assert.strictEqual(yield* Ref.get(identityStoreCalls), 0); @@ -499,6 +563,7 @@ describe("Hermes Kubernetes workload HTTP authorization", () => { clock: Effect.succeed(now), id: Effect.succeed("99999999999999999999999999999999"), hermes, + budgets, }); assert.strictEqual((yield* handler(request())).status, 503); assert.strictEqual(yield* Ref.get(identityStoreCalls), 0); diff --git a/packages/agentos/src/access/tests/kubernetes-workload-policy.effect.test.ts b/packages/agentos/src/access/tests/kubernetes-workload-policy.effect.test.ts index 314d500c..11c6c4f5 100644 --- a/packages/agentos/src/access/tests/kubernetes-workload-policy.effect.test.ts +++ b/packages/agentos/src/access/tests/kubernetes-workload-policy.effect.test.ts @@ -33,6 +33,11 @@ const validPolicy = { spendWindowMillis: 3_600_000, maximumSpendMicros: 10_000_000, }, + pricing: { + version: 1, + inputMicrosPerMillionTokens: 2_000_000, + outputMicrosPerMillionTokens: 8_000_000, + }, }, ], expiresAtMillis: null, @@ -218,7 +223,8 @@ describe("Hermes Kubernetes workload provider policy", () => { model: "gpt-5.6-sol", capability: "responses.create", rateClass: "standard", - limits: validPolicy.bindings[0]?.providers[0]?.limits, + limits: validPolicy.bindings[0]!.providers[0]!.limits, + pricing: validPolicy.bindings[0]!.providers[0]!.pricing, }); })); diff --git a/packages/agentos/src/access/tests/policy-decision.effect.test.ts b/packages/agentos/src/access/tests/policy-decision.effect.test.ts index 4380fff9..b622694d 100644 --- a/packages/agentos/src/access/tests/policy-decision.effect.test.ts +++ b/packages/agentos/src/access/tests/policy-decision.effect.test.ts @@ -139,6 +139,8 @@ function decisionLayer(input?: { next: Effect.succeed("44444444444444444444444444444444"), }), Layer.succeed(ProviderBudgetEnforcer, { + validateWorkload: () => Effect.die("workload validation not expected in legacy PDP"), + reserveWorkload: () => Effect.die("workload reservation not expected in legacy PDP"), reserve: input?.reserve ?? ((reservation) => Effect.succeed({ schemaVersion: 1, decisionRef: reservation.decisionRef, @@ -153,6 +155,8 @@ function decisionLayer(input?: { settle: () => Effect.die("settlement not expected in PDP"), settleProvider: () => Effect.die("provider settlement not expected in PDP"), + renewProviderAttempt: () => + Effect.die("provider renewal not expected in PDP"), }), ); return makeProviderPolicyDecisionPointLayer({ diff --git a/packages/agentos/src/access/tests/provider-budget-postgres.effect.test.ts b/packages/agentos/src/access/tests/provider-budget-postgres.effect.test.ts index ae3d9aa0..76a9e265 100644 --- a/packages/agentos/src/access/tests/provider-budget-postgres.effect.test.ts +++ b/packages/agentos/src/access/tests/provider-budget-postgres.effect.test.ts @@ -12,10 +12,56 @@ import { type ProviderBudgetProviderSettlementInputV1, type ProviderBudgetReservationInputV1, type ProviderBudgetSettlementInputV1, + type ProviderBudgetWorkloadReservationInputV1, } from "../provider-budget.ts"; import { ProviderBudgetEnforcerPostgresLayer } from "../provider-budget-postgres.ts"; const now = 1_785_585_600_000; +const workloadInput = { + schemaVersion: 1, + decisionRef: `decision_${"a".repeat(32)}`, + correlationId: `corr_${"b".repeat(32)}`, + principal: { + kind: "kubernetes_workload", + namespace: "agentos", + serviceAccountName: "hermes", + serviceAccountUid: "11111111-1111-4111-8111-111111111111", + podName: "hermes-0", + podUid: "22222222-2222-4222-8222-222222222222", + policyRevision: 7, + policyResourceVersion: "18422", + hermesProfile: "fleet-codex", + }, + provider: "openai", + credentialDomain: "openai-responses", + capability: "openai.responses.create", + resource: { + kind: "provider_service", + provider: "openai", + service: "responses", + }, + environment: "production", + model: "gpt-5.6-sol", + rateClass: "low", + limits: { + requestWindowMillis: 60_000, + maximumRequests: 12, + maximumConcurrent: 2, + tokenWindowMillis: 60_000, + maximumTokens: 100_000, + spendWindowMillis: 3_600_000, + maximumSpendMicros: 1_000_000, + }, + policyExpiresAtMillis: now + 60_000, + pricing: { + version: 1, + inputMicrosPerMillionTokens: 1_000_000, + outputMicrosPerMillionTokens: 1_000_000, + }, + requestedTokens: 500, + requestedSpendMicros: 5_000, + nowMillis: now, +} satisfies ProviderBudgetWorkloadReservationInputV1; const input: ProviderBudgetReservationInputV1 = { schemaVersion: 1, decisionRef: `decision_${"1".repeat(32)}`, @@ -96,6 +142,30 @@ function liveLayer( } describe("PostgreSQL provider budget enforcer", () => { + it.effect("uses the workload reservation function without a legacy binding", () => + Effect.gen(function*() { + const calls: Array<{ readonly statement: string; readonly parameters: ReadonlyArray }> = []; + const layer = liveLayer((statement, parameters) => { + calls.push({ statement, parameters }); + return Effect.succeed([{ + outcome: "reserved", + effectiveRateClass: "low", + retryAtMillis: null, + requestWindowEndsAtMillis: now + 60_000, + tokenWindowEndsAtMillis: now + 60_000, + spendWindowEndsAtMillis: now + 3_600_000, + leaseExpiresAtMillis: now + 15_000, + }]); + }); + yield* ProviderBudgetEnforcer.pipe( + Effect.flatMap((budgets) => budgets.reserveWorkload(workloadInput)), + Effect.provide(layer), + ); + assert.match(calls[0]!.statement, /reserve_workload_provider_budget/); + assert.notInclude(calls[0]!.statement, "access_bindings"); + assert.notInclude(calls[0]!.parameters, "binding_"); + })); + it.effect("reserves one stable subject/route budget through the narrow function", () => Effect.gen(function*() { const calls: Array<{ diff --git a/packages/agentos/src/index.ts b/packages/agentos/src/index.ts index 75aae881..48a78f98 100644 --- a/packages/agentos/src/index.ts +++ b/packages/agentos/src/index.ts @@ -194,7 +194,12 @@ export { ProviderBudgetKeyInputV1Schema, ProviderBudgetRateClassV1Schema, ProviderBudgetProviderSettlementInputV1Schema, + ProviderBudgetAttemptRenewalInputV1Schema, + ProviderBudgetAttemptRenewalV1Schema, ProviderBudgetReservationInputV1Schema, + ProviderBudgetWorkloadPrincipalV1Schema, + ProviderBudgetWorkloadReservationInputV1Schema, + ProviderBudgetWorkloadValidationInputV1Schema, ProviderBudgetReservationV1Schema, ProviderBudgetSettlementInputV1Schema, ProviderBudgetSettlementReportV1Schema, @@ -205,7 +210,11 @@ export { type ProviderBudgetKeyInputV1, type ProviderBudgetRateClassV1, type ProviderBudgetProviderSettlementInputV1, + type ProviderBudgetAttemptRenewalInputV1, + type ProviderBudgetAttemptRenewalV1, type ProviderBudgetReservationInputV1, + type ProviderBudgetWorkloadReservationInputV1, + type ProviderBudgetWorkloadValidationInputV1, type ProviderBudgetReservationV1, type ProviderBudgetSettlementInputV1, type ProviderBudgetSettlementReportV1, @@ -232,9 +241,14 @@ export { ProviderBudgetSettlementReadiness, ProviderBudgetSettlementReceiptV1Schema, ProviderBudgetSettlementReporter, + ProviderBudgetAttemptRenewalReporter, + ProviderBudgetAttemptRenewalRequestV1Schema, + ProviderBudgetAttemptRenewalReceiptV1Schema, makeProviderBudgetSettlementHttpLayer, type ProviderBudgetSettlementHttpOptions, type ProviderBudgetSettlementReceiptV1, + type ProviderBudgetAttemptRenewalRequestV1, + type ProviderBudgetAttemptRenewalReceiptV1, } from "./access/provider-budget-settlement-http.ts"; export { ProviderBudgetEnforcerPostgresLayer, diff --git a/packages/agentos/src/resilience/conformance.ts b/packages/agentos/src/resilience/conformance.ts index 0096e32e..2ede9aac 100644 --- a/packages/agentos/src/resilience/conformance.ts +++ b/packages/agentos/src/resilience/conformance.ts @@ -835,7 +835,7 @@ const RuntimeStalePane = regressionTest( ); const GatewayConfig = regressionTest( "services/ai-gateway/tests/config.effect.test.ts", - "rejects missing shared auth and malformed runtime bounds", + "rejects malformed runtime bounds", "#32", ); const AgentgatewayContract = regressionTest( @@ -875,7 +875,7 @@ const ForwardTransport = regressionTest( ); const ForwardStream = regressionTest( "services/ai-gateway/tests/forward.effect.test.ts", - "keeps a provider stream failure distinct and does not settle unknown usage", + "settles a provider stream failure as transport failed", "#32", ); const QuotaFailures = regressionTest( diff --git a/packages/agentos/src/workloads/compiler.ts b/packages/agentos/src/workloads/compiler.ts index d12c477e..09389954 100644 --- a/packages/agentos/src/workloads/compiler.ts +++ b/packages/agentos/src/workloads/compiler.ts @@ -708,11 +708,13 @@ function workloadPatch( spec.resources.init, environment, ), - { - name: "prepare-github-provider", - image: spec.image.reference, - imagePullPolicy: spec.image.pullPolicy, - }, + ...(profile.definition.mechanics.projectedSupervisionIdentity + ? [{ + name: "prepare-github-provider", + image: spec.image.reference, + imagePullPolicy: spec.image.pullPolicy, + }] + : []), ], containers: [ containerPatch( diff --git a/packages/agentos/src/workloads/profiles.ts b/packages/agentos/src/workloads/profiles.ts index c40278ae..2374e9ec 100644 --- a/packages/agentos/src/workloads/profiles.ts +++ b/packages/agentos/src/workloads/profiles.ts @@ -186,7 +186,7 @@ const persistentMateProfile = profileDefinition({ podLocalHerdr: true, stableWorkload: true, oneWriter: true, - projectedSupervisionIdentity: true, + projectedSupervisionIdentity: false, longLivedServiceIdentity: true, }, defaults: persistentDefaults, diff --git a/packages/agentos/src/workloads/tests/compiler-kustomize.effect.test.ts b/packages/agentos/src/workloads/tests/compiler-kustomize.effect.test.ts index f0ea7cfa..4041a631 100644 --- a/packages/agentos/src/workloads/tests/compiler-kustomize.effect.test.ts +++ b/packages/agentos/src/workloads/tests/compiler-kustomize.effect.test.ts @@ -3,10 +3,6 @@ import { assert, layer } from "@effect/vitest"; import { Effect, FileSystem, Path, Schema } from "effect"; import { renderKustomize } from "../../../../../tooling/testing/kubernetes.ts"; -import { - AGENTOS_EGRESS_TOKEN_AUDIENCE, - AGENTOS_EGRESS_TOKEN_EXPIRATION_SECONDS, -} from "../../access/identity.ts"; import { compileAgentWorkloadSpec } from "../compiler.ts"; const packageRootUrl = new URL("../../../", import.meta.url); @@ -214,35 +210,11 @@ const findStatefulSet = Effect.fn("test.workloadCompiler.findStatefulSet")( }, ); -function assertEgressIdentityProjection(pod: PodSpec) { - const identityVolume = pod.volumes.find( - ({ name }) => name === "agentos-egress-identity", +function assertNoEgressIdentityProjection(pod: PodSpec) { + assert.isUndefined( + pod.volumes.find(({ name }) => name === "agentos-egress-identity"), ); - assert.deepStrictEqual(identityVolume?.projected, { - defaultMode: 288, - sources: [ - { - serviceAccountToken: { - audience: AGENTOS_EGRESS_TOKEN_AUDIENCE, - expirationSeconds: AGENTOS_EGRESS_TOKEN_EXPIRATION_SECONDS, - path: "token", - }, - }, - ], - }); - for (const container of pod.containers) { - assert.deepStrictEqual( - container.volumeMounts?.find(({ name }) => - name === "agentos-egress-identity" - ), - { - mountPath: "/var/run/secrets/agentos-egress", - name: "agentos-egress-identity", - readOnly: true, - }, - ); - } - for (const container of pod.initContainers) { + for (const container of [...pod.initContainers, ...pod.containers]) { assert.isUndefined( container.volumeMounts?.find(({ name }) => name === "agentos-egress-identity" @@ -284,7 +256,7 @@ layer(BunServices.layer)("AgentWorkloadSpec native Kustomize output", (it) => { ); assert.deepStrictEqual( pod.initContainers.map(({ image, name }) => ({ image, name })), - ["install-tools", "prepare-home", "prepare-github-provider"].map( + ["install-tools", "prepare-home"].map( (name) => ({ name, image: workloadImage }), ), ); @@ -299,7 +271,7 @@ layer(BunServices.layer)("AgentWorkloadSpec native Kustomize output", (it) => { databaseVolume?.secret?.secretName, "agentos-crewmate-postgres", ); - assertEgressIdentityProjection(pod); + assertNoEgressIdentityProjection(pod); })); it.effect("renders the persistent Mate and exact released domain controls", () => @@ -330,7 +302,7 @@ layer(BunServices.layer)("AgentWorkloadSpec native Kustomize output", (it) => { ], plan.summary.profileDefinitionDigest, ); - assert.isTrue(pod.automountServiceAccountToken); + assert.isFalse(pod.automountServiceAccountToken); assert.strictEqual(pod.serviceAccountName, "agentos-platform-mate"); assert.deepStrictEqual( pod.containers.map(({ args, image, name }) => ({ args, image, name })), @@ -342,7 +314,7 @@ layer(BunServices.layer)("AgentWorkloadSpec native Kustomize output", (it) => { ); assert.deepStrictEqual( pod.initContainers.map(({ image, name }) => ({ image, name })), - ["install-tools", "prepare-home", "prepare-github-provider"].map( + ["install-tools", "prepare-home"].map( (name) => ({ name, image: workloadImage }), ), ); @@ -357,6 +329,6 @@ layer(BunServices.layer)("AgentWorkloadSpec native Kustomize output", (it) => { databaseVolume?.secret?.secretName, "agentos-secondmate-postgres", ); - assertEgressIdentityProjection(pod); + assertNoEgressIdentityProjection(pod); })); }); diff --git a/packages/agentos/src/workloads/tests/disposable-recovery.effect.test.ts b/packages/agentos/src/workloads/tests/disposable-recovery.effect.test.ts index 84c5a83c..a53f230b 100644 --- a/packages/agentos/src/workloads/tests/disposable-recovery.effect.test.ts +++ b/packages/agentos/src/workloads/tests/disposable-recovery.effect.test.ts @@ -840,7 +840,7 @@ layer(platform)("disposable typed workload recovery", (it) => { "/ready", ); } - assert.isTrue( + assert.isFalse( persistent.spec.template.spec.automountServiceAccountToken, ); assert.isFalse( diff --git a/packages/agentos/src/workloads/tests/fixtures/disposable-distribution/resources/crewmates/default/kubernetes/base/resources.yaml b/packages/agentos/src/workloads/tests/fixtures/disposable-distribution/resources/crewmates/default/kubernetes/base/resources.yaml index 546b5cf8..283f5632 100644 --- a/packages/agentos/src/workloads/tests/fixtures/disposable-distribution/resources/crewmates/default/kubernetes/base/resources.yaml +++ b/packages/agentos/src/workloads/tests/fixtures/disposable-distribution/resources/crewmates/default/kubernetes/base/resources.yaml @@ -67,11 +67,6 @@ spec: command: ["sh", "-c", "exit 0"] securityContext: *container-security resources: *small-resources - - name: prepare-github-provider - image: docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662 - command: ["sh", "-c", "exit 0"] - securityContext: *container-security - resources: *small-resources containers: - name: crewmate image: docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662 @@ -99,18 +94,6 @@ spec: volumeMounts: - name: home mountPath: /home/agent - - name: agentos-egress-identity - mountPath: /var/run/secrets/agentos-egress - readOnly: true - volumes: - - name: agentos-egress-identity - projected: - defaultMode: 0440 - sources: - - serviceAccountToken: - audience: agentos-egress-authz - expirationSeconds: 600 - path: token volumeClaimTemplates: - metadata: name: home diff --git a/packages/agentos/src/workloads/tests/fixtures/disposable-distribution/resources/roles/secondmate/kubernetes/domain/resources.yaml b/packages/agentos/src/workloads/tests/fixtures/disposable-distribution/resources/roles/secondmate/kubernetes/domain/resources.yaml index 7186eb52..7b128363 100644 --- a/packages/agentos/src/workloads/tests/fixtures/disposable-distribution/resources/roles/secondmate/kubernetes/domain/resources.yaml +++ b/packages/agentos/src/workloads/tests/fixtures/disposable-distribution/resources/roles/secondmate/kubernetes/domain/resources.yaml @@ -122,7 +122,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: agentos-secondmate -automountServiceAccountToken: true +automountServiceAccountToken: false --- apiVersion: v1 kind: Service @@ -155,7 +155,7 @@ spec: labels: app.kubernetes.io/name: agentos-secondmate spec: - automountServiceAccountToken: true + automountServiceAccountToken: false serviceAccountName: agentos-secondmate terminationGracePeriodSeconds: 1 securityContext: @@ -187,11 +187,6 @@ spec: command: ["sh", "-c", "exit 0"] securityContext: *container-security resources: *small-resources - - name: prepare-github-provider - image: docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662 - command: ["sh", "-c", "exit 0"] - securityContext: *container-security - resources: *small-resources containers: - name: agentos image: docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662 @@ -219,18 +214,6 @@ spec: volumeMounts: - name: home mountPath: /home/agent - - name: agentos-egress-identity - mountPath: /var/run/secrets/agentos-egress - readOnly: true - volumes: - - name: agentos-egress-identity - projected: - defaultMode: 0440 - sources: - - serviceAccountToken: - audience: agentos-egress-authz - expirationSeconds: 600 - path: token volumeClaimTemplates: - metadata: name: home diff --git a/packages/agentos/src/workloads/tests/profiles.effect.test.ts b/packages/agentos/src/workloads/tests/profiles.effect.test.ts index 5c384392..0ddf266f 100644 --- a/packages/agentos/src/workloads/tests/profiles.effect.test.ts +++ b/packages/agentos/src/workloads/tests/profiles.effect.test.ts @@ -84,7 +84,7 @@ describe("versioned Agent workload profiles", () => { podLocalHerdr: true, stableWorkload: true, oneWriter: true, - projectedSupervisionIdentity: true, + projectedSupervisionIdentity: false, longLivedServiceIdentity: true, }); diff --git a/release/kubernetes/render.ts b/release/kubernetes/render.ts index cd5a60bb..7d757fb3 100755 --- a/release/kubernetes/render.ts +++ b/release/kubernetes/render.ts @@ -367,10 +367,10 @@ function validateFirstMate( } const pod = statefulSet.spec.template.spec; const containers = [...pod.initContainers, ...pod.containers]; - if (containers.length !== 4) { + if (containers.length !== 3) { return Effect.fail(releaseError( "invalid_manifest", - `Expected four First Mate containers, found ${containers.length}.`, + `Expected three First Mate containers, found ${containers.length}.`, )); } if ( @@ -450,8 +450,6 @@ patches: imagePullPolicy: IfNotPresent - name: prepare-home imagePullPolicy: IfNotPresent - - name: prepare-github-provider - imagePullPolicy: IfNotPresent containers: - name: agentos imagePullPolicy: IfNotPresent diff --git a/release/kubernetes/tests/release.effect.test.ts b/release/kubernetes/tests/release.effect.test.ts index 7355d749..d847557b 100644 --- a/release/kubernetes/tests/release.effect.test.ts +++ b/release/kubernetes/tests/release.effect.test.ts @@ -176,11 +176,10 @@ describe("First Mate release artifacts", () => { image, image, image, - image, ]); assert.deepStrictEqual( containers.map(({ imagePullPolicy }) => imagePullPolicy), - ["IfNotPresent", "IfNotPresent", "IfNotPresent", "IfNotPresent"], + ["IfNotPresent", "IfNotPresent", "IfNotPresent"], ); } assert.deepStrictEqual(database, [ diff --git a/services/agentgateway/kubernetes/openai/resources.yaml b/services/agentgateway/kubernetes/openai/resources.yaml index 4676b7ea..352e99dd 100644 --- a/services/agentgateway/kubernetes/openai/resources.yaml +++ b/services/agentgateway/kubernetes/openai/resources.yaml @@ -97,9 +97,11 @@ spec: requests: cpu: 100m memory: 128Mi + ephemeral-storage: 64Mi limits: cpu: "1" memory: 512Mi + ephemeral-storage: 256Mi securityContext: allowPrivilegeEscalation: false capabilities: @@ -107,6 +109,10 @@ spec: - ALL readOnlyRootFilesystem: true runAsNonRoot: true + runAsUser: 2000 + runAsGroup: 2000 + seccompProfile: + type: RuntimeDefault volumeMounts: - name: config mountPath: /config @@ -147,9 +153,11 @@ spec: requests: cpu: 25m memory: 32Mi + ephemeral-storage: 32Mi limits: cpu: 250m memory: 128Mi + ephemeral-storage: 128Mi securityContext: allowPrivilegeEscalation: false capabilities: @@ -159,6 +167,8 @@ spec: runAsNonRoot: true runAsUser: 1000 runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault volumes: - name: config configMap: @@ -188,22 +198,63 @@ spec: app.kubernetes.io/name: agentgateway-openai policyTypes: - Ingress + - Egress ingress: - from: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: agentos podSelector: + matchLabels: + agentos.akua.dev/agentgateway-client: "true" matchExpressions: - - key: agentos.akua.dev/agent - operator: Exists + - key: app.kubernetes.io/name + operator: In + values: + - agentos-crewmate + - agentos-firstmate + - agentos-secondmate - namespaceSelector: matchLabels: agentos.akua.dev/managed-by: agentos-firstmate podSelector: - matchExpressions: - - key: agentos.akua.dev/agent - operator: Exists + matchLabels: + agentos.akua.dev/agentgateway-client: "true" + app.kubernetes.io/name: agentos-crewmate ports: - protocol: TCP port: 4000 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: agentos-egress-authz + ports: + - protocol: TCP + port: 8787 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: ai-gateway + ports: + - protocol: TCP + port: 8787 + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: agentos-otel-collector + ports: + - protocol: TCP + port: 4317 diff --git a/services/agentgateway/tests/kubernetes.effect.test.ts b/services/agentgateway/tests/kubernetes.effect.test.ts index e1887249..445445f8 100644 --- a/services/agentgateway/tests/kubernetes.effect.test.ts +++ b/services/agentgateway/tests/kubernetes.effect.test.ts @@ -23,6 +23,11 @@ const ProbeSchema = Schema.Struct({ port: Schema.Union([Schema.String, Schema.Number]), }), }); +const ResourceListSchema = Schema.Struct({ + cpu: Schema.String, + memory: Schema.String, + "ephemeral-storage": Schema.optional(Schema.String), +}); const ContainerSchema = Schema.Struct({ name: Schema.String, @@ -32,6 +37,21 @@ const ContainerSchema = Schema.Struct({ livenessProbe: Schema.optional(ProbeSchema), readinessProbe: Schema.optional(ProbeSchema), startupProbe: Schema.optional(ProbeSchema), + resources: Schema.Struct({ + requests: ResourceListSchema, + limits: ResourceListSchema, + }), + securityContext: Schema.Struct({ + allowPrivilegeEscalation: Schema.Literal(false), + capabilities: Schema.Struct({ drop: Schema.Tuple([Schema.Literal("ALL")]) }), + readOnlyRootFilesystem: Schema.Literal(true), + runAsNonRoot: Schema.Literal(true), + runAsUser: Schema.optional(Schema.Number), + runAsGroup: Schema.optional(Schema.Number), + seccompProfile: Schema.optional( + Schema.Struct({ type: Schema.Literal("RuntimeDefault") }), + ), + }), volumeMounts: Schema.optional(Schema.Array(Schema.Struct({ name: Schema.String, mountPath: Schema.String, @@ -67,8 +87,58 @@ const DeploymentSpecSchema = Schema.Struct({ const PdbSpecSchema = Schema.Struct({ minAvailable: Schema.Literal(1) }); const NetworkPolicySpecSchema = Schema.Struct({ - policyTypes: Schema.Tuple([Schema.Literal("Ingress")]), + policyTypes: Schema.Array(Schema.Literals(["Ingress", "Egress"])), +}); +const LabelSelectorSchema = Schema.Struct({ + matchLabels: Schema.optional(Schema.Record(Schema.String, Schema.String)), + matchExpressions: Schema.optional(Schema.Array(Schema.Struct({ + key: Schema.String, + operator: Schema.Literal("In"), + values: Schema.Array(Schema.String), + }))), +}); +const OpenaiNetworkPolicySpecSchema = Schema.Struct({ + policyTypes: Schema.Tuple([ + Schema.Literal("Ingress"), + Schema.Literal("Egress"), + ]), + ingress: Schema.Tuple([Schema.Struct({ + from: Schema.Tuple([ + Schema.Struct({ + namespaceSelector: LabelSelectorSchema, + podSelector: LabelSelectorSchema, + }), + Schema.Struct({ + namespaceSelector: LabelSelectorSchema, + podSelector: LabelSelectorSchema, + }), + ]), + ports: Schema.Tuple([Schema.Struct({ + protocol: Schema.Literal("TCP"), + port: Schema.Literal(4000), + })]), + })]), }); + +const selectorMatches = ( + selector: typeof LabelSelectorSchema.Type, + labels: Readonly>, +) => + Object.entries(selector.matchLabels ?? {}).every( + ([key, value]) => labels[key] === value, + ) && (selector.matchExpressions ?? []).every( + ({ key, values }) => labels[key] !== undefined && values.includes(labels[key]), + ); + +const peerMatches = ( + peer: { + readonly namespaceSelector: typeof LabelSelectorSchema.Type; + readonly podSelector: typeof LabelSelectorSchema.Type; + }, + namespaceLabels: Readonly>, + podLabels: Readonly>, +) => selectorMatches(peer.namespaceSelector, namespaceLabels) && + selectorMatches(peer.podSelector, podLabels); const repositoryRoot = new URL("../../..", import.meta.url); const render = Effect.fn("test.agentgateway.renderKustomize")(function*() { @@ -147,6 +217,23 @@ describe("owned agentgateway Kustomize workloads", () => { ? ["agentos-github-tls"] : [], ); + for (const container of deployment.metadata.name === "agentgateway-openai" + ? spec.template.spec.containers + : []) { + assert.deepInclude(container.securityContext, { + allowPrivilegeEscalation: false, + capabilities: { drop: ["ALL"] }, + readOnlyRootFilesystem: true, + runAsNonRoot: true, + seccompProfile: { type: "RuntimeDefault" }, + }); + assert.hasAllKeys(container.resources.requests, [ + "cpu", "memory", "ephemeral-storage", + ]); + assert.hasAllKeys(container.resources.limits, [ + "cpu", "memory", "ephemeral-storage", + ]); + } } const serviceAccounts = resources.filter( @@ -171,8 +258,102 @@ describe("owned agentgateway Kustomize workloads", () => { assert.lengthOf(policies, 2); yield* Effect.forEach(policies, ({ spec }) => Schema.decodeUnknownEffect(NetworkPolicySpecSchema)(spec)); - assert.notInclude(manifest, "policyTypes:\n - Egress"); + const openaiPolicy = policies.find(({ metadata }) => + metadata.name === "agentgateway-openai" + ); + assert.deepStrictEqual( + yield* Schema.decodeUnknownEffect(OpenaiNetworkPolicySpecSchema)( + openaiPolicy?.spec, + ), + { + policyTypes: ["Ingress", "Egress"], + ingress: [{ + from: [ + { + namespaceSelector: { + matchLabels: { "kubernetes.io/metadata.name": "agentos" }, + }, + podSelector: { + matchLabels: { + "agentos.akua.dev/agentgateway-client": "true", + }, + matchExpressions: [{ + key: "app.kubernetes.io/name", + operator: "In", + values: [ + "agentos-crewmate", + "agentos-firstmate", + "agentos-secondmate", + ], + }], + }, + }, + { + namespaceSelector: { + matchLabels: { + "agentos.akua.dev/managed-by": "agentos-firstmate", + }, + }, + podSelector: { + matchLabels: { + "agentos.akua.dev/agentgateway-client": "true", + "app.kubernetes.io/name": "agentos-crewmate", + }, + }, + }, + ], + ports: [{ protocol: "TCP", port: 4000 }], + }], + }, + ); + const policy = yield* Schema.decodeUnknownEffect( + OpenaiNetworkPolicySpecSchema, + )(openaiPolicy?.spec); + const [agentosPeer, managedPeer] = policy.ingress[0].from; + const client = { "agentos.akua.dev/agentgateway-client": "true" }; + for (const name of [ + "agentos-crewmate", + "agentos-firstmate", + "agentos-secondmate", + ]) { + assert.isTrue(peerMatches( + agentosPeer, + { "kubernetes.io/metadata.name": "agentos" }, + { ...client, "app.kubernetes.io/name": name }, + )); + } + assert.isFalse(peerMatches( + agentosPeer, + { "kubernetes.io/metadata.name": "agentos" }, + { ...client, "app.kubernetes.io/name": "unrelated" }, + )); + assert.isFalse(peerMatches( + agentosPeer, + { "kubernetes.io/metadata.name": "agentos" }, + { "app.kubernetes.io/name": "agentos-firstmate" }, + )); + assert.isTrue(peerMatches( + managedPeer, + { "agentos.akua.dev/managed-by": "agentos-firstmate" }, + { ...client, "app.kubernetes.io/name": "agentos-crewmate" }, + )); + for (const name of ["agentos-firstmate", "agentos-secondmate"]) { + assert.isFalse(peerMatches( + managedPeer, + { "agentos.akua.dev/managed-by": "agentos-firstmate" }, + { ...client, "app.kubernetes.io/name": name }, + )); + } + assert.isFalse(peerMatches( + managedPeer, + { "agentos.akua.dev/managed-by": "agentos-firstmate" }, + { "app.kubernetes.io/name": "agentos-crewmate" }, + )); assert.notInclude(manifest, "kind: Secret"); + assert.notInclude(manifest, "kind: Ingress"); + assert.notInclude(manifest, "kind: PersistentVolumeClaim"); + assert.notInclude(manifest, "type: LoadBalancer"); + assert.notInclude(manifest, "type: NodePort"); }).pipe(Effect.provide(BunServices.layer)))); it.effect("rolls immutable route configuration through generated ConfigMaps", () => diff --git a/services/ai-gateway/README.md b/services/ai-gateway/README.md index 330289e4..bc5fc725 100644 --- a/services/ai-gateway/README.md +++ b/services/ai-gateway/README.md @@ -62,8 +62,7 @@ native Pi/Codex configuration. Responses clients that cannot reread a projected Kubernetes token on every request use the loopback-only `ai-gateway-workload-proxy` sidecar. Its exact Responses paths, identity and failure contract, timeout settings, health -checks, Pod wiring, and migration from the legacy shared-token topology are -documented in +checks, and Pod wiring are documented in [`docs/integrations/hermes-ai-gateway.md`](../../docs/integrations/hermes-ai-gateway.md). `GET /healthz` proves only that the process is serving. `GET /readyz` requires diff --git a/services/ai-gateway/kubernetes/statefulset.yaml b/services/ai-gateway/kubernetes/statefulset.yaml index 3a794234..be63d68c 100644 --- a/services/ai-gateway/kubernetes/statefulset.yaml +++ b/services/ai-gateway/kubernetes/statefulset.yaml @@ -124,8 +124,6 @@ spec: value: /opt/agentos - name: AI_GATEWAY_STATE_DIR value: /var/lib/ai-gateway - - name: AI_GATEWAY_CLIENT_AUTH_MODE - value: workload_identity - name: AGENTOS_PROVIDER_BUDGET_SETTLEMENT_BASE_URL value: http://agentos-egress-authz.agentos.svc.cluster.local:9001 - name: AGENTOS_PROVIDER_BUDGET_SETTLEMENT_TOKEN_FILE diff --git a/services/ai-gateway/kubernetes/workload-proxy/component/kustomization.yaml b/services/ai-gateway/kubernetes/workload-proxy/component/kustomization.yaml new file mode 100644 index 00000000..920d7fbd --- /dev/null +++ b/services/ai-gateway/kubernetes/workload-proxy/component/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +patches: + - path: workload-proxy.patch.yaml + target: + group: apps + version: v1 + kind: StatefulSet diff --git a/services/ai-gateway/kubernetes/workload-proxy/component/workload-proxy.patch.yaml b/services/ai-gateway/kubernetes/workload-proxy/component/workload-proxy.patch.yaml new file mode 100644 index 00000000..22ca4091 --- /dev/null +++ b/services/ai-gateway/kubernetes/workload-proxy/component/workload-proxy.patch.yaml @@ -0,0 +1,72 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ignored-by-target +spec: + template: + metadata: + labels: + agentos.akua.dev/agentgateway-client: "true" + spec: + containers: + - name: ai-gateway-workload-proxy + image: agentos:dev + imagePullPolicy: Never + command: + - ai-gateway-workload-proxy + env: + - name: AI_GATEWAY_URL + value: http://agentgateway-openai.agentos.svc.cluster.local:8788 + - name: AI_GATEWAY_IDLE_TIMEOUT_SECONDS + value: "255" + - name: AI_GATEWAY_GRACEFUL_SHUTDOWN_MILLIS + value: "20000" + - name: AGENTOS_EGRESS_TOKEN_FILE + value: /var/run/secrets/agentos-egress/token + ports: + - name: workload-proxy + containerPort: 8790 + protocol: TCP + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /livez + port: workload-proxy + readinessProbe: + httpGet: + host: 127.0.0.1 + path: /readyz + port: workload-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 25m + memory: 64Mi + ephemeral-storage: 32Mi + limits: + cpu: 250m + memory: 256Mi + ephemeral-storage: 128Mi + volumeMounts: + - name: agentos-egress-identity + mountPath: /var/run/secrets/agentos-egress + readOnly: true + volumes: + - name: agentos-egress-identity + projected: + defaultMode: 0440 + sources: + - serviceAccountToken: + audience: agentos-egress-authz + expirationSeconds: 600 + path: token diff --git a/services/ai-gateway/src/cli.ts b/services/ai-gateway/src/cli.ts index bd02da61..d8f4e1a9 100644 --- a/services/ai-gateway/src/cli.ts +++ b/services/ai-gateway/src/cli.ts @@ -101,7 +101,7 @@ export const runAIGatewayCli = Effect.fn( const token = operatorCredential(config); if (Redacted.value(token) === "") { yield* output.error( - "AI_GATEWAY_OPERATOR_TOKEN or legacy AI_GATEWAY_TOKEN is required for status", + "AI_GATEWAY_OPERATOR_TOKEN is required for status", ); return 1; } @@ -139,11 +139,7 @@ export const runAIGatewayCli = Effect.fn( requireAIGatewayServeConfig(config), ); if (Result.isFailure(serveConfigResult)) { - yield* output.error( - serveConfigResult.failure.code === "client_identity_unavailable" - ? "AI_GATEWAY_TOKEN is required to serve" - : "ai-gateway serve failed (AIGatewayEntrypointError)", - ); + yield* output.error("ai-gateway serve failed (AIGatewayEntrypointError)"); return 1; } const serveConfig = serveConfigResult.success; @@ -166,9 +162,7 @@ export const runAIGatewayCli = Effect.fn( function operatorCredential( config: AIGatewayConfig, ): Redacted.Redacted { - return Redacted.value(config.operatorToken) === "" - ? config.clientToken - : config.operatorToken; + return config.operatorToken; } function exitStatus( diff --git a/services/ai-gateway/src/config.ts b/services/ai-gateway/src/config.ts index 2a392bc0..25a795a6 100644 --- a/services/ai-gateway/src/config.ts +++ b/services/ai-gateway/src/config.ts @@ -29,12 +29,6 @@ const RawConfig = Config.all({ gracefulShutdownMillis: Config.int( "AI_GATEWAY_GRACEFUL_SHUTDOWN_MILLIS", ).pipe(Config.withDefault(defaultAIGatewayGracefulShutdownMillis)), - clientAuthenticationMode: Config.string( - "AI_GATEWAY_CLIENT_AUTH_MODE", - ).pipe(Config.withDefault("shared_token")), - clientToken: Config.redacted("AI_GATEWAY_TOKEN").pipe( - Config.withDefault(Redacted.make("")), - ), operatorToken: Config.redacted("AI_GATEWAY_OPERATOR_TOKEN").pipe( Config.withDefault(Redacted.make("")), ), @@ -92,8 +86,6 @@ export interface AIGatewayConfig { readonly port: number; readonly idleTimeoutSeconds: number; readonly gracefulShutdownMillis: number; - readonly clientAuthenticationMode: "shared_token" | "workload_identity"; - readonly clientToken: Redacted.Redacted; readonly operatorToken: Redacted.Redacted; readonly allowApiKeyFallback: boolean; readonly openAIApiKey: Redacted.Redacted; @@ -107,14 +99,7 @@ export interface AIGatewayConfig { readonly settlementMaximumResponseBytes: number; } -export type AIGatewayServeAuthentication = - | { - readonly kind: "shared_token"; - readonly token: Redacted.Redacted; - } - | { - readonly kind: "workload_identity"; - }; +export type AIGatewayServeAuthentication = { readonly kind: "workload_identity" }; export interface AIGatewayServeConfig extends AIGatewayConfig { readonly authentication: AIGatewayServeAuthentication; @@ -133,18 +118,12 @@ export const loadAIGatewayConfig = Effect.fn( if (!validConfiguration(raw, stateDirectory)) { return yield* aiGatewayEntrypointError("invalid_configuration"); } - const clientAuthenticationMode = raw.clientAuthenticationMode === - "workload_identity" - ? "workload_identity" - : "shared_token"; return { stateDirectory, hostname: raw.hostname, port: raw.port, idleTimeoutSeconds: raw.idleTimeoutSeconds, gracefulShutdownMillis: raw.gracefulShutdownMillis, - clientAuthenticationMode, - clientToken: raw.clientToken, operatorToken: raw.operatorToken, allowApiKeyFallback: raw.allowApiKeyFallback, openAIApiKey: raw.openAIApiKey, @@ -162,21 +141,8 @@ export const loadAIGatewayConfig = Effect.fn( export const requireAIGatewayServeConfig = Effect.fn( "agentos.aiGateway.requireServeConfig", )(function*(config: AIGatewayConfig) { - const clientToken = Redacted.value(config.clientToken); - const configuredOperatorToken = Redacted.value(config.operatorToken); - const operatorToken = configuredOperatorToken === "" - ? config.clientToken - : config.operatorToken; - if ( - config.clientAuthenticationMode === "shared_token" && - clientToken === "" - ) { - return yield* aiGatewayEntrypointError("client_identity_unavailable"); - } - const authentication: AIGatewayServeAuthentication = - config.clientAuthenticationMode === "workload_identity" - ? { kind: "workload_identity" } - : { kind: "shared_token", token: config.clientToken }; + const operatorToken = config.operatorToken; + const authentication: AIGatewayServeAuthentication = { kind: "workload_identity" }; return { ...config, authentication, @@ -188,17 +154,12 @@ function validConfiguration( raw: Config.Success, stateDirectory: string, ): boolean { - const clientToken = Redacted.value(raw.clientToken); const operatorToken = Redacted.value(raw.operatorToken); const openAIApiKey = Redacted.value(raw.openAIApiKey); return ( - ["shared_token", "workload_identity"].includes( - raw.clientAuthenticationMode, - ) && validString(raw.hostname, 253) && validPath(stateDirectory) && validPath(raw.settlementTokenPath) && - validSecret(clientToken) && validSecret(operatorToken) && validSecret(openAIApiKey) && validPositiveInteger(raw.port, 65_535) && diff --git a/services/ai-gateway/src/forward.ts b/services/ai-gateway/src/forward.ts index 209d79fb..a008dc0d 100644 --- a/services/ai-gateway/src/forward.ts +++ b/services/ai-gateway/src/forward.ts @@ -10,11 +10,13 @@ import { type ProviderAuthorizationError, type ProviderAuthorizationGrantV1, type ProviderBudgetSettlementReporter, + type ProviderBudgetAttemptRenewalReporter, type ProviderBudgetSettlementReportV1, } from "@akua-dev/agentos"; import { Cause, Crypto, + Deferred, Effect, Exit, Option, @@ -25,7 +27,6 @@ import { } from "effect"; import { attributedSessionKey } from "./attribution.ts"; -import { timingSafeStringEqual } from "./constant-time.ts"; import { AIGatewayTelemetry, type AIGatewayRequestTelemetry, @@ -71,14 +72,7 @@ export interface AIForwardLease { ) => Effect.Effect; } -export type AIForwardClientAuthentication = - | { - readonly kind: "shared_token"; - readonly token: string; - } - | { - readonly kind: "workload_identity"; - }; +export type AIForwardClientAuthentication = { readonly kind: "workload_identity" }; export interface AIForwardOptions { readonly authentication: AIForwardClientAuthentication; @@ -94,6 +88,7 @@ export interface AIForwardOptions { ) => Effect.Effect; readonly provider: AIProviderHttp["Service"]; readonly settlements: ProviderBudgetSettlementReporter["Service"]; + readonly attemptRenewals?: ProviderBudgetAttemptRenewalReporter["Service"]; readonly now: Effect.Effect; readonly heartbeatMillis: number; readonly maximumUsageEventBytes: number; @@ -156,6 +151,22 @@ export const makeAIForwardHandler = Effect.fn( authentication.status === 403 ? "forbidden" : "unauthorized", ); } + if ( + authentication.authorization !== undefined && + "model" in authentication.authorization + ) { + const bounded = yield* Effect.result(validateWorkloadRequestCeiling( + request, + authentication.authorization, + )); + if (Result.isFailure(bounded)) { + yield* diagnostic(requestTelemetry.end({ + status: 403, + streamOutcome: "not_streamed", + })); + return jsonResponse(403, "forbidden"); + } + } if ( request.method !== "POST" || !isSupportedResponsePath(url.pathname) @@ -201,6 +212,11 @@ export const makeAIForwardHandler = Effect.fn( (lease, transfer) => { if (lease === undefined) { return Effect.gen(function*() { + yield* settleAttempt( + options.settlements, + authentication.authorization, + "transport_failed", + ); yield* diagnostic(requestTelemetry.routeEnded("unavailable")); yield* diagnostic(requestTelemetry.end({ status: 503, @@ -238,6 +254,11 @@ export const makeAIForwardHandler = Effect.fn( yield* diagnostic(requestTelemetry.routeEnded("acquired")); const upstreamRequest = yield* makeUpstreamRequest(request, url, lease); if (upstreamRequest === undefined) { + yield* settleAttempt( + options.settlements, + authentication.authorization, + "transport_failed", + ); yield* releaseLeaseOnce; yield* diagnostic(requestTelemetry.end({ status: 400, @@ -255,6 +276,11 @@ export const makeAIForwardHandler = Effect.fn( yield* diagnostic( requestTelemetry.upstreamFailed(upstreamResult.failure), ); + yield* settleAttempt( + options.settlements, + authentication.authorization, + "transport_failed", + ); yield* releaseLeaseOnce; yield* diagnostic(requestTelemetry.end({ status: 502, @@ -274,6 +300,11 @@ export const makeAIForwardHandler = Effect.fn( yield* diagnostic( requestTelemetry.upstreamFailed(headersResult.failure), ); + yield* settleAttempt( + options.settlements, + authentication.authorization, + "transport_failed", + ); yield* releaseLeaseOnce; yield* diagnostic(requestTelemetry.end({ status: 502, @@ -296,6 +327,11 @@ export const makeAIForwardHandler = Effect.fn( yield* diagnostic( requestTelemetry.upstreamFailed(responseMetadata.failure), ); + yield* settleAttempt( + options.settlements, + authentication.authorization, + "transport_failed", + ); yield* releaseLeaseOnce; yield* diagnostic(requestTelemetry.end({ status: 502, @@ -323,6 +359,11 @@ export const makeAIForwardHandler = Effect.fn( code: "invalid_configuration", }); yield* diagnostic(requestTelemetry.upstreamFailed(failure)); + yield* settleAttempt( + options.settlements, + authentication.authorization, + "transport_failed", + ); yield* releaseLeaseOnce; yield* diagnostic(requestTelemetry.end({ status: 502, @@ -339,12 +380,18 @@ export const makeAIForwardHandler = Effect.fn( })) : undefined; if (observerResult !== undefined && Result.isFailure(observerResult)) { + yield* settleAttempt( + options.settlements, + authentication.authorization, + "transport_failed", + ); yield* releaseLeaseOnce; return jsonResponse(503, "accounting_unavailable"); } const observer = observerResult === undefined ? undefined : observerResult.success; + const heartbeatFailed = yield* Ref.make(false); const monitored = upstream.body.pipe( Stream.tap((chunk) => diagnostic(requestTelemetry.streamChunk(chunk.byteLength)) @@ -362,14 +409,24 @@ export const makeAIForwardHandler = Effect.fn( observer, requestTelemetry, request.signal, + heartbeatFailed, ) ), ); const scoped = Stream.unwrap(Effect.gen(function*() { - yield* heartbeat(lease, options.heartbeatMillis).pipe( + if (options.attemptRenewals === undefined) return monitored; + const heartbeatFailure = yield* Deferred.make(); + yield* heartbeat( + lease, + options.attemptRenewals, + authentication.authorization?.decisionRef, + options.heartbeatMillis, + heartbeatFailure, + heartbeatFailed, + ).pipe( Effect.forkScoped({ startImmediately: true }), ); - return monitored; + return monitored.pipe(Stream.interruptWhen(Deferred.await(heartbeatFailure))); })); const body = yield* Stream.toReadableStreamEffect(scoped); const responseResult = yield* Effect.result(finiteResponse( @@ -378,6 +435,11 @@ export const makeAIForwardHandler = Effect.fn( responseHeaders, )); if (Result.isFailure(responseResult)) { + yield* settleAttempt( + options.settlements, + authentication.authorization, + "transport_failed", + ); yield* releaseLeaseOnce; return jsonResponse(502, "invalid_provider_response"); } @@ -390,6 +452,11 @@ export const makeAIForwardHandler = Effect.fn( ).pipe( Effect.catchTag("AIForwardRouteError", (failure) => Effect.gen(function*() { + yield* settleAttempt( + options.settlements, + authentication.authorization, + "transport_failed", + ); yield* diagnostic(requestTelemetry.routeEnded("error", failure)); yield* diagnostic(requestTelemetry.end({ status: 503, @@ -420,13 +487,6 @@ function authenticateClient( url: URL, now: Effect.Effect, ): Effect.Effect { - if (authentication.kind === "shared_token") { - return Effect.succeed( - isClientAuthorized(request, authentication.token) - ? { authenticated: true, authorization: undefined } - : { authenticated: false, status: 401 }, - ); - } return Effect.gen(function*() { const currentTime = yield* now; const body = request.headers.get("x-agentos-authz-principal-kind") === @@ -482,15 +542,48 @@ function authorizationFailureStatus( : 401; } -function isClientAuthorized(request: Request, expected: string): boolean { - if (expected.length === 0) return false; - const dedicated = request.headers.get("x-ai-gateway-token")?.trim(); - const authorization = request.headers.get("authorization")?.trim(); - const bearer = authorization?.toLowerCase().startsWith("bearer ") - ? authorization.slice(7).trim() - : undefined; - const actual = dedicated ?? bearer ?? ""; - return timingSafeStringEqual(actual, expected); +const WorkloadResponsesRequestSchema = Schema.fromJsonString(Schema.Struct({ + model: Schema.String, + stream: Schema.Literal(true), + max_output_tokens: Schema.Number.pipe( + Schema.check(Schema.isInt(), Schema.isGreaterThan(0)), + ), +})); + +function validateWorkloadRequestCeiling( + request: Request, + grant: Extract, +) { + return Effect.tryPromise({ + try: () => request.clone().text(), + catch: () => AIForwardConfigurationError.make({ code: "invalid_configuration" }), + }).pipe( + Effect.flatMap((body) => + Schema.decodeUnknownEffect(WorkloadResponsesRequestSchema)(body).pipe( + Effect.map((payload) => ({ body, payload })), + ) + ), + Effect.filterOrFail( + ({ body, payload }) => + payload.model === grant.model && (() => { + const inputCeiling = new TextEncoder().encode(body).byteLength; + const requestedTokens = inputCeiling + payload.max_output_tokens; + const requestedSpendMicros = pricedSpend( + inputCeiling, + payload.max_output_tokens, + grant.pricing.inputMicrosPerMillionTokens, + grant.pricing.outputMicrosPerMillionTokens, + ); + return Number.isSafeInteger(requestedTokens) && + requestedTokens === grant.requestedTokens && + requestedTokens <= grant.limits.maximumTokens && + requestedSpendMicros === grant.requestedSpendMicros && + requestedSpendMicros <= grant.limits.maximumSpendMicros; + })(), + () => AIForwardConfigurationError.make({ code: "invalid_configuration" }), + ), + Effect.asVoid, + ); } function makeUpstreamRequest( @@ -533,12 +626,34 @@ function makeUpstreamRequest( }); } -function heartbeat(lease: AIForwardLease, heartbeatMillis: number) { +function heartbeat( + lease: AIForwardLease, + attemptRenewals: ProviderBudgetAttemptRenewalReporter["Service"], + decisionRef: string | undefined, + heartbeatMillis: number, + failure: Deferred.Deferred, + failed: Ref.Ref, +) { return Effect.sleep(heartbeatMillis).pipe( - Effect.andThen(lease.renew), - Effect.flatMap((renewed) => renewed ? Effect.void : Effect.interrupt), + Effect.andThen(Effect.all([ + lease.renew, + decisionRef === undefined + ? Effect.succeed(false) + : attemptRenewals.renew({ schemaVersion: 1, decisionRef }).pipe(Effect.as(true)), + ])), + Effect.flatMap(([routerRenewed, attemptRenewed]) => routerRenewed && attemptRenewed + ? Effect.void + : Ref.set(failed, true).pipe( + Effect.andThen(Deferred.succeed(failure, undefined)), + Effect.andThen(Effect.interrupt), + )), + Effect.catch((error) => + Ref.set(failed, true).pipe( + Effect.andThen(Deferred.succeed(failure, undefined)), + Effect.andThen(Effect.fail(error)), + ) + ), Effect.forever, - Effect.catchCause(() => Effect.void), ); } @@ -551,6 +666,7 @@ function finalizeStream( observer: OpenAITerminalUsageObserver | undefined, telemetry: AIGatewayRequestTelemetry, signal: AbortSignal, + heartbeatFailed: Ref.Ref, ): Effect.Effect { const maximumReleaseAttempts = 3; let attempts = 0; @@ -570,6 +686,7 @@ function finalizeStream( }, }); return Effect.gen(function*() { + const failedHeartbeat = yield* Ref.get(heartbeatFailed); if (authorization !== undefined) { if (status >= 400) { yield* reportSettlement( @@ -579,13 +696,36 @@ function finalizeStream( } else if (observer !== undefined) { const usage = yield* Effect.option(observer.finish); if (Option.isSome(usage)) { + const spendMicros = "model" in authorization + ? pricedSpend( + usage.value.inputTokens, + usage.value.outputTokens, + authorization.pricing.inputMicrosPerMillionTokens, + authorization.pricing.outputMicrosPerMillionTokens, + ) + : usage.value.spendMicros; yield* reportSettlement(settlements, { schemaVersion: 1, decisionRef: authorization.decisionRef, - forwardOutcome: streamOutcome(exit), + forwardOutcome: streamOutcome(exit, failedHeartbeat), ...usage.value, + spendMicros, }); + } else if (Exit.isSuccess(exit)) { + return yield* Effect.die("successful provider stream omitted terminal usage"); + } else { + yield* reportSettlement(settlements, zeroUsageReport( + authorization.decisionRef, + streamOutcome(exit, failedHeartbeat), + )); } + } else if (status < 400 && Exit.isSuccess(exit)) { + return yield* Effect.die("successful provider response was not an accounted event stream"); + } else { + yield* reportSettlement( + settlements, + zeroUsageReport(authorization.decisionRef, streamOutcome(exit, failedHeartbeat)), + ); } } yield* releaseStreamLease; @@ -599,23 +739,49 @@ function finalizeStream( ...(failure === undefined ? {} : { error: failure }), })); }).pipe( - Effect.catchCause(() => releaseStreamLease), - Effect.catchCause(() => Effect.void), + Effect.catchCause((cause) => + releaseStreamLease.pipe(Effect.andThen(Effect.failCause(cause))) + ), Effect.uninterruptible, ); } +function pricedSpend( + inputTokens: number, + outputTokens: number, + inputMicrosPerMillionTokens: number, + outputMicrosPerMillionTokens: number, +) { + const numerator = inputTokens * inputMicrosPerMillionTokens + + outputTokens * outputMicrosPerMillionTokens; + return Number.isSafeInteger(numerator) && numerator > 0 + ? Math.ceil(numerator / 1_000_000) + : Number.NaN; +} + function settleWithoutBody( settlements: ProviderBudgetSettlementReporter["Service"], authorization: ProviderAuthorizationGrantV1 | undefined, status: number, ) { - return authorization !== undefined && status >= 400 - ? reportSettlement( + return settleAttempt( + settlements, + authorization, + status >= 400 ? "provider_rejected" : "completed", + ); +} + +function settleAttempt( + settlements: ProviderBudgetSettlementReporter["Service"], + authorization: ProviderAuthorizationGrantV1 | undefined, + outcome: ProviderBudgetSettlementReportV1["forwardOutcome"], +) { + return authorization === undefined + ? Effect.void + : reportSettlement( settlements, - zeroUsageReport(authorization.decisionRef, "provider_rejected"), - ) - : Effect.void; + zeroUsageReport(authorization.decisionRef, outcome), + ); } function zeroUsageReport( @@ -638,8 +804,9 @@ function reportSettlement( report: ProviderBudgetSettlementReportV1, ) { return settlements.report(report).pipe( + Effect.retry({ times: 2 }), + Effect.orDie, Effect.asVoid, - Effect.catchCause(() => Effect.void), Effect.uninterruptible, ); } @@ -678,8 +845,10 @@ function recordResponse( function streamOutcome( exit: Exit.Exit, + heartbeatFailed = false, ): ProviderBudgetSettlementReportV1["forwardOutcome"] { if (Exit.isSuccess(exit)) return "completed"; + if (heartbeatFailed) return "transport_failed"; return Cause.interruptors(exit.cause).size > 0 ? "cancelled" : "transport_failed"; diff --git a/services/ai-gateway/src/gateway-service.ts b/services/ai-gateway/src/gateway-service.ts index 266eac25..cdcf624a 100644 --- a/services/ai-gateway/src/gateway-service.ts +++ b/services/ai-gateway/src/gateway-service.ts @@ -5,6 +5,7 @@ import { ProviderBudgetReservationRequester, ProviderBudgetSettlementReadiness, ProviderBudgetSettlementReporter, + ProviderBudgetAttemptRenewalReporter, denyProviderBudgetReservationRequester, providerAuthorizationGrantHeaders, type ProviderAuthorizationGrantV1, @@ -75,9 +76,7 @@ export const makeAIGatewayApplication = Effect.fn( options.maximumUsageEventBytes < 1 || !Number.isSafeInteger(options.usageCacheMillis) || options.usageCacheMillis < 1 || - options.operatorToken.length > 16 * 1_024 || - (options.authentication.kind === "shared_token" && - options.authentication.token.length === 0) + options.operatorToken.length > 16 * 1_024 ) { return yield* AIGatewayApplicationError.make({ code: "invalid_configuration", @@ -93,6 +92,9 @@ export const makeAIGatewayApplication = Effect.fn( ); const settlementReadiness = yield* ProviderBudgetSettlementReadiness; const settlements = yield* ProviderBudgetSettlementReporter; + const attemptRenewals = Option.getOrUndefined( + yield* Effect.serviceOption(ProviderBudgetAttemptRenewalReporter), + ); const usage = yield* Ref.make>(new Map()); const fallbackKey = options.openAIApiKey?.trim(); @@ -277,6 +279,7 @@ export const makeAIGatewayApplication = Effect.fn( acquire(sessionKey, signal, authorization, telemetry, use), provider, settlements, + ...(attemptRenewals === undefined ? {} : { attemptRenewals }), now: Clock.currentTimeMillis, heartbeatMillis: options.heartbeatMillis, maximumUsageEventBytes: options.maximumUsageEventBytes, diff --git a/services/ai-gateway/src/main.ts b/services/ai-gateway/src/main.ts index 550039ed..872e804f 100755 --- a/services/ai-gateway/src/main.ts +++ b/services/ai-gateway/src/main.ts @@ -8,7 +8,7 @@ import * as BunPath from "@effect/platform-bun/BunPath"; import * as BunRuntime from "@effect/platform-bun/BunRuntime"; import { ProviderBudgetReservationRequester, - denyProviderBudgetReservationRequester, + ProviderBudgetAttemptRenewalReporter, makeProviderBudgetSettlementHttpLayer, ProviderBudgetSettlementReadiness, ProviderBudgetSettlementReporter, @@ -184,6 +184,8 @@ function makeAIGatewayRuntimeLive( const settlementServices = yield* Effect.all({ readiness: ProviderBudgetSettlementReadiness, reporter: ProviderBudgetSettlementReporter, + reservation: ProviderBudgetReservationRequester, + renewal: ProviderBudgetAttemptRenewalReporter, }).pipe(Effect.provide(settlementLayer)); const serve = Effect.fn("agentos.aiGateway.serve")( @@ -212,14 +214,9 @@ function makeAIGatewayRuntimeLive( aiGatewayEntrypointError("invalid_configuration") ), ); - const clientAuthentication: AIForwardClientAuthentication = - serveConfig.authentication.kind === - "workload_identity" - ? { kind: "workload_identity" } - : { - kind: "shared_token", - token: Redacted.value(serveConfig.authentication.token), - }; + const clientAuthentication: AIForwardClientAuthentication = { + kind: "workload_identity", + }; const openAIApiKey = Redacted.value(serveConfig.openAIApiKey); const application = yield* makeAIGatewayApplication({ authentication: clientAuthentication, @@ -237,12 +234,16 @@ function makeAIGatewayRuntimeLive( Effect.provideService(CodexQuota, quota), Effect.provideService( ProviderBudgetReservationRequester, - denyProviderBudgetReservationRequester, + settlementServices.reservation, ), Effect.provideService( ProviderBudgetSettlementReporter, settlementServices.reporter, ), + Effect.provideService( + ProviderBudgetAttemptRenewalReporter, + settlementServices.renewal, + ), Effect.provideService( ProviderBudgetSettlementReadiness, settlementServices.readiness, diff --git a/services/ai-gateway/tests/attribution.effect.test.ts b/services/ai-gateway/tests/attribution.effect.test.ts index 41adfebb..7305f596 100644 --- a/services/ai-gateway/tests/attribution.effect.test.ts +++ b/services/ai-gateway/tests/attribution.effect.test.ts @@ -73,6 +73,9 @@ function workloadGrant(): Extract< spendWindowMillis: 3_600_000, maximumSpendMicros: 2_000_000, }, + pricing: { version: 1, inputMicrosPerMillionTokens: 2_000_000, outputMicrosPerMillionTokens: 8_000_000 }, + requestedTokens: 1_024, + requestedSpendMicros: 8_192, }; } diff --git a/services/ai-gateway/tests/cli.effect.test.ts b/services/ai-gateway/tests/cli.effect.test.ts index a68cdac1..f0bc2d77 100644 --- a/services/ai-gateway/tests/cli.effect.test.ts +++ b/services/ai-gateway/tests/cli.effect.test.ts @@ -41,8 +41,6 @@ function configuration( port: 8787, idleTimeoutSeconds: 255, gracefulShutdownMillis: 20_000, - clientAuthenticationMode: "shared_token", - clientToken: Redacted.make("fleet-secret"), operatorToken: Redacted.make(""), allowApiKeyFallback: false, openAIApiKey: Redacted.make(""), @@ -161,20 +159,6 @@ describe("AI Gateway Effect CLI", () => { assert.notInclude(lines, "provider-a"); })); - it.effect("fails closed before serving when shared client identity is absent", () => - Effect.gen(function*() { - const harness = yield* makeHarness(); - const exitCode = yield* runAIGatewayCli( - ["serve"], - configuration({ clientToken: Redacted.make("") }), - ).pipe(Effect.provide(harness.layer)); - assert.strictEqual(exitCode, 1); - assert.deepStrictEqual(yield* Ref.get(harness.served), []); - assert.deepStrictEqual(yield* Ref.get(harness.errors), [ - "AI_GATEWAY_TOKEN is required to serve", - ]); - })); - it.effect("passes HTTP server settings to the Effect runtime", () => Effect.gen(function*() { const harness = yield* makeHarness(); @@ -191,7 +175,7 @@ describe("AI Gateway Effect CLI", () => { hostname: "127.0.0.2", port: 9876, idleTimeoutSeconds: 120, - authentication: "shared_token", + authentication: "workload_identity", }]); })); @@ -201,8 +185,6 @@ describe("AI Gateway Effect CLI", () => { const exitCode = yield* runAIGatewayCli( ["serve"], configuration({ - clientAuthenticationMode: "workload_identity", - clientToken: Redacted.make(""), operatorToken: Redacted.make("operator-only"), }), ).pipe(Effect.provide(harness.layer)); @@ -224,7 +206,6 @@ describe("AI Gateway Effect CLI", () => { env: { HOME: "/home/agentos", AI_GATEWAY_PORT: "tcp://10.96.0.42:8787", - AI_GATEWAY_TOKEN: "fleet-secret", }, }), ), @@ -232,23 +213,6 @@ describe("AI Gateway Effect CLI", () => { assert.strictEqual(config.port, 8787); })); - it.effect("uses the client token for status without exposing it in output", () => - Effect.gen(function*() { - const harness = yield* makeHarness(); - assert.strictEqual( - yield* runAIGatewayCli(["status"], configuration()).pipe( - Effect.provide(harness.layer), - ), - 0, - ); - assert.deepStrictEqual(yield* Ref.get(harness.statusTokens), [ - "fleet-secret", - ]); - assert.deepStrictEqual(yield* Ref.get(harness.output), [ - '{"accounts":[],"apiKeyFallback":false}', - ]); - })); - it.effect("prefers the dedicated operator credential for status", () => Effect.gen(function*() { const harness = yield* makeHarness(); diff --git a/services/ai-gateway/tests/config.effect.test.ts b/services/ai-gateway/tests/config.effect.test.ts index 89149134..9d515036 100644 --- a/services/ai-gateway/tests/config.effect.test.ts +++ b/services/ai-gateway/tests/config.effect.test.ts @@ -16,7 +16,6 @@ describe("AI Gateway Effect configuration", () => { const config = yield* loadAIGatewayConfig().pipe( Effect.provide(environment({ HOME: "/home/agentos", - AI_GATEWAY_TOKEN: "fleet-secret", OPENAI_API_KEY: "provider-secret", })), ); @@ -28,9 +27,7 @@ describe("AI Gateway Effect configuration", () => { assert.strictEqual(config.hostname, "0.0.0.0"); assert.strictEqual(config.port, 8787); assert.strictEqual(config.idleTimeoutSeconds, 255); - assert.strictEqual(String(config.clientToken), ""); assert.strictEqual(String(config.openAIApiKey), ""); - assert.strictEqual(Redacted.value(config.clientToken), "fleet-secret"); })); it.effect("accepts workload identity without a shared client token", () => @@ -38,7 +35,6 @@ describe("AI Gateway Effect configuration", () => { const config = yield* loadAIGatewayConfig().pipe( Effect.provide(environment({ HOME: "/home/agentos", - AI_GATEWAY_CLIENT_AUTH_MODE: "workload_identity", AI_GATEWAY_OPERATOR_TOKEN: "operator-only", })), ); @@ -72,15 +68,8 @@ describe("AI Gateway Effect configuration", () => { assert.strictEqual(disabled.idleTimeoutSeconds, 0); })); - it.effect("rejects missing shared auth and malformed runtime bounds", () => + it.effect("rejects malformed runtime bounds", () => Effect.gen(function*() { - const missingToken = yield* loadAIGatewayConfig().pipe( - Effect.provide(environment({ HOME: "/home/agentos" })), - Effect.flatMap(requireAIGatewayServeConfig), - Effect.flip, - ); - assert.strictEqual(missingToken.code, "client_identity_unavailable"); - const invalidEnvironments: ReadonlyArray>> = [ { AI_GATEWAY_LISTEN_PORT: "0" }, { AI_GATEWAY_LISTEN_PORT: "65536" }, diff --git a/services/ai-gateway/tests/forward.effect.test.ts b/services/ai-gateway/tests/forward.effect.test.ts index 44dbe362..b9e08f85 100644 --- a/services/ai-gateway/tests/forward.effect.test.ts +++ b/services/ai-gateway/tests/forward.effect.test.ts @@ -2,6 +2,7 @@ import { layer as BunCryptoLayer } from "@effect/platform-bun/BunCrypto"; import { assert, describe, layer } from "@effect/vitest"; import { ProviderBudgetSettlementReporter, + ProviderBudgetAttemptRenewalReporter, ProviderBudgetSettlementHttpError, providerAuthorizationGrantHeaders, type ProviderAuthorizationGrantV1, @@ -73,6 +74,8 @@ function workloadGrant(): Extract< ProviderAuthorizationGrantV1, { readonly model: string } > { + const body = JSON.stringify({ model: "gpt-policy", stream: true, max_output_tokens: 100 }); + const inputTokens = new TextEncoder().encode(body).byteLength; return { schemaVersion: 1, correlationId: "corr_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", @@ -104,6 +107,9 @@ function workloadGrant(): Extract< spendWindowMillis: 3_600_000, maximumSpendMicros: 2_000_000, }, + pricing: { version: 1, inputMicrosPerMillionTokens: 2_000_000, outputMicrosPerMillionTokens: 8_000_000 }, + requestedTokens: inputTokens + 100, + requestedSpendMicros: Math.ceil((inputTokens * 2_000_000 + 100 * 8_000_000) / 1_000_000), }; } @@ -119,7 +125,7 @@ function gatewayRequest( return new Request("http://ai-gateway.test/v1/responses", { method: "POST", headers, - body: JSON.stringify({ model: "gpt-test", stream: true }), + body: JSON.stringify({ model: "gpt-test", stream: true, max_output_tokens: 100 }), }); } @@ -328,9 +334,8 @@ describe("Effect AI Gateway forwarding", () => { ]); })); - it.effect("traces a successful non-streaming response through lease release", () => + it.effect("rejects a non-streaming workload before credential or provider access", () => Effect.gen(function*() { - const route = yield* makeLease(); const settlement = yield* makeSettlementRecorder(); const spans: Array = []; const tracer = Tracer.make({ @@ -343,44 +348,26 @@ describe("Effect AI Gateway forwarding", () => { const telemetry = yield* makeAIGatewayTelemetry(); const handler = yield* makeAIForwardHandler({ authentication: { kind: "workload_identity" }, - acquire: acquireLease(route.lease), + acquire: () => Effect.die("non-streaming request must not acquire credentials"), provider: AIProviderHttp.of({ - execute: () => Effect.succeed({ - status: 200, - headers: { "content-type": "application/json" }, - body: Stream.make(encoder.encode('{"id":"response-safe"}')), - }), + execute: () => Effect.die("non-streaming request must not reach provider"), }), settlements: settlement.settlements, now: Effect.succeed(now), heartbeatMillis: 40_000, maximumUsageEventBytes: 4_096, }).pipe(Effect.provideService(AIGatewayTelemetry, telemetry)); - const base = gatewayRequest(); + const base = gatewayRequest(workloadGrant()); const headers = new Headers(base.headers); headers.set("x-agentos-stream-mode", "non_streaming"); const request = new Request(base.url, { method: "POST", headers, - body: JSON.stringify({ model: "gpt-test", stream: false }), + body: JSON.stringify({ model: "gpt-policy", stream: false, max_output_tokens: 100 }), }); const response = yield* handler(request).pipe(Effect.withTracer(tracer)); - assert.strictEqual( - yield* Effect.tryPromise(() => response.text()), - '{"id":"response-safe"}', - ); - assert.strictEqual(yield* Ref.get(route.releases), 1); - assert.include(spans.map(({ name }) => name), "ai-gateway.route.release"); - assert.strictEqual( - spans.find(({ name }) => name === "ai-gateway.request") - ?.attributes.get("agentos.ai.stream.mode"), - "non_streaming", - ); - assert.strictEqual( - spans.find(({ name }) => name === "ai-gateway.stream") - ?.attributes.get("agentos.ai.stream.outcome"), - "completed", - ); + assert.strictEqual(response.status, 403); + assert.deepStrictEqual(yield* Ref.get(settlement.reports), []); })); it.effect("ends telemetry at authentication rejection without starting a route", () => @@ -442,48 +429,36 @@ describe("Effect AI Gateway forwarding", () => { assert.strictEqual((yield* handler(gatewayRequest(workloadGrant()))).status, 403); })); - it.effect("ends telemetry for an authenticated unsupported route", () => + it.effect("revalidates the exact modern ceiling and settles policy-priced usage", () => Effect.gen(function*() { + const route = yield* makeLease(); const settlement = yield* makeSettlementRecorder(); - const telemetry = yield* makeTelemetryRecorder(); + const authorization = workloadGrant(); + const headers = providerAuthorizationGrantHeaders(authorization); + headers.set("authorization", "Bearer projected-workload-token"); + headers.set("content-type", "application/json"); + const request = new Request("http://ai-gateway.test/v1/responses", { + method: "POST", + headers, + body: JSON.stringify({ model: "gpt-policy", stream: true, max_output_tokens: 100 }), + }); const handler = yield* makeAIForwardHandler({ - authentication: { kind: "shared_token", token: "client-secret" }, - acquire: acquireLease(undefined), - provider: AIProviderHttp.of({ - execute: () => Effect.succeed({ status: 500, headers: {}, body: null }), - }), + authentication: { kind: "workload_identity" }, + acquire: acquireLease(route.lease), + provider: AIProviderHttp.of({ execute: () => Effect.succeed({ + status: 200, + headers: { "content-type": "text/event-stream" }, + body: Stream.make(encoder.encode(completedEvent)), + }) }), settlements: settlement.settlements, now: Effect.succeed(now), heartbeatMillis: 40_000, maximumUsageEventBytes: 4_096, - }).pipe( - Effect.provideService(AIGatewayTelemetry, telemetry.telemetry), - ); - const response = yield* handler(new Request( - "http://ai-gateway.test/v1/unsupported", - { - method: "POST", - headers: { "x-ai-gateway-token": "client-secret" }, - }, - )); - assert.strictEqual(response.status, 404); - assert.deepStrictEqual(yield* Ref.get(telemetry.events), [ - "start", - { - authenticated: true, - authorized: false, - failureStatus: undefined, - kind: "authenticate", - }, - { - kind: "end", - outcome: { - failed: false, - status: 404, - streamOutcome: "not_streamed", - }, - }, - ]); + }); + const response = yield* handler(request); + yield* Effect.tryPromise(() => response.arrayBuffer()); + assert.strictEqual(response.status, 200); + assert.strictEqual((yield* Ref.get(settlement.reports))[0]?.spendMicros, 134); })); it.effect("ends telemetry for an invalid explicit session before acquisition", () => @@ -727,7 +702,7 @@ describe("Effect AI Gateway forwarding", () => { }]); })); - it.effect("fails closed without settlement when a successful stream lacks terminal usage", () => + it.effect("fails a nominally successful stream closed when terminal usage is unavailable", () => Effect.gen(function*() { const route = yield* makeLease(); const settlement = yield* makeSettlementRecorder(); @@ -749,7 +724,8 @@ describe("Effect AI Gateway forwarding", () => { maximumUsageEventBytes: 4_096, }); const response = yield* handler(gatewayRequest()); - yield* Effect.tryPromise(() => response.arrayBuffer()); + const read = yield* Effect.exit(Effect.tryPromise(() => response.arrayBuffer())); + assert.strictEqual(read._tag, "Failure"); assert.strictEqual(yield* Ref.get(route.releases), 1); assert.deepStrictEqual(yield* Ref.get(settlement.reports), []); })); @@ -1000,7 +976,7 @@ describe("Effect AI Gateway forwarding", () => { ); const response = yield* handler(gatewayRequest()); assert.strictEqual(response.status, 502); - assert.deepStrictEqual(yield* Ref.get(settlement.reports), []); + assert.strictEqual((yield* Ref.get(settlement.reports))[0]?.forwardOutcome, "transport_failed"); assert.deepStrictEqual(yield* Ref.get(telemetry.events), [ "start", { @@ -1051,7 +1027,7 @@ describe("Effect AI Gateway forwarding", () => { ); const response = yield* handler(gatewayRequest()); assert.strictEqual(response.status, 502); - assert.deepStrictEqual(yield* Ref.get(settlement.reports), []); + assert.strictEqual((yield* Ref.get(settlement.reports))[0]?.forwardOutcome, "transport_failed"); assert.deepStrictEqual(yield* Ref.get(telemetry.events), [ "start", { @@ -1102,7 +1078,7 @@ describe("Effect AI Gateway forwarding", () => { ); const response = yield* handler(gatewayRequest()); assert.strictEqual(response.status, 502); - assert.deepStrictEqual(yield* Ref.get(settlement.reports), []); + assert.strictEqual((yield* Ref.get(settlement.reports))[0]?.forwardOutcome, "transport_failed"); assert.deepStrictEqual(yield* Ref.get(telemetry.events), [ "start", { @@ -1180,7 +1156,7 @@ describe("Effect AI Gateway forwarding", () => { const response = yield* handler(gatewayRequest()); assert.strictEqual(response.status, 502); assert.strictEqual(yield* Ref.get(route.releases), 1); - assert.deepStrictEqual(yield* Ref.get(settlement.reports), []); + assert.strictEqual((yield* Ref.get(settlement.reports))[0]?.forwardOutcome, "transport_failed"); assert.deepStrictEqual(yield* Ref.get(telemetry.events), [ "start", { @@ -1258,7 +1234,7 @@ describe("Effect AI Gateway forwarding", () => { ]); })); - it.effect("releases an interrupted downstream stream without releasing an unmeasured budget", () => + it.effect("settles an interrupted downstream stream as cancelled", () => Effect.gen(function*() { const route = yield* makeLease(); const settlement = yield* makeSettlementRecorder(); @@ -1287,7 +1263,7 @@ describe("Effect AI Gateway forwarding", () => { yield* Effect.tryPromise(() => reader.cancel("client disconnected")); } assert.strictEqual(yield* Ref.get(route.releases), 1); - assert.deepStrictEqual(yield* Ref.get(settlement.reports), []); + assert.strictEqual((yield* Ref.get(settlement.reports))[0]?.forwardOutcome, "cancelled"); assert.deepStrictEqual(yield* Ref.get(telemetry.events), [ "start", { @@ -1313,7 +1289,7 @@ describe("Effect AI Gateway forwarding", () => { ]); })); - it.effect("keeps a provider stream failure distinct and does not settle unknown usage", () => + it.effect("settles a provider stream failure as transport failed", () => Effect.gen(function*() { const route = yield* makeLease(); const settlement = yield* makeSettlementRecorder(); @@ -1343,7 +1319,7 @@ describe("Effect AI Gateway forwarding", () => { ); assert.notInclude(String(failure), "provider-secret"); assert.strictEqual(yield* Ref.get(route.releases), 1); - assert.deepStrictEqual(yield* Ref.get(settlement.reports), []); + assert.strictEqual((yield* Ref.get(settlement.reports))[0]?.forwardOutcome, "transport_failed"); assert.deepStrictEqual(yield* Ref.get(telemetry.events), [ "start", { @@ -1369,7 +1345,45 @@ describe("Effect AI Gateway forwarding", () => { ]); })); - it.effect("never replaces a completed provider stream with settlement failure", () => + it.effect("settles an unusable claimed credential before provider contact", () => + Effect.gen(function*() { + const route = yield* makeLease(); + const providerCalls = yield* Ref.make(0); + const settlement = yield* makeSettlementRecorder(); + const handler = yield* makeAIForwardHandler({ + authentication: { kind: "workload_identity" }, + acquire: acquireLease({ + ...route.lease, + kind: "codex_oauth", + }), + provider: AIProviderHttp.of({ + execute: () => Ref.update(providerCalls, (count) => count + 1).pipe( + Effect.andThen(Effect.die("provider must not be contacted")), + ), + }), + settlements: settlement.settlements, + now: Effect.succeed(now), + heartbeatMillis: 40_000, + maximumUsageEventBytes: 4_096, + }); + const response = yield* handler(gatewayRequest()); + assert.strictEqual(response.status, 400); + assert.strictEqual(yield* Ref.get(providerCalls), 0); + assert.deepStrictEqual(yield* Ref.get(settlement.reports), [ + { + schemaVersion: 1, + decisionRef, + forwardOutcome: "transport_failed", + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + spendMicros: 0, + }, + ]); + assert.strictEqual(yield* Ref.get(route.releases), 1); + })); + + it.effect("fails the client stream closed when settlement fails after every retry", () => Effect.gen(function*() { const route = yield* makeLease(); const settlementAttempts = yield* Ref.make(0); @@ -1399,11 +1413,9 @@ describe("Effect AI Gateway forwarding", () => { maximumUsageEventBytes: 4_096, }); const response = yield* handler(gatewayRequest()); - assert.strictEqual( - yield* Effect.tryPromise(() => response.text()), - completedEvent, - ); - assert.strictEqual(yield* Ref.get(settlementAttempts), 1); + const read = yield* Effect.exit(Effect.tryPromise(() => response.text())); + assert.strictEqual(read._tag, "Failure"); + assert.strictEqual(yield* Ref.get(settlementAttempts), 3); assert.strictEqual(yield* Ref.get(route.releases), 1); })); @@ -1442,6 +1454,7 @@ describe("Effect AI Gateway forwarding", () => { Effect.gen(function*() { const route = yield* makeLease(); const settlement = yield* makeSettlementRecorder(); + const attemptRenewals = yield* Ref.make(0); const handler = yield* makeAIForwardHandler({ authentication: { kind: "workload_identity" }, acquire: acquireLease(route.lease), @@ -1453,6 +1466,11 @@ describe("Effect AI Gateway forwarding", () => { }), }), settlements: settlement.settlements, + attemptRenewals: ProviderBudgetAttemptRenewalReporter.of({ + renew: (request) => Ref.updateAndGet(attemptRenewals, (count) => count + 1).pipe( + Effect.map(() => ({ schemaVersion: 1, decisionRef: request.decisionRef, outcome: "renewed" })), + ), + }), now: Effect.succeed(now), heartbeatMillis: 40_000, maximumUsageEventBytes: 4_096, @@ -1467,12 +1485,142 @@ describe("Effect AI Gateway forwarding", () => { yield* TestClock.adjust(40_000); yield* Effect.yieldNow; assert.strictEqual(yield* Ref.get(route.renewals), 1); + assert.strictEqual(yield* Ref.get(attemptRenewals), 1); yield* Effect.tryPromise(() => reader.cancel("test complete")); yield* Fiber.interrupt(pending); yield* TestClock.adjust(80_000); assert.strictEqual(yield* Ref.get(route.renewals), 1); + assert.strictEqual(yield* Ref.get(attemptRenewals), 1); } assert.strictEqual(yield* Ref.get(route.releases), 1); })); + + it.effect("fails the active provider stream when routing lease renewal is refused", () => + Effect.gen(function*() { + const route = yield* makeLease(); + const settlement = yield* makeSettlementRecorder(); + const terminalEmissions = yield* Ref.make(0); + const lease: AIForwardLease = { + ...route.lease, + renew: Ref.update(route.renewals, (count) => count + 1).pipe( + Effect.as(false), + ), + }; + const handler = yield* makeAIForwardHandler({ + authentication: { kind: "workload_identity" }, + acquire: acquireLease(lease), + provider: AIProviderHttp.of({ + execute: () => Effect.succeed({ + status: 200, + headers: { "content-type": "text/event-stream" }, + body: Stream.never.pipe(Stream.concat( + Ref.update(terminalEmissions, (count) => count + 1).pipe( + Effect.as(encoder.encode(completedEvent)), + Stream.fromEffect, + ), + )), + }), + }), + settlements: settlement.settlements, + attemptRenewals: ProviderBudgetAttemptRenewalReporter.of({ + renew: (request) => Effect.succeed({ + schemaVersion: 1, + decisionRef: request.decisionRef, + outcome: "renewed", + }), + }), + now: Effect.succeed(now), + heartbeatMillis: 40_000, + maximumUsageEventBytes: 4_096, + }); + const response = yield* handler(gatewayRequest()); + const reader = response.body?.getReader(); + assert.isDefined(reader); + if (reader === undefined) return; + const pending = yield* Effect.forkChild(Effect.tryPromise(() => reader.read())); + yield* TestClock.adjust(40_000); + const readExit = yield* Fiber.await(pending); + assert.strictEqual(readExit._tag, "Success"); + if (Exit.isSuccess(readExit)) assert.isTrue(readExit.value.done); + assert.strictEqual(yield* Ref.get(route.renewals), 1); + assert.strictEqual(yield* Ref.get(terminalEmissions), 0); + assert.deepStrictEqual(yield* Ref.get(settlement.reports), [{ + schemaVersion: 1, + decisionRef, + forwardOutcome: "transport_failed", + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + spendMicros: 0, + }]); + assert.strictEqual(yield* Ref.get(route.releases), 1); + yield* TestClock.adjust(120_000); + assert.strictEqual(yield* Ref.get(route.renewals), 1); + assert.strictEqual((yield* Ref.get(settlement.reports)).length, 1); + assert.strictEqual(yield* Ref.get(route.releases), 1); + })); + + it.effect("fails the active provider stream when durable attempt renewal is unavailable", () => + Effect.gen(function*() { + const route = yield* makeLease(); + const settlement = yield* makeSettlementRecorder(); + const attemptRenewals = yield* Ref.make(0); + const terminalEmissions = yield* Ref.make(0); + const handler = yield* makeAIForwardHandler({ + authentication: { kind: "workload_identity" }, + acquire: acquireLease(route.lease), + provider: AIProviderHttp.of({ + execute: () => Effect.succeed({ + status: 200, + headers: { "content-type": "text/event-stream" }, + body: Stream.never.pipe(Stream.concat( + Ref.update(terminalEmissions, (count) => count + 1).pipe( + Effect.as(encoder.encode(completedEvent)), + Stream.fromEffect, + ), + )), + }), + }), + settlements: settlement.settlements, + attemptRenewals: ProviderBudgetAttemptRenewalReporter.of({ + renew: () => Ref.update(attemptRenewals, (count) => count + 1).pipe( + Effect.andThen(Effect.fail(ProviderBudgetSettlementHttpError.make({ + code: "dependency_unavailable", + status: 503, + }))), + ), + }), + now: Effect.succeed(now), + heartbeatMillis: 40_000, + maximumUsageEventBytes: 4_096, + }); + const response = yield* handler(gatewayRequest()); + const reader = response.body?.getReader(); + assert.isDefined(reader); + if (reader === undefined) return; + const pending = yield* Effect.forkChild(Effect.tryPromise(() => reader.read())); + yield* TestClock.adjust(40_000); + const readExit = yield* Fiber.await(pending); + assert.strictEqual(readExit._tag, "Success"); + if (Exit.isSuccess(readExit)) assert.isTrue(readExit.value.done); + assert.strictEqual(yield* Ref.get(route.renewals), 1); + assert.strictEqual(yield* Ref.get(attemptRenewals), 1); + assert.strictEqual(yield* Ref.get(terminalEmissions), 0); + assert.deepStrictEqual(yield* Ref.get(settlement.reports), [{ + schemaVersion: 1, + decisionRef, + forwardOutcome: "transport_failed", + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + spendMicros: 0, + }]); + assert.strictEqual(yield* Ref.get(route.releases), 1); + yield* TestClock.adjust(120_000); + assert.strictEqual(yield* Ref.get(route.renewals), 1); + assert.strictEqual(yield* Ref.get(attemptRenewals), 1); + assert.strictEqual((yield* Ref.get(settlement.reports)).length, 1); + assert.strictEqual(yield* Ref.get(route.releases), 1); + })); }); }); diff --git a/services/ai-gateway/tests/gateway-service.effect.test.ts b/services/ai-gateway/tests/gateway-service.effect.test.ts index d322b0e8..05f37582 100644 --- a/services/ai-gateway/tests/gateway-service.effect.test.ts +++ b/services/ai-gateway/tests/gateway-service.effect.test.ts @@ -78,6 +78,8 @@ function workloadGrant(): Extract< ProviderAuthorizationGrantV1, { readonly model: string } > { + const body = JSON.stringify({ model: "gpt-test", max_output_tokens: 100, stream: true }); + const inputTokens = new TextEncoder().encode(body).byteLength; return { schemaVersion: 1, correlationId: "corr_55555555555555555555555555555555", @@ -109,6 +111,9 @@ function workloadGrant(): Extract< spendWindowMillis: 3_600_000, maximumSpendMicros: 1_000, }, + pricing: { version: 1, inputMicrosPerMillionTokens: 2_000_000, outputMicrosPerMillionTokens: 8_000_000 }, + requestedTokens: inputTokens + 100, + requestedSpendMicros: Math.ceil((inputTokens * 2_000_000 + 100 * 8_000_000) / 1_000_000), }; } @@ -122,7 +127,7 @@ function providerRequest( return new Request("http://ai-gateway.test/v1/responses", { method: "POST", headers, - body: JSON.stringify({ model: "gpt-test", stream: true }), + body: JSON.stringify({ model: "gpt-test", max_output_tokens: 100, stream: true }), }); } diff --git a/services/ai-gateway/tests/kubernetes.effect.test.ts b/services/ai-gateway/tests/kubernetes.effect.test.ts index 114bfa65..e6c08c14 100644 --- a/services/ai-gateway/tests/kubernetes.effect.test.ts +++ b/services/ai-gateway/tests/kubernetes.effect.test.ts @@ -251,10 +251,6 @@ describe("optional Fleet AI Gateway", () => { environment.AI_GATEWAY_STATE_DIR, "/var/lib/ai-gateway", ); - assert.strictEqual( - environment.AI_GATEWAY_CLIENT_AUTH_MODE, - "workload_identity", - ); assert.deepStrictEqual(environment.AI_GATEWAY_OPERATOR_TOKEN, { secretKeyRef: { key: "token", name: "ai-gateway-operator" }, }); diff --git a/services/egress-authz/kubernetes/deployment.yaml b/services/egress-authz/kubernetes/deployment.yaml index 5b75310c..d6626da1 100644 --- a/services/egress-authz/kubernetes/deployment.yaml +++ b/services/egress-authz/kubernetes/deployment.yaml @@ -129,15 +129,22 @@ spec: requests: cpu: 100m memory: 128Mi + ephemeral-storage: 64Mi limits: cpu: "1" memory: 512Mi + ephemeral-storage: 256Mi securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 capabilities: drop: - ALL + seccompProfile: + type: RuntimeDefault volumeMounts: - name: database mountPath: /var/run/secrets/agentos-database diff --git a/services/egress-authz/kubernetes/rbac.yaml b/services/egress-authz/kubernetes/rbac.yaml index 90310b56..2901d043 100644 --- a/services/egress-authz/kubernetes/rbac.yaml +++ b/services/egress-authz/kubernetes/rbac.yaml @@ -20,6 +20,14 @@ rules: - serviceaccounts verbs: - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - agentos-hermes-provider-access-v1 + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/services/egress-authz/src/app.ts b/services/egress-authz/src/app.ts index 93e9c709..23e5c5c8 100644 --- a/services/egress-authz/src/app.ts +++ b/services/egress-authz/src/app.ts @@ -4,6 +4,9 @@ import { ProviderBudgetSettlementCallerAuthenticator, ProviderBudgetSettlementCallerAuthenticationError, ProviderBudgetSettlementReportV1Schema, + ProviderBudgetAttemptRenewalInputV1Schema, + ProviderBudgetReservationAcceptanceV1Schema, + ProviderBudgetReservationRequestV1Schema, ProviderDecisionReferenceGenerator, HermesProviderAuthorizer, type ProviderAccessTelemetry, @@ -92,6 +95,7 @@ export const makeEgressAuthorizerRequestHandler = Effect.fn( clock, id: decisionReferences.next, hermes, + budgets: providerBudgets, ...(options.telemetry === undefined ? {} : { telemetry: options.telemetry }), @@ -124,11 +128,75 @@ export const makeEgressAuthorizerRequestHandler = Effect.fn( request, limits.maximumSettlementBodyBytes, ); + const settledAtMillis = yield* clock; const result = yield* providerBudgets.settleProvider({ ...report, provider: caller.provider, credentialDomain: caller.credentialDomain, - settledAtMillis: yield* clock, + settledAtMillis, + }); + return Response.json({ + schemaVersion: 1, + decisionRef: result.decisionRef, + outcome: result.outcome, + }); + }, + ); + + const validate = Effect.fn("agentos.egressAuthz.validateProviderBudget")( + function*(request: Request) { + const bearerToken = settlementBearerToken(request.headers); + if (bearerToken === null) return unauthorizedResponse(); + const caller = yield* settlementCallers.authenticate(bearerToken); + if (caller.provider !== "openai" || caller.credentialDomain !== "openai-responses") { + return forbiddenResponse(); + } + const body = yield* readReservationValidationRequest( + request, + limits.maximumSettlementBodyBytes, + ); + yield* providerBudgets.validateWorkload({ + schemaVersion: 1, + decisionRef: body.grant.decisionRef, + correlationId: body.grant.correlationId, + principal: body.subject, + provider: "openai", + credentialDomain: "openai-responses", + capability: body.grant.capability === "openai.responses.create" + ? "openai.responses.create" + : "openai.responses.compact", + resource: body.grant.resource, + model: body.grant.model, + rateClass: body.grant.rateClass, + limits: body.grant.limits, + pricing: body.grant.pricing, + requestedTokens: body.grant.requestedTokens, + requestedSpendMicros: body.grant.requestedSpendMicros, + expiresAtMillis: body.grant.expiresAtMillis, + nowMillis: yield* clock, + }); + const acceptance = yield* Schema.decodeUnknownEffect( + ProviderBudgetReservationAcceptanceV1Schema, + { onExcessProperty: "error" }, + )({ ...body, outcome: "reserved" }); + return Response.json(acceptance); + }, + ); + + const renew = Effect.fn("agentos.egressAuthz.renewProviderBudgetAttempt")( + function*(request: Request) { + const bearerToken = settlementBearerToken(request.headers); + if (bearerToken === null) return unauthorizedResponse(); + const caller = yield* settlementCallers.authenticate(bearerToken); + const body = yield* readAttemptRenewalRequest( + request, + limits.maximumSettlementBodyBytes, + ); + const result = yield* providerBudgets.renewProviderAttempt({ + ...body, + provider: caller.provider, + credentialDomain: caller.credentialDomain, + renewedAtMillis: yield* clock, }); return Response.json({ schemaVersion: 1, @@ -201,20 +269,64 @@ export const makeEgressAuthorizerRequestHandler = Effect.fn( } return yield* handleSettlementReadiness(request); } - if (url.pathname === "/authorize" || url.pathname === "/settle") { + if ( + url.pathname === "/authorize" || url.pathname === "/settle" || + url.pathname === "/validate" || url.pathname === "/renew" + ) { if (request.method !== "POST") return methodNotAllowedResponse(); if (!headersWithinLimits(request.headers, limits)) { return invalidRequestResponse(); } - return url.pathname === "/authorize" - ? yield* handleAuthorization(request) - : yield* handleSettlement(request); + if (url.pathname === "/authorize") return yield* handleAuthorization(request); + if (url.pathname === "/settle") return yield* handleSettlement(request); + if (url.pathname === "/renew") { + return yield* permits.withPermitsIfAvailable(1)( + renew(request).pipe( + Effect.catch((error) => Effect.succeed(responseForSettlementFailure(error))), + Effect.timeoutOption(limits.requestTimeoutMillis), + ), + ).pipe(Effect.map((result) => Option.isNone(result) + ? overloadedResponse() + : Option.match(result.value, { + onNone: unavailableResponse, + onSome: (response) => response, + }))); + } + return yield* permits.withPermitsIfAvailable(1)( + validate(request).pipe( + Effect.catch((error) => + Effect.succeed(responseForSettlementFailure(error)) + ), + Effect.timeoutOption(limits.requestTimeoutMillis), + ), + ).pipe(Effect.map((result) => Option.isNone(result) + ? overloadedResponse() + : Option.match(result.value, { + onNone: unavailableResponse, + onSome: (response) => response, + }))); } return notFoundResponse(); }); return handler; }); +const readReservationValidationRequest = Effect.fn( + "agentos.egressAuthz.readReservationValidationRequest", +)(function*(request: Request, maximumBytes: number) { + const text = yield* Effect.tryPromise({ + try: () => request.text(), + catch: () => InvalidSettlementRequest.make(), + }); + if (new TextEncoder().encode(text).byteLength > maximumBytes) { + return yield* InvalidSettlementRequest.make(); + } + return yield* Schema.decodeUnknownEffect( + Schema.fromJsonString(ProviderBudgetReservationRequestV1Schema), + { onExcessProperty: "error" }, + )(text).pipe(Effect.mapError(() => InvalidSettlementRequest.make())); +}); + export function makeEgressAuthorizerRoutesLayer( options: EgressAuthorizerLimits, ) { @@ -326,6 +438,40 @@ const readSettlementReport = Effect.fn( ); }); +const readAttemptRenewalRequest = Effect.fn( + "agentos.egressAuthz.readAttemptRenewalRequest", +)(function*(request: Request, maximumBytes: number) { + const report = yield* readSettlementReportSource(request, maximumBytes); + return yield* Schema.decodeUnknownEffect( + Schema.Struct({ + schemaVersion: Schema.Literal(1), + decisionRef: ProviderBudgetAttemptRenewalInputV1Schema.fields.decisionRef, + }), + { onExcessProperty: "error" }, + )(report).pipe(Effect.mapError(() => InvalidSettlementRequest.make())); +}); + +const readSettlementReportSource = Effect.fn( + "agentos.egressAuthz.readSettlementReportSource", +)(function*(request: Request, maximumBytes: number) { + const contentType = request.headers.get("content-type")?.toLowerCase() ?? ""; + const declaredLength = request.headers.get("content-length"); + if (!/^application\/json(?:\s*;.*)?$/.test(contentType) || declaredLength === null || + !/^(?:0|[1-9][0-9]*)$/.test(declaredLength) || Number(declaredLength) > maximumBytes) { + return yield* InvalidSettlementRequest.make(); + } + const source = yield* Effect.tryPromise({ + try: () => request.text(), + catch: () => InvalidSettlementRequest.make(), + }); + if (source.length === 0 || new TextEncoder().encode(source).byteLength > maximumBytes) { + return yield* InvalidSettlementRequest.make(); + } + return yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))(source).pipe( + Effect.mapError(() => InvalidSettlementRequest.make()), + ); +}); + function settlementBearerToken(headers: Headers): string | null { const authorization = headers.get("authorization")?.trim(); const token = authorization === undefined diff --git a/services/egress-authz/src/layers.ts b/services/egress-authz/src/layers.ts index bbae811c..e70fba26 100644 --- a/services/egress-authz/src/layers.ts +++ b/services/egress-authz/src/layers.ts @@ -72,13 +72,18 @@ export function makeEgressAuthorizerLiveLayer( SELECT has_function_privilege( current_user, - 'agentos.reserve_provider_budget(text,text,text,jsonb,text,text,text,jsonb,text,text,text,bigint)', + 'agentos.reserve_workload_provider_budget(text,text,text,jsonb,text,text,text,jsonb,text,text,text,jsonb,jsonb,bigint,bigint,bigint,bigint)', 'EXECUTE' ) AND has_function_privilege( current_user, 'agentos.settle_provider_budget_for_provider(text,text,text,text,bigint,bigint,bigint,bigint,bigint)', 'EXECUTE' + ) + AND has_function_privilege( + current_user, + 'agentos.renew_workload_provider_attempt(text,text,text,bigint)', + 'EXECUTE' ) AS ready ` ), diff --git a/services/egress-authz/tests/app.effect.test.ts b/services/egress-authz/tests/app.effect.test.ts index 68adc655..2fddfa12 100644 --- a/services/egress-authz/tests/app.effect.test.ts +++ b/services/egress-authz/tests/app.effect.test.ts @@ -70,6 +70,13 @@ const hermesAuthorized: HermesProviderAuthorizationResult = { kind: "authorized", tokenExpiresAtMillis: now + 10_000, policyExpiresAtMillis: null, + requestedTokens: 1_024, + requestedSpendMicros: 4_096, + workloadIdentity: { + serviceAccountUid: "service-account-uid-1", + podName: "worker-0", + podUid: "pod-uid-1", + }, grant: { decision: "allow", principal: { @@ -94,6 +101,11 @@ const hermesAuthorized: HermesProviderAuthorizationResult = { spendWindowMillis: 60_000, maximumSpendMicros: 1_000_000, }, + pricing: { + version: 1, + inputMicrosPerMillionTokens: 2_000_000, + outputMicrosPerMillionTokens: 8_000_000, + }, }, }; @@ -146,12 +158,30 @@ function settlementRequest( }); } +function renewalRequest(body: unknown = { + schemaVersion: 1, + decisionRef: settlementBody.decisionRef, +}) { + const encoded = JSON.stringify(body); + return new Request("http://egress-authz.test/renew", { + method: "POST", + headers: { + authorization: "Bearer projected-provider-jwt", + "content-type": "application/json", + "content-length": String(new TextEncoder().encode(encoded).byteLength), + }, + body: encoded, + }); +} + function services(options?: { readonly authenticate?: WorkloadIdentityAuthenticator["Service"]["authenticate"]; readonly authorizeHermes?: HermesProviderAuthorization["authorize"]; readonly decide?: ProviderPolicyDecisionPoint["Service"]["decide"]; readonly authenticateSettlement?: ProviderBudgetSettlementCallerAuthenticator["Service"]["authenticate"]; readonly settleProvider?: ProviderBudgetEnforcer["Service"]["settleProvider"]; + readonly renewProviderAttempt?: ProviderBudgetEnforcer["Service"]["renewProviderAttempt"]; + readonly reserveWorkload?: ProviderBudgetEnforcer["Service"]["reserveWorkload"]; readonly ready?: Effect.Effect; }) { return Layer.mergeAll( @@ -176,6 +206,18 @@ function services(options?: { (() => Effect.succeed(settlementCaller)), }), Layer.succeed(ProviderBudgetEnforcer, { + validateWorkload: () => Effect.void, + reserveWorkload: options?.reserveWorkload ?? ((input) => Effect.succeed({ + schemaVersion: 1, + decisionRef: input.decisionRef, + budgetKey: `budget_${"6".repeat(64)}`, + outcome: "reserved", + effectiveRateClass: input.rateClass, + requestWindowEndsAtMillis: input.nowMillis + 60_000, + tokenWindowEndsAtMillis: input.nowMillis + 60_000, + spendWindowEndsAtMillis: input.nowMillis + 60_000, + leaseExpiresAtMillis: input.policyExpiresAtMillis, + })), reserve: () => Effect.die("reserve is owned by the policy decision point"), settle: () => Effect.die("subject settlement is not an HTTP boundary"), settleProvider: options?.settleProvider ?? ((input) => Effect.succeed({ @@ -189,6 +231,12 @@ function services(options?: { spendMicros: input.spendMicros, settledAtMillis: input.settledAtMillis, })), + renewProviderAttempt: options?.renewProviderAttempt ?? ((input) => Effect.succeed({ + schemaVersion: 1, + decisionRef: input.decisionRef, + outcome: "renewed", + leaseExpiresAtMillis: input.renewedAtMillis + 60_000, + })), }), Layer.succeed(EgressAuthorizerReadiness, { check: options?.ready ?? Effect.succeed(true), @@ -491,6 +539,38 @@ describe("Effect egress authorization HTTP application", () => { }); })); + it.effect("authenticates attempt renewal and derives provider authority outside the closed body", () => + Effect.gen(function*() { + const seen = yield* Ref.make(null); + const handler = yield* makeHandler.pipe(Effect.provide(services({ + renewProviderAttempt: (input) => Ref.set(seen, input).pipe(Effect.as({ + schemaVersion: 1, + decisionRef: input.decisionRef, + outcome: "renewed", + leaseExpiresAtMillis: input.renewedAtMillis + 60_000, + })), + }))); + const response = yield* handler(renewalRequest()); + assert.strictEqual(response.status, 200); + assert.deepStrictEqual(yield* Effect.tryPromise(() => response.json()), { + schemaVersion: 1, + decisionRef: settlementBody.decisionRef, + outcome: "renewed", + }); + assert.deepStrictEqual(yield* Ref.get(seen), { + schemaVersion: 1, + decisionRef: settlementBody.decisionRef, + provider: "github", + credentialDomain: "github", + renewedAtMillis: now, + }); + assert.strictEqual((yield* handler(renewalRequest({ + schemaVersion: 1, + decisionRef: settlementBody.decisionRef, + provider: "openai", + }))).status, 400); + })); + it.effect("rejects malformed or oversized settlement bodies before database work", () => Effect.gen(function*() { const calls = yield* Ref.make(0); diff --git a/services/egress-authz/tests/kubernetes.effect.test.ts b/services/egress-authz/tests/kubernetes.effect.test.ts index 1c302c60..7c076ee7 100644 --- a/services/egress-authz/tests/kubernetes.effect.test.ts +++ b/services/egress-authz/tests/kubernetes.effect.test.ts @@ -108,6 +108,21 @@ describe("egress authorizer Kubernetes boundary", () => { assert.include(rendered, '"verbs":["create"]'); assert.include(rendered, '"resources":["pods","serviceaccounts"]'); assert.include(rendered, '"verbs":["get"]'); + assert.include(rendered, '"resources":["configmaps"]'); + assert.include( + rendered, + '"resourceNames":["agentos-hermes-provider-access-v1"]', + ); + for (const forbidden of [ + '"resources":["secrets"]', + '"resources":["pods"],"verbs":["list"', + '"verbs":["*"', + '"resources":["*"', + '"apiGroups":["*"', + '"verbs":["update"', + '"verbs":["patch"', + '"verbs":["delete"', + ]) assert.notInclude(rendered, forbidden); assert.include(rendered, '"policyTypes":["Ingress"]'); assert.include( rendered, @@ -117,5 +132,9 @@ describe("egress authorizer Kubernetes boundary", () => { assert.notInclude(rendered, '"egress":'); assert.include(rendered, '"port":9001'); assert.include(rendered, '"targetPort":"http"'); + assert.notInclude(rendered, '"kind":"Ingress"'); + assert.notInclude(rendered, '"kind":"PersistentVolumeClaim"'); + assert.notInclude(rendered, '"type":"LoadBalancer"'); + assert.notInclude(rendered, '"type":"NodePort"'); }).pipe(Effect.provide(platform))); }); diff --git a/services/github-broker/tests/kubernetes.effect.test.ts b/services/github-broker/tests/kubernetes.effect.test.ts index 2c64a74f..9a6b026b 100644 --- a/services/github-broker/tests/kubernetes.effect.test.ts +++ b/services/github-broker/tests/kubernetes.effect.test.ts @@ -104,7 +104,7 @@ describe("GitHub broker Kubernetes boundary", () => { assert.notInclude(rendered, "ClusterRoleBinding"); }).pipe(Effect.provide(platform))); - it.effect("gives every Mate type only workload identity and the public CA", () => + it.effect("keeps legacy GitHub client wiring out of default Agent workloads", () => Effect.gen(function*() { for (const directory of [ "packages/agentos/resources/roles/firstmate/kubernetes/base", @@ -113,13 +113,32 @@ describe("GitHub broker Kubernetes boundary", () => { ]) { const resources = yield* render(directory); const rendered = JSON.stringify(resources); - assert.include(rendered, '"AGENTOS_GITHUB_PROVIDER_MODE"'); - assert.include(rendered, '"prepare-github-provider"'); - assert.include(rendered, '"agentos-egress-identity"'); - assert.include(rendered, '"configMap":{"defaultMode":292'); + assert.notInclude(rendered, '"agentos.akua.dev/github-client"'); + assert.notInclude(rendered, '"AGENTOS_GITHUB_'); + assert.notInclude(rendered, '"prepare-github-provider"'); + assert.notInclude(rendered, '"agentos-github-ca"'); + assert.notInclude(rendered, '"AGENTOS_EGRESS_TOKEN_FILE"'); + assert.notInclude(rendered, '"agentos-egress-identity"'); assert.notInclude(rendered, '"secretName":"agentos-github-app"'); assert.notInclude(rendered, '"GITHUB_APP_PRIVATE_KEY_FILE"'); assert.notInclude(rendered, '"GITHUB_APP_ID"'); } }).pipe(Effect.provide(platform))); + + it.effect("keeps GitHub credential material out of the broker client boundary", () => + Effect.gen(function*() { + const resources = yield* render( + "packages/agentos/resources/crewmates/default/kubernetes/base", + ); + const rendered = JSON.stringify(resources); + assert.notInclude(rendered, '"agentos.akua.dev/github-client"'); + assert.notInclude(rendered, '"prepare-github-provider"'); + assert.notInclude(rendered, '"agentos-github-ca"'); + assert.notInclude(rendered, '"AGENTOS_GITHUB_'); + assert.notInclude(rendered, '"AGENTOS_EGRESS_TOKEN_FILE"'); + assert.notInclude(rendered, '"agentos-egress-identity"'); + assert.notInclude(rendered, '"secretName":"agentos-github-app"'); + assert.notInclude(rendered, '"GITHUB_APP_PRIVATE_KEY_FILE"'); + assert.notInclude(rendered, '"GITHUB_APP_ID"'); + }).pipe(Effect.provide(platform))); });