diff --git a/.gitignore b/.gitignore index 28682a5..5d0ba31 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,15 @@ build/ !.env.*.example /.local/ +# Private key and credential exports outside the local state directory +*.pem +*.key +*.p12 +*.pfx +*.jks +*.keystore +*.kdbx + # Test, coverage, and tool caches .pytest_cache/ .mypy_cache/ diff --git a/deploy/runtime/Dockerfile b/deploy/runtime/Dockerfile index 673080d..a83267d 100644 --- a/deploy/runtime/Dockerfile +++ b/deploy/runtime/Dockerfile @@ -12,10 +12,12 @@ COPY chipload/ /usr/local/lib/python3.12/site-packages/chipload/ COPY LICENSE /usr/local/share/licenses/chipload/LICENSE ARG SDK_SHA256 ARG DEPENDENCIES_SHA256 +ARG RECIPE_SHA256 LABEL org.chipload.runtime-role="api" \ org.chipload.runtime-protocol="1" \ org.chipload.sdk-sha256=$SDK_SHA256 \ - org.chipload.dependencies-sha256=$DEPENDENCIES_SHA256 + org.chipload.dependencies-sha256=$DEPENDENCIES_SHA256 \ + org.chipload.recipe-sha256=$RECIPE_SHA256 USER 65532:65532 ENTRYPOINT ["python", "-I", "-B", "-m", "chipload.server"] CMD ["--config", "/etc/chipload/settings.json", "--bind", "0.0.0.0", "--port", "8088"] @@ -40,10 +42,12 @@ COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker COPY licenses/ /usr/local/share/licenses/docker-cli/ ARG SDK_SHA256 ARG DEPENDENCIES_SHA256 +ARG RECIPE_SHA256 LABEL org.chipload.runtime-role="broker" \ org.chipload.runtime-protocol="1" \ org.chipload.sdk-sha256=$SDK_SHA256 \ - org.chipload.dependencies-sha256=$DEPENDENCIES_SHA256 + org.chipload.dependencies-sha256=$DEPENDENCIES_SHA256 \ + org.chipload.recipe-sha256=$RECIPE_SHA256 USER 65532:65532 ENTRYPOINT ["python", "-I", "-B", "-m", "chipload.broker"] @@ -55,9 +59,11 @@ COPY chipload/ /usr/local/lib/python3.12/site-packages/chipload/ COPY LICENSE /usr/local/share/licenses/chipload/LICENSE ARG SDK_SHA256 ARG DEPENDENCIES_SHA256 +ARG RECIPE_SHA256 LABEL org.chipload.runtime-role="executor" \ org.chipload.runtime-protocol="1" \ org.chipload.sdk-sha256=$SDK_SHA256 \ - org.chipload.dependencies-sha256=$DEPENDENCIES_SHA256 + org.chipload.dependencies-sha256=$DEPENDENCIES_SHA256 \ + org.chipload.recipe-sha256=$RECIPE_SHA256 USER 65532:65532 ENTRYPOINT ["python", "-I", "-B", "-m", "chipload.executor"] diff --git a/deploy/worker/Dockerfile b/deploy/worker/Dockerfile index 509cba4..2015c05 100644 --- a/deploy/worker/Dockerfile +++ b/deploy/worker/Dockerfile @@ -40,10 +40,12 @@ COPY LICENSE /usr/local/share/licenses/chipload/LICENSE ARG SDK_SHA256 ARG DEPENDENCIES_SHA256 ARG CORE_DEPENDENCIES_SHA256 +ARG RECIPE_SHA256 LABEL org.chipload.worker-protocol="1" \ org.chipload.sdk-sha256=$SDK_SHA256 \ org.chipload.dependencies-sha256=$DEPENDENCIES_SHA256 \ org.chipload.core-dependencies-sha256=$CORE_DEPENDENCIES_SHA256 \ + org.chipload.recipe-sha256=$RECIPE_SHA256 \ org.chipload.python-version="3.12.15" USER 65532:65532 diff --git a/docs/FEDERATION_CONTAINERS.md b/docs/FEDERATION_CONTAINERS.md index b51dffc..541954f 100644 --- a/docs/FEDERATION_CONTAINERS.md +++ b/docs/FEDERATION_CONTAINERS.md @@ -16,9 +16,9 @@ Build the images from the same checkout before preparing deployments: The runtime build produces `api`, `broker`, `executor`, `cloud` and `connector` images and records their immutable IDs under `.local/`. The launcher checks the -SDK and lock fingerprints and pins the selected IDs. Use a non-root Linux -account with a local Unix Docker socket. Configuration files and credential -files must be owner-only; keep state directories at mode 0700. +SDK, lock and Dockerfile recipe fingerprints and pins the selected IDs. Use a +non-root Linux account with a local Unix Docker socket. Configuration and +credential files must be owner-only; keep state directories at mode 0700. ## Cloud directory and relay @@ -57,6 +57,13 @@ the existing directory and paired accounting stores before clearing the marker; it never creates or migrates missing accounting state. Restore all three stores from a consistent stopped backup if any is lost. Keep deployment settings and instance-owner bindings with that backup. Do not initialize over a partial store. +Before reopening a restored cloud listener, reconcile current owner bindings +and credential revocations with the identity and secrets services. An older +snapshot can expose a withdrawn advertisement until its lease expires (at most +300 seconds). If post-backup withdrawal history is unavailable, keep the listener +stopped until all restored leases expire. Do not reopen while owner or credential +changes remain unresolved. This launcher has no automated restore or post-backup +reconciliation command. Cloud relay sessions are ephemeral and reconnect after restart; old deliveries are never replayed. This is a single-process, single-host cloud package. diff --git a/docs/RUNTIME.md b/docs/RUNTIME.md index 81e4c6d..c12d470 100644 --- a/docs/RUNTIME.md +++ b/docs/RUNTIME.md @@ -42,9 +42,10 @@ shop projects, local databases, and `.env` files are excluded. Python and Docker CLI base images are pinned by digest. Installation accepts hash-locked binary wheels. Image IDs are saved in `.local/worker-image`, `.local/api-image`, and `.local/broker-image`, plus `.local/executor-image` for the separate executor. -Startup checks image roles, SDK fingerprint, and dependency lock against the -checkout. Rebuild these images after SDK, lock, image recipe or version-pin -changes, then apply them through the deployment's maintenance procedure. +Startup checks image roles, SDK fingerprint, dependency lock, and exact +Dockerfile recipe against the checkout. Rebuild these images after SDK, lock, +image recipe or version-pin changes, then apply them through the deployment's +maintenance procedure. Images carry Chipload's MIT license. The broker also includes the Docker CLI 29.9.2 [license](https://github.com/docker/cli/blob/v29.9.2/LICENSE) and [notice](https://github.com/docker/cli/blob/v29.9.2/NOTICE) under diff --git a/scripts/build-runtime b/scripts/build-runtime index 0abd9d2..8147aa0 100755 --- a/scripts/build-runtime +++ b/scripts/build-runtime @@ -16,6 +16,7 @@ def main(): b"".join(name.encode() + b"\0" + content + b"\0" for name, content in source.items()) ).hexdigest() lock = (root / "uv.lock").read_bytes() + recipe_hash = hashlib.sha256((root / "deploy/runtime/Dockerfile").read_bytes()).hexdigest() with tempfile.TemporaryDirectory(prefix="chipload-runtime-image-") as directory: context = Path(directory) (context / "chipload").mkdir() @@ -56,6 +57,8 @@ def main(): f"SDK_SHA256={sdk_hash}", "--build-arg", f"DEPENDENCIES_SHA256={hashlib.sha256(lock).hexdigest()}", + "--build-arg", + f"RECIPE_SHA256={recipe_hash}", "--iidfile", str(context / "image-id"), str(context), diff --git a/scripts/build-worker b/scripts/build-worker index 54c3103..9c2ba1f 100755 --- a/scripts/build-worker +++ b/scripts/build-worker @@ -43,6 +43,7 @@ def main(): lock = (root / "uv.lock").read_bytes() core_hash = hashlib.sha256(lock).hexdigest() recipe = (root / "deploy/worker/Dockerfile").read_bytes() + recipe_hash = hashlib.sha256(recipe).hexdigest() dependencies_hash = fingerprint(lock, extensions, recipe if args.graphics else b"") with tempfile.TemporaryDirectory(prefix="chipload-worker-image-") as directory: context = Path(directory) @@ -83,6 +84,8 @@ def main(): f"DEPENDENCIES_SHA256={dependencies_hash}", "--build-arg", f"CORE_DEPENDENCIES_SHA256={core_hash}", + "--build-arg", + f"RECIPE_SHA256={recipe_hash}", "--iidfile", str(context / "image-id"), str(context), diff --git a/scripts/runtime b/scripts/runtime index 6a1bbf2..0d460c7 100755 --- a/scripts/runtime +++ b/scripts/runtime @@ -35,6 +35,10 @@ def check_images(images): b"".join(p.name.encode() + b"\0" + p.read_bytes() + b"\0" for p in files) ).hexdigest() lock = hashlib.sha256((ROOT / "uv.lock").read_bytes()).hexdigest() + recipes = { + "worker": hashlib.sha256((ROOT / "deploy/worker/Dockerfile").read_bytes()).hexdigest(), + "runtime": hashlib.sha256((ROOT / "deploy/runtime/Dockerfile").read_bytes()).hexdigest(), + } for role, identifier in images.items(): result = subprocess.run( ["docker", "image", "inspect", identifier, "--format", "{{json .Config.Labels}}"], @@ -50,6 +54,9 @@ def check_images(images): if ( labels.get("org.chipload.sdk-sha256") != expected or core_dependencies != lock + or labels.get("org.chipload.recipe-sha256") != recipes[ + "worker" if role == "worker" else "runtime" + ] ): raise ValueError( f"{role} image differs from the checkout; rebuild worker and runtime images" diff --git a/src/chipload/network.py b/src/chipload/network.py index dea9c45..96ae3cb 100644 --- a/src/chipload/network.py +++ b/src/chipload/network.py @@ -54,8 +54,8 @@ def submit(self, principal, request): encoded = canonical_json(request) if len(encoded) > MAX_NETWORK_REQUEST: raise ValueError("network intent exceeds its byte limit") - when = self.clock().timestamp() with self.repository.transaction() as db: + when = self.clock().timestamp() self.expire(db, when) row = db.execute("SELECT * FROM network_jobs WHERE shop_id=? AND actor=? AND request_key=?", (request.shop_id, who.subject, request.request_key)).fetchone() @@ -133,10 +133,10 @@ def complete(self, principal, completion): who = require(principal, completion.shop_id, "network:dispatch") if (completion.result is None) == (completion.error is None): raise ValueError("network completion requires exactly one result or error") - when = self.clock() # Read-only requests are never reclaimed/replayed after a lost connector. # Rechecking under the write lock fences cancellation and concurrent replies. with self.repository.transaction() as db: + when = self.clock() self.expire(db, when.timestamp()) row = db.execute("SELECT * FROM network_jobs WHERE id=? AND shop_id=? AND dispatcher=?", (completion.job_id, completion.shop_id, who.subject)).fetchone() diff --git a/tests/test_network.py b/tests/test_network.py index 8c5d7da..fe20fe0 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -5,13 +5,14 @@ import unittest import uuid from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import patch from chipload.access import Forbidden, Grant, Principal from chipload.artifact import canonical_json -from chipload.directory_model import SearchPage, SupplierSearch +from chipload.directory_model import ListingSummary, SearchPage, SupplierCandidate, SupplierSearch from chipload.lifecycle import Conflict, NotFound, Repository, ShopService from chipload.network import NetworkService from chipload.network_model import NetworkComplete, NetworkSubmit, search_hash @@ -77,6 +78,48 @@ def test_cancellation_and_deadline_fence_late_or_replaced_replies(self): self.service.complete(self.dispatcher, NetworkComplete(shop_id="mill", job_id=record.id, claim=lease["claim"], result=self.result(intent))) self.assertIsNone(self.service.claim(self.dispatcher, "mill")["lease"]) + def test_submit_and_complete_recheck_time_after_waiting_for_write_lock(self): + transaction = self.repository.transaction + + @contextmanager + def delayed(seconds): + self.now += timedelta(seconds=seconds) + with transaction() as db: + yield db + + with patch.object(self.repository, "transaction", lambda: delayed(31)): + with self.assertRaises(ValueError): + self.service.submit(self.owner, self.intent()) + with self.repository.connection() as db: + self.assertEqual(db.execute("SELECT count(*) FROM network_jobs").fetchone()[0], 0) + + intent = self.intent() + record = self.service.submit(self.owner, intent) + lease = self.service.claim(self.dispatcher, "mill")["lease"] + completion = NetworkComplete(shop_id="mill", job_id=record.id, + claim=lease["claim"], result=self.result(intent)) + with patch.object(self.repository, "transaction", lambda: delayed(31)): + with self.assertRaises((Conflict, NotFound)): + self.service.complete(self.dispatcher, completion) + self.assertEqual(self.service.read(self.owner, "mill", record.id).state, "expired") + + intent = self.intent() + record = self.service.submit(self.owner, intent) + lease = self.service.claim(self.dispatcher, "mill")["lease"] + candidate = SupplierCandidate(instance_id="a" * 32, shop_id="seller", + shop_name="Dummy Seller", listing=ListingSummary(id="stock", name="Stock", + profile=intent.query.query.profile, site=None, target_kind="capability", + target="stock"), source_generation=1, source_revision="b" * 64, + expires_at=self.now + timedelta(seconds=1), direct_origin=None, + relay_available=True, location=None, distance_m=None, + distance_uncertainty_m=None) + completion = NetworkComplete(shop_id="mill", job_id=record.id, + claim=lease["claim"], result=self.result(intent, candidates=(candidate,))) + with patch.object(self.repository, "transaction", lambda: delayed(2)): + with self.assertRaises(ValueError): + self.service.complete(self.dispatcher, completion) + self.assertEqual(self.service.read(self.owner, "mill", record.id).state, "running") + def test_result_identity_and_query_must_match_before_settlement(self): intent = self.intent() record = self.service.submit(self.owner, intent) diff --git a/tests/test_runtime.py b/tests/test_runtime.py index d02d4d5..4a781ce 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -95,6 +95,9 @@ def test_all_roles_sdk_and_dependencies_are_checked_before_startup(self): "org.chipload.dependencies-sha256": hashlib.sha256( (ROOT / "uv.lock").read_bytes() ).hexdigest(), + "org.chipload.recipe-sha256": hashlib.sha256( + (ROOT / "deploy/runtime/Dockerfile").read_bytes() + ).hexdigest(), } labels = { role: { @@ -104,7 +107,13 @@ def test_all_roles_sdk_and_dependencies_are_checked_before_startup(self): } for role in ("api", "broker") } - labels["worker"] = {**common, "org.chipload.worker-protocol": "1"} + labels["worker"] = { + **common, + "org.chipload.worker-protocol": "1", + "org.chipload.recipe-sha256": hashlib.sha256( + (ROOT / "deploy/worker/Dockerfile").read_bytes() + ).hexdigest(), + } def inspect(command, **kwargs): return SimpleNamespace(stdout=json.dumps(labels[command[3]])) @@ -124,6 +133,8 @@ def inspect(command, **kwargs): ("broker", "org.chipload.sdk-sha256", "old-sdk"), ("worker", "org.chipload.dependencies-sha256", "old-lock"), ("worker", "org.chipload.worker-protocol", "0"), + ("api", "org.chipload.recipe-sha256", "old-recipe"), + ("worker", "org.chipload.recipe-sha256", "old-recipe"), ): with self.subTest(role=role, key=key), patch.dict(labels[role], {key: invalid}): with self.assertRaises(ValueError): diff --git a/tests/test_secrets_workload_live.py b/tests/test_secrets_workload_live.py index 825c966..fa38f5f 100644 --- a/tests/test_secrets_workload_live.py +++ b/tests/test_secrets_workload_live.py @@ -276,6 +276,21 @@ def lost_retirement(api, method, path, *args, **kwargs): service("up") service("unseal", share_file=shares[0]) service("unseal", share_file=shares[2]) + # An unsealed restored node may not be active yet. Wait for + # readiness before exercising the restored AppRole credential. + deadline = time.monotonic() + 30 + while True: + status, health = request("GET", "sys/health", allowed=(429, 500)) + if status == 200: + self.assertTrue(health["initialized"]) + self.assertFalse(health["sealed"]) + self.assertEqual( + health["cluster_id"], + script.read_json(state / "initialized.json")["cluster_id"], + ) + break + self.assertLess(time.monotonic(), deadline, f"OpenBao readiness: HTTP {status}") + time.sleep(.2) args.action = "verify" self.assertTrue(workload.operate(args)["verified"]) self.assertEqual(real_client(), hashlib.sha256(bearer.encode()).hexdigest()) diff --git a/tests/test_service_pool.py b/tests/test_service_pool.py index 5c7e641..c94a81f 100644 --- a/tests/test_service_pool.py +++ b/tests/test_service_pool.py @@ -289,7 +289,7 @@ def test_parent_death_terminates_idle_service_processes(self): ) if "Z (zombie)" not in state: running.append(identifier) - except FileNotFoundError: + except (FileNotFoundError, ProcessLookupError): pass if not running: break