From 00a383f95cbae477438900a3ed7fb9c903d01d24 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 20 Sep 2026 15:42:43 +0800 Subject: [PATCH 1/9] feat(demo): add safe deployment and benchmark evidence policies --- deploy/demo/demo_lib.py | 124 ++++++++++++++++++++++++++ deploy/demo/tests/test_demo.py | 153 +++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 deploy/demo/demo_lib.py create mode 100644 deploy/demo/tests/test_demo.py diff --git a/deploy/demo/demo_lib.py b/deploy/demo/demo_lib.py new file mode 100644 index 0000000000..3c33965f8f --- /dev/null +++ b/deploy/demo/demo_lib.py @@ -0,0 +1,124 @@ +"""Pure safety and evidence policies for the single-host Podman demo. + +Author: SqlRush +""" + +import math +from pathlib import Path, PurePosixPath +import re + +SOURCE_REVISION = "83d8c5a002643581f153058b7a766afe1ffa1eb0" +DEMO_VERSION = "v0.131.0-demo.1" +DEFAULT_IMAGE = "ghcr.io/sqlrush/pgrac-demo:" + DEMO_VERSION +DATABASE_UID = 10001 + + +def validate_name(name): + if not re.fullmatch(r"[a-z][a-z0-9-]{0,31}", name): + raise ValueError("name must be 1..32 lowercase letters/digits/hyphens, starting with a letter") + return name + + +def safe_root(base, name): + validate_name(name) + path = Path(base) / name + if not path.is_absolute() or ".." in path.parts: + raise ValueError("storage root must be an absolute, non-traversing path") + for part in [path, *path.parents]: + if part.is_symlink(): + raise ValueError("symlink in storage root: " + str(part)) + return path + + +def storage_relative(path): + if not isinstance(path, str) or not path.startswith("/demo/"): + raise ValueError("path is outside the deployment storage") + parsed = PurePosixPath(path) + if str(parsed) != path or ".." in parsed.parts or len(parsed.parts) < 3: + raise ValueError("non-canonical storage path") + return Path(*parsed.parts[2:]) + + +def start_action(state, running, clean, image_matches): + if not image_matches: + raise ValueError("image differs from initialized deployment; in-place upgrade is not supported") + if state == "READY" and sorted(running) == [0, 1, 2, 3]: + return "verify" + if state == "INITIALIZED" and not running: + return "first" + if state == "CLEAN" and not running and clean == [True] * 4: + return "restart" + raise ValueError("deployment is not a complete running or proven clean cluster; preserve evidence") + + +def exact_loop(device, backing, major_minor, size, actual_device, actual_backing, + actual_major_minor, actual_size): + return ((device, backing, major_minor, size) == + (actual_device, actual_backing, actual_major_minor, actual_size)) + + +def parse_pgbench(text, rc): + values = {} + for key, pattern in ( + ("transactions", r"^number of transactions actually processed: (\d+)(?:/\d+)?\s*$"), + ("failed", r"^number of failed transactions: (\d+) \([^\n]+\)\s*$"), + ("tps", r"^tps = ([0-9.]+) \([^\n]+\)\s*$")): + matches = re.findall(pattern, text, flags=re.MULTILINE) + if len(matches) != 1: + raise ValueError("missing or ambiguous pgbench field: " + key) + values[key] = float(matches[0]) if key == "tps" else int(matches[0]) + if not math.isfinite(values["tps"]): + raise ValueError("non-finite throughput") + values["rc"] = rc + return values + + +def benchmark_verdict(nodes, before, after, server_errors, healthy): + reasons = [] + if len(nodes) != 4 or any(n["rc"] != 0 or n["failed"] != 0 or n["transactions"] <= 0 + for n in nodes): + reasons.append("CLIENT_ERROR_OR_MISSING_NODE") + if len(server_errors) != 4 or any(server_errors): + reasons.append("SERVER_ERROR_OR_MISSING_LOG") + if healthy != [True] * 4: + reasons.append("HEALTH_NOT_PROVEN") + total = sum(n["transactions"] for n in nodes) + for label, snapshots in (("BEFORE", before), ("AFTER", after)): + if len(snapshots) != 4 or any(s != snapshots[0] for s in snapshots): + reasons.append(label + "_ROW_MISMATCH") + if len(before) == 4 and len(after) == 4: + if (before[0]["rows"] <= 0 or before[0]["rows"] != after[0]["rows"] + or before[0]["key_payload_sha256"] != after[0]["key_payload_sha256"]): + reasons.append("KEY_PAYLOAD_OR_COUNT_CHANGED") + if after[0]["sum"] - before[0]["sum"] != total: + reasons.append("COMMITTED_SUM_MISMATCH") + return {"verdict": "FAIL" if reasons else "PASS", "reasons": reasons, + "transactions": total, "reported_tps_sum": sum(n["tps"] for n in nodes), + "nodes": nodes, "is_formal_pre": False} + + +def pod_documents(name, image, storage, loops): + validate_name(name) + if len(loops) != 3 or any(not re.fullmatch(r"/dev/loop\d+", d) for d in loops): + raise ValueError("exactly three Linux loop devices are required") + documents = [] + for i in range(4): + volumes = [{"name": "storage", "hostPath": {"path": storage, "type": "Directory"}}, + {"name": "shm", "emptyDir": {"medium": "Memory", "sizeLimit": "256Mi"}}] + mounts = [{"name": "storage", "mountPath": "/demo"}, + {"name": "shm", "mountPath": "/dev/shm"}] + for j, device in enumerate(loops): + volumes.append({"name": "vote" + str(j), "hostPath": { + "path": device, "type": "BlockDevice"}}) + mounts.append({"name": "vote" + str(j), "mountPath": "/dev/pgrac-vote" + str(j)}) + documents.append({"apiVersion": "v1", "kind": "Pod", "metadata": { + "name": "pgrac-" + name + "-" + str(i), + "labels": {"app": "pgrac-demo", "pgrac-demo": name}}, "spec": { + "hostNetwork": True, "restartPolicy": "Never", "terminationGracePeriodSeconds": 600, + "containers": [{"name": "db", "image": image, "imagePullPolicy": "Never", + "args": ["node", str(i)], "securityContext": { + "runAsUser": DATABASE_UID, "runAsGroup": DATABASE_UID, + "allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}}, + "volumeMounts": mounts}], "volumes": volumes}}) + return documents diff --git a/deploy/demo/tests/test_demo.py b/deploy/demo/tests/test_demo.py new file mode 100644 index 0000000000..2020868577 --- /dev/null +++ b/deploy/demo/tests/test_demo.py @@ -0,0 +1,153 @@ +"""Behavior tests for customer demo safety and evidence. + +Author: SqlRush +""" + +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest + +MODULE = Path(__file__).resolve().parents[1] / "demo_lib.py" +core = None +if MODULE.exists(): + spec = importlib.util.spec_from_file_location("demo_lib", MODULE) + core = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = core + spec.loader.exec_module(core) + + +class DemoPolicies(unittest.TestCase): + def setUp(self): + self.assertIsNotNone(core, "demo safety policy is not implemented") + + def test_names_cannot_escape_storage_or_become_options(self): + for name in ("../data", "", "-rf", "a/b", "a b", "x;id", "X", "a" * 33): + with self.subTest(name=name), self.assertRaises(ValueError): + core.validate_name(name) + self.assertEqual(core.validate_name("customer-01"), "customer-01") + + def test_symlink_parent_or_target_is_rejected_without_touching_it(self): + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + real = base / "real" + real.mkdir() + link = base / "link" + link.symlink_to(real, target_is_directory=True) + with self.assertRaises(ValueError): + core.safe_root(link, "demo") + (real / "demo").symlink_to(base, target_is_directory=True) + with self.assertRaises(ValueError): + core.safe_root(real, "demo") + self.assertTrue((real / "demo").is_symlink()) + + def test_container_paths_cannot_escape_owned_storage(self): + for path in ("/etc/passwd", "/demo/../other", "/demo", "/demo/a//b"): + with self.assertRaises(ValueError): + core.storage_relative(path) + self.assertEqual(core.storage_relative("/demo/tap/node/pgdata"), + Path("tap/node/pgdata")) + + def test_repeat_up_does_not_initialize_or_restart(self): + self.assertEqual(core.start_action("READY", [0, 1, 2, 3], [], True), "verify") + + def test_only_clean_four_member_shutdown_allows_restart(self): + self.assertEqual(core.start_action("CLEAN", [], [True] * 4, True), "restart") + for state, running, clean in (("CLEAN", [], [True, True, False, True]), + ("READY", [0, 1, 2], []), + ("STARTING", [], [True] * 4), + ("INITIALIZING", [], []), + ("CLEAN", [1], [True] * 4)): + with self.subTest(state=state), self.assertRaises(ValueError): + core.start_action(state, running, clean, True) + + def test_new_image_cannot_silently_take_over_old_data(self): + with self.assertRaises(ValueError): + core.start_action("CLEAN", [], [True] * 4, False) + + def test_first_start_is_distinct_from_restart(self): + self.assertEqual(core.start_action("INITIALIZED", [], [], True), "first") + + def test_reused_loop_device_is_not_owned(self): + self.assertTrue(core.exact_loop("/dev/loop7", "/owned/vote0", "7:7", 525824, + "/dev/loop7", "/owned/vote0", "7:7", 525824)) + self.assertFalse(core.exact_loop("/dev/loop7", "/owned/vote0", "7:7", 525824, + "/dev/loop7", "/other/vote0", "7:7", 525824)) + + def test_pgbench_output_preserves_failed_transactions_and_rc(self): + text = ("number of transactions actually processed: 123\n" + "number of failed transactions: 2 (1.6%)\n" + "tps = 12.300000 (without initial connection time)\n") + self.assertEqual(core.parse_pgbench(text, 3), + {"transactions": 123, "failed": 2, "tps": 12.3, "rc": 3}) + + def test_missing_or_duplicate_benchmark_fields_are_not_zero_errors(self): + for text in ("", "tps = 5.0\n", "number of transactions actually processed: 12\n", + self.output() + self.output()): + with self.assertRaises(ValueError): + core.parse_pgbench(text, 0) + + @staticmethod + def output(): + return ("number of transactions actually processed: 10\n" + "number of failed transactions: 0 (0.0%)\n" + "tps = 1.000000 (without initial connection time)\n") + + @staticmethod + def snapshots(total=0, digest="aaa"): + return [{"rows": 100, "sum": total, "sha256": digest, + "key_payload_sha256": "bbb"} for _ in range(4)] + + def test_benchmark_pass_requires_actual_committed_sum_and_full_rows(self): + nodes = [core.parse_pgbench(self.output(), 0) for _ in range(4)] + result = core.benchmark_verdict(nodes, self.snapshots(), self.snapshots(40, "ccc"), + [0] * 4, [True] * 4) + self.assertEqual(result["verdict"], "PASS") + self.assertEqual(result["transactions"], 40) + self.assertEqual(result["reported_tps_sum"], 4.0) + self.assertFalse(result["is_formal_pre"]) + + def test_errors_bad_sums_missing_nodes_and_server_errors_fail(self): + nodes = [core.parse_pgbench(self.output(), 0) for _ in range(4)] + cases = [([dict(nodes[0], rc=2)] + nodes[1:], self.snapshots(40, "ccc"), [0]*4), + ([dict(nodes[0], failed=1)] + nodes[1:], self.snapshots(40, "ccc"), [0]*4), + (nodes, self.snapshots(39, "ccc"), [0]*4), + (nodes[:3], self.snapshots(40, "ccc"), [0]*4), + (nodes, self.snapshots(40, "ccc"), [0, 1, 0, 0])] + for clients, after, errors in cases: + with self.subTest(clients=clients, errors=errors): + result = core.benchmark_verdict(clients, self.snapshots(), after, + errors, [True] * 4) + self.assertEqual(result["verdict"], "FAIL") + self.assertTrue(result["reasons"]) + + def test_equal_totals_do_not_hide_cross_node_different_rows(self): + nodes = [core.parse_pgbench(self.output(), 0) for _ in range(4)] + after = self.snapshots(40, "ccc") + after[3]["sha256"] = "ddd" + self.assertEqual(core.benchmark_verdict(nodes, self.snapshots(), after, + [0]*4, [True]*4)["verdict"], "FAIL") + + def test_four_pods_have_no_automatic_restart_or_privileged_database(self): + manifests = core.pod_documents("demo", "localhost/pgrac:test", "/owned/storage", + ["/dev/loop7", "/dev/loop8", "/dev/loop9"]) + self.assertEqual(len(manifests), 4) + self.assertEqual(len({p["metadata"]["name"] for p in manifests}), 4) + for i, pod in enumerate(manifests): + spec = pod["spec"] + self.assertEqual(spec["restartPolicy"], "Never") + db = spec["containers"][0] + self.assertEqual(db["args"], ["node", str(i)]) + self.assertEqual(db["securityContext"]["runAsUser"], 10001) + self.assertFalse(db["securityContext"].get("privileged", False)) + self.assertFalse(db["securityContext"]["allowPrivilegeEscalation"]) + devices = [v["hostPath"]["path"] for v in spec["volumes"] + if v.get("hostPath", {}).get("type") == "BlockDevice"] + self.assertEqual(devices, ["/dev/loop7", "/dev/loop8", "/dev/loop9"]) + self.assertEqual(json.loads(json.dumps(pod)), pod) + + +if __name__ == "__main__": + unittest.main() From c07f8373d76d49ea64f23492f8669aa8056688e6 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 20 Sep 2026 16:02:15 +0800 Subject: [PATCH 2/9] feat(demo): package stable MVP as four persistent Podman pods Keep database processes non-root, bind device ownership to a single deployment, and allow reuse only after coordinated clean shutdown. Freeze product sources at v0.131.0 without kernel changes. --- deploy/demo/.containerignore | 6 + deploy/demo/.gitignore | 3 + deploy/demo/Containerfile | 41 +++ deploy/demo/bootstrap.pl | 94 ++++++ deploy/demo/demo_lib.py | 44 +++ deploy/demo/entrypoint.py | 42 +++ deploy/demo/pgrac-demo | 430 +++++++++++++++++++++++++++ deploy/demo/tests/test_controller.py | 58 ++++ deploy/demo/tests/test_demo.py | 40 +++ 9 files changed, 758 insertions(+) create mode 100644 deploy/demo/.containerignore create mode 100644 deploy/demo/.gitignore create mode 100644 deploy/demo/Containerfile create mode 100644 deploy/demo/bootstrap.pl create mode 100644 deploy/demo/entrypoint.py create mode 100755 deploy/demo/pgrac-demo create mode 100644 deploy/demo/tests/test_controller.py diff --git a/deploy/demo/.containerignore b/deploy/demo/.containerignore new file mode 100644 index 0000000000..c9bf896833 --- /dev/null +++ b/deploy/demo/.containerignore @@ -0,0 +1,6 @@ +__pycache__ +tests +*.pyc +*.tar +*.tar.gz +reports diff --git a/deploy/demo/.gitignore b/deploy/demo/.gitignore new file mode 100644 index 0000000000..85b95e0ea7 --- /dev/null +++ b/deploy/demo/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +tests/__pycache__/ +*.pyc diff --git a/deploy/demo/Containerfile b/deploy/demo/Containerfile new file mode 100644 index 0000000000..330f3b1b93 --- /dev/null +++ b/deploy/demo/Containerfile @@ -0,0 +1,41 @@ +# Single-host customer demo. Author: SqlRush +FROM docker.io/library/ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 AS build +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential bison flex pkg-config curl ca-certificates libreadline-dev \ + zlib1g-dev libssl-dev libicu-dev liblz4-dev libzstd-dev perl libipc-run-perl +WORKDIR /build +RUN curl --fail --location --retry 3 --output source.tar.gz \ + https://codeload.github.com/sqlrush/pgrac/tar.gz/83d8c5a002643581f153058b7a766afe1ffa1eb0 \ + && echo 'c5b37e5413db30d3a882da4eac9b52208e9dafeb7b5175d1d6c5caff3201d1f2 source.tar.gz' | sha256sum -c - \ + && tar -xzf source.tar.gz --strip-components=1 +RUN ./configure --prefix=/opt/pgrac --enable-cluster --enable-tap-tests \ + --with-openssl --with-icu --with-lz4 --with-zstd \ + && make -j4 && make install && make -C src/test/regress pg_regress \ + && mkdir -p /opt/pgrac/test /opt/pgrac/licenses \ + && cp -a src/test/perl /opt/pgrac/test/ \ + && cp src/test/regress/pg_regress /opt/pgrac/test/ \ + && cp COPYRIGHT /opt/pgrac/licenses/PostgreSQL-COPYRIGHT \ + && sha256sum /opt/pgrac/bin/postgres > /opt/pgrac/postgres.sha256 + +FROM docker.io/library/ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates libreadline8t64 zlib1g libssl3t64 libicu74 liblz4-1 libzstd1 \ + python3 perl libipc-run-perl procps util-linux \ + && groupadd --gid 10001 pgrac && useradd --uid 10001 --gid 10001 --create-home pgrac +COPY --from=build /opt/pgrac /opt/pgrac +COPY bootstrap.pl entrypoint.py demo_lib.py /opt/pgrac/demo/ +COPY sql /opt/pgrac/demo/sql +LABEL org.opencontainers.image.title="PGRAC single-host four-Pod demo" \ + org.opencontainers.image.source="https://github.com/sqlrush/pgrac" \ + org.opencontainers.image.version="v0.131.0-demo.1" \ + org.opencontainers.image.revision="83d8c5a002643581f153058b7a766afe1ffa1eb0" \ + org.opencontainers.image.licenses="PostgreSQL" +ENV PATH="/opt/pgrac/bin:$PATH" LANG=C.UTF-8 \ + PERL5LIB=/opt/pgrac/test/perl PG_REGRESS=/opt/pgrac/test/pg_regress \ + PG_TEST_NOCLEAN=1 PG_TEST_TIMEOUT_DEFAULT=180 PGRAC_STAGE8_HAPPY_PATH_ONLY=1 \ + TESTDATADIR=/demo/tap TESTLOGDIR=/demo/setup-log TEMP_CONFIG=/demo/seed.conf \ + PGPASSFILE=/demo/client.pgpass PGCONNECT_TIMEOUT=5 +USER 10001:10001 +WORKDIR /demo +STOPSIGNAL SIGTERM +ENTRYPOINT ["python3", "/opt/pgrac/demo/entrypoint.py"] diff --git a/deploy/demo/bootstrap.pl b/deploy/demo/bootstrap.pl new file mode 100644 index 0000000000..58d6fba0a3 --- /dev/null +++ b/deploy/demo/bootstrap.pl @@ -0,0 +1,94 @@ +#!/usr/bin/env perl +# New demo seed only; uses the frozen public initializer. +# Author: SqlRush +use strict; +use warnings; +use JSON::PP; +use Fcntl qw(O_CREAT O_EXCL O_WRONLY); +use File::Path qw(make_path); +use PostgreSQL::Test::ClusterQuad; +use Test::More; + +die "expected non-root demo account\n" unless $< == 10001; +sysopen(my $claim, '/demo/INITIALIZED', O_WRONLY | O_CREAT | O_EXCL, 0600) + or die "seed already claimed; preserve this directory\n"; +close($claim) or die $!; +open(my $pf, '<', '/demo/password') or die "password file missing\n"; +my $password = <$pf>; +close($pf); +chomp($password); +die "invalid password format\n" unless $password =~ /\A[0-9a-f]{64}\z/; +open(my $settings, '<', '/demo/settings.json') or die $!; +my $config = decode_json(do { local $/; <$settings> }); +close($settings); +my $base = $config->{port_base}; +die "invalid port base\n" unless defined $base && $base =~ /^\d+$/ && $base >= 1024 && $base <= 65000; + +my $quad; +eval { + $quad = PostgreSQL::Test::ClusterQuad->new_quad('pgrac_demo', + quorum_voting_disks => 3, shared_data => 1, shared_system_identifier => 1, + shared_system_identifier_seed_sql => + "CREATE TABLE demo_account(id integer PRIMARY KEY, value bigint NOT NULL, pad text NOT NULL) WITH (fillfactor=70); " . + "CREATE TABLE demo_smoke(id integer PRIMARY KEY, value bigint NOT NULL); " . + "ALTER ROLE pgrac PASSWORD '$password';", + extra_conf => [ + 'fsync = on', 'full_page_writes = on', 'synchronous_commit = on', + 'autovacuum = off', 'shared_buffers = 128MB', 'max_connections = 32', + 'max_parallel_workers_per_gather = 0', 'log_statement = none', + 'log_min_error_statement = panic', 'restart_after_crash = off', + 'cluster.clean_leave_enabled = on', 'cluster.read_scache = on', + 'cluster.online_join = on', 'cluster.quorum_poll_interval_ms = 2000', + 'cluster.write_fence_lease_ms = 60000', 'cluster.join_convergence_timeout_ms = 30000', + 'cluster.xid_striping = on', 'cluster.crossnode_runtime_visibility = on', + 'cluster.page_scn_shortcut = on', 'cluster.past_image = on', + 'cluster.crossnode_write_write = on', 'cluster.undo_gcs_coherence = on', + 'cluster.crossnode_cr_data_plane = on', 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.gcs_block_retransmit_max_retries = 8', + 'cluster.cssd_heartbeat_interval_ms = 2000', 'cluster.cssd_dead_deadband_factor = 10', + 'cluster.undo_buffers = 8192', 'cluster.pcm_grd_max_entries = 32768', + 'cluster.ges_dedup_max_entries = 16384', 'cluster.gcs_block_dedup_max_entries = 16384', + ]); +}; +if ($@) { + my $error = "$@"; + $error =~ s/\Q$password\E/[REDACTED]/g; + die $error; +} +make_path('/demo/server-log', '/demo/sockets', $quad->shared_data_root . '/pg_undo'); +my $peers = "[cluster]\nname = pgrac_demo\n\n"; +for my $i (0 .. 3) { + $peers .= "[node.$i]\ninterconnect_addr = 127.0.0.1:" . ($base + 100 + $i) . + "\ndata_addr = 127.0.0.1:" . ($base + 200 + 2 * $i) . "\n\n"; +} +my @nodes; +for my $i (0 .. 3) { + my $node = $quad->node($i); + my $dir = $node->data_dir; + $node->append_conf('postgresql.conf', + "port = " . ($base + $i) . "\nlisten_addresses = '127.0.0.1'\n" . + "unix_socket_directories = '/demo/sockets'\nunix_socket_permissions = 0700\n" . + "cluster_name = 'pgrac_demo_node$i'\n" . + "cluster.voting_disks = '/dev/pgrac-vote0,/dev/pgrac-vote1,/dev/pgrac-vote2'\n" . + "cluster.voting_disk_size_bytes = 525824\n"); + open(my $pc, '>', "$dir/pgrac.conf") or die $!; + print {$pc} $peers; + close($pc) or die $!; + open(my $hba, '>', "$dir/pg_hba.conf") or die $!; + print {$hba} "local all all peer\nhost all all 127.0.0.1/32 scram-sha-256\n"; + close($hba) or die $!; + push @nodes, {id => $i, pgdata => $dir, port => $base + $i, + log => "/demo/server-log/node$i.log"}; +} +open(my $pass, '>', '/demo/client.pgpass') or die $!; +print {$pass} "127.0.0.1:*:postgres:pgrac:$password\n"; +close($pass) or die $!; +chmod(0600, '/demo/client.pgpass') or die $!; +open(my $out, '>', '/demo/bootstrap.json') or die $!; +print {$out} JSON::PP->new->canonical->pretty->encode({ + nodes => \@nodes, shared_data => $quad->shared_data_root, + voting => [$quad->voting_disk_paths], source_revision => + '83d8c5a002643581f153058b7a766afe1ffa1eb0'}); +close($out) or die $!; +ok(1, 'homogeneous demo seed prepared; no four-node workload started'); +done_testing(); diff --git a/deploy/demo/demo_lib.py b/deploy/demo/demo_lib.py index 3c33965f8f..e00a972119 100644 --- a/deploy/demo/demo_lib.py +++ b/deploy/demo/demo_lib.py @@ -3,9 +3,11 @@ Author: SqlRush """ +import hashlib import math from pathlib import Path, PurePosixPath import re +import socket SOURCE_REVISION = "83d8c5a002643581f153058b7a766afe1ffa1eb0" DEMO_VERSION = "v0.131.0-demo.1" @@ -13,6 +15,20 @@ DATABASE_UID = 10001 +def check_port_free(port): + # Match PostgreSQL's SO_REUSEADDR: old TIME_WAIT is not a live listener. + with socket.socket() as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", port)) + sock.listen() + + +def removable_pod(info, name, node): + return (info.get("Name") == "pgrac-%s-%d" % (name, node) + and info.get("Labels", {}).get("pgrac-demo") == name + and info.get("State") in ("Exited", "Stopped", "Created")) + + def validate_name(name): if not re.fullmatch(r"[a-z][a-z0-9-]{0,31}", name): raise ValueError("name must be 1..32 lowercase letters/digits/hyphens, starting with a letter") @@ -122,3 +138,31 @@ def pod_documents(name, image, storage, loops): "capabilities": {"drop": ["ALL"]}}, "volumeMounts": mounts}], "volumes": volumes}}) return documents + + +def population_rows(existing, requested): + if existing != 0: + raise ValueError("demo_account already contains data; initialization never truncates/replaces it") + if not isinstance(requested, int) or not 1000 <= requested <= 1000000: + raise ValueError("rows must be in 1000..1000000") + return requested + + +def row_snapshot(stream): + full = hashlib.sha256() + immutable = hashlib.sha256() + rows = total = previous = 0 + for line in stream: + columns = line.rstrip(b"\n").split(b"\t") + if len(columns) != 3: + raise ValueError("unexpected COPY row shape") + key, value = int(columns[0]), int(columns[1]) + if key <= previous or value < 0: + raise ValueError("duplicate/unsorted key or negative counter") + previous = key + rows += 1 + total += value + full.update(line) + immutable.update(columns[0]+b"\t"+columns[2]+b"\n") + return {"rows": rows, "sum": total, "sha256": full.hexdigest(), + "key_payload_sha256": immutable.hexdigest()} diff --git a/deploy/demo/entrypoint.py b/deploy/demo/entrypoint.py new file mode 100644 index 0000000000..e1c9aab185 --- /dev/null +++ b/deploy/demo/entrypoint.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Non-root image entry point. Author: SqlRush """ + +import json +import os +from pathlib import Path +import signal +import subprocess +import sys + +from demo_lib import storage_relative + + +def main(): + if os.getuid() != 10001: + raise ValueError("database image must run as UID 10001, never root") + if sys.argv[1:] == ["bootstrap"]: + os.execvp("perl", ["perl", "/opt/pgrac/demo/bootstrap.pl"]) + if len(sys.argv) != 3 or sys.argv[1] != "node" or sys.argv[2] not in ("0", "1", "2", "3"): + raise ValueError("use bootstrap or node 0..3") + node = json.loads(Path("/demo/bootstrap.json").read_text())["nodes"][int(sys.argv[2])] + storage_relative(node["pgdata"]) + storage_relative(node["log"]) + with open(node["log"], "ab", buffering=0) as log: + process = subprocess.Popen(["postgres", "-D", node["pgdata"]], stdout=log, stderr=log) + + def stop(_sig, _frame): + # SIGINT is PostgreSQL's normal fast shutdown, not immediate/kill. + if process.poll() is None: + process.send_signal(signal.SIGINT) + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + return process.wait() + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (ValueError, OSError, KeyError) as error: + print("demo entry point refused: " + str(error), file=sys.stderr) + sys.exit(1) diff --git a/deploy/demo/pgrac-demo b/deploy/demo/pgrac-demo new file mode 100755 index 0000000000..8546ea93ca --- /dev/null +++ b/deploy/demo/pgrac-demo @@ -0,0 +1,430 @@ +#!/usr/bin/env python3 +"""Single-host four-Pod demo lifecycle. Author: SqlRush """ + +import argparse +import fcntl +import json +import os +from pathlib import Path +import re +import secrets +import shutil +import stat +import subprocess +import sys +import tempfile +import time + +from demo_lib import (DATABASE_UID, DEFAULT_IMAGE, DEMO_VERSION, SOURCE_REVISION, + check_port_free, exact_loop, pod_documents, removable_pod, + safe_root, start_action, storage_relative) + +BASE = Path("/var/lib/pgrac-demo") +HERE = Path(__file__).resolve().parent + + +def run(args, *, check=True, data=None, stdout=None, timeout=180): + result = subprocess.run(args, input=data, text=True, stdout=stdout or subprocess.PIPE, + stderr=subprocess.PIPE, timeout=timeout) + if check and result.returncode: + raise RuntimeError("command failed: " + args[0] + " " + " ".join(args[1:3]) + + "\n" + result.stderr[-3000:]) + return result + + +def save_json(path, value): + with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, prefix=".manifest-", + delete=False) as output: + json.dump(value, output, indent=2) + output.write("\n") + output.flush() + os.fsync(output.fileno()) + temporary = output.name + os.replace(temporary, path) + fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +class Demo: + def __init__(self, args): + self.args = args + self.root = safe_root(BASE, args.name) + self.storage = self.root / "storage" + self.control = self.root / "control.json" + self.state = json.loads(self.control.read_text()) if self.control.exists() else None + if self.state: + if self.control.is_symlink() or self.control.stat().st_uid != 0: + raise ValueError("controller manifest must be a root-owned regular file") + if self.state["name"] != args.name or self.state["version"] != DEMO_VERSION: + raise ValueError("deployment identity/version mismatch") + current = self.storage.stat() + if [current.st_dev, current.st_ino] != self.state["storage_identity"]: + raise ValueError("storage directory identity changed") + + def save(self): + save_json(self.control, self.state) + + def host_path(self, path): + result = self.storage / storage_relative(path) + for part in [result, *result.parents]: + if part == self.root: + break + if part.is_symlink(): + raise ValueError("symlink in managed storage path") + return result + + def container(self, node): + return "pgrac-" + self.args.name + "-" + str(node) + "-db" + + def image(self): + image = self.args.image or (self.state["image"] if self.state else DEFAULT_IMAGE) + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:/@-]+", image): + raise ValueError("invalid image reference") + if run(["podman", "image", "exists", image], check=False).returncode: + run(["podman", "pull", image], stdout=sys.stdout, timeout=900) + metadata = json.loads(run(["podman", "image", "inspect", image]).stdout)[0] + labels = metadata.get("Labels") or metadata.get("Config", {}).get("Labels") or {} + if (labels.get("org.opencontainers.image.revision") != SOURCE_REVISION or + labels.get("org.opencontainers.image.version") != DEMO_VERSION): + raise ValueError("image is not this frozen demo/source version") + return image, metadata["Id"] + + def preflight(self, starting=False): + if sys.platform != "linux" or os.geteuid() != 0: + raise ValueError("use sudo on a Linux host (rootful Podman; databases remain non-root)") + for name in ("podman", "losetup", "blockdev", "findmnt"): + if not shutil.which(name): + raise ValueError("missing dependency: " + name) + version = run(["podman", "--version"]).stdout.strip() + if not Path("/dev/loop-control").exists(): + raise ValueError("/dev/loop-control missing; load the Linux loop module first") + parent = self.storage if self.storage.exists() else BASE + fs = run(["findmnt", "-n", "-o", "FSTYPE", "-T", str(parent)]).stdout.strip() + if fs not in ("ext4", "xfs", "btrfs"): + raise ValueError("local ext4/XFS/Btrfs required; rejected filesystem: " + fs) + free = shutil.disk_usage(parent).free + mem = dict(re.findall(r"^(\w+):\s+(\d+) kB", Path("/proc/meminfo").read_text(), re.M)) + if free < 10 * 1024**3 or int(mem["MemAvailable"]) < 4 * 1024**2: + raise ValueError("need at least 10 GiB free disk and 4 GiB available memory") + if starting: + base = self.state["port_base"] if self.state else self.args.port_base + for port in [*(base+i for i in range(4)), *(base+100+i for i in range(4)), + *(base+200+i for i in range(8))]: + check_port_free(port) + print(json.dumps({"check": "PASS", "runtime": version, "architecture": os.uname().machine, + "filesystem": fs, "free_disk_GiB": round(free/1024**3, 1), + "available_memory_GiB": round(int(mem["MemAvailable"])/1024**2, 1)})) + + def initialize(self, image, image_id): + if self.root.exists(): + raise ValueError("directory already exists without a valid deployment; not overwriting") + self.root.mkdir(mode=0o700) + self.storage.mkdir(mode=0o700) + os.chown(self.storage, DATABASE_UID, DATABASE_UID) + identity = self.storage.stat() + self.state = {"name": self.args.name, "version": DEMO_VERSION, "source_revision": SOURCE_REVISION, + "image": image, "image_id": image_id, "phase": "INITIALIZING", "loops": [], + "port_base": self.args.port_base, "storage_identity": [identity.st_dev, identity.st_ino]} + self.save() + for dirname in ("tap", "setup-log"): + path = self.storage / dirname + path.mkdir(mode=0o700) + os.chown(path, DATABASE_UID, DATABASE_UID) + files = {"password": secrets.token_hex(32) + "\n", + "settings.json": json.dumps({"port_base": self.args.port_base}), + "seed.conf": "fsync=on\nfull_page_writes=on\nsynchronous_commit=on\n" + "log_statement=none\nlog_min_error_statement=panic\nlisten_addresses=''\n"} + for name, content in files.items(): + path = self.storage / name + with path.open("x") as output: + output.write(content) + path.chmod(0o600) + os.chown(path, DATABASE_UID, DATABASE_UID) + with (self.root / "bootstrap.log").open("x") as log: + run(["podman", "run", "--rm", "--network=host", "--user=10001:10001", + "--volume", str(self.storage)+":/demo:rw,z", image, "bootstrap"], + stdout=log, timeout=900) + fixture = json.loads((self.storage / "bootstrap.json").read_text()) + if fixture["source_revision"] != SOURCE_REVISION or len(fixture["nodes"]) != 4: + raise ValueError("invalid bootstrap identity") + for node in fixture["nodes"]: + self.host_path(node["pgdata"]) + self.host_path(node["log"]) + self.host_path(fixture["shared_data"]) + self.state["fixture"] = fixture + voting = [] + for path in fixture["voting"]: + info = self.host_path(path).stat() + if not stat.S_ISREG(info.st_mode) or info.st_size != 525824: + raise ValueError("invalid voting backing") + voting.append({"path": path, "identity": [info.st_dev, info.st_ino], "size": info.st_size}) + if len(voting) != 3: + raise ValueError("three voting backings required") + self.state["voting"] = voting + self.state["phase"] = "INITIALIZED" + self.save() + + def loop_info(self, device): + rows = json.loads(run(["losetup", "--json", "--output", "NAME,BACK-FILE,DIO", device]).stdout)["loopdevices"] + if len(rows) != 1 or rows[0]["dio"] not in (1, True, "1"): + raise ValueError("loop device lacks exact mapping/direct I/O") + info = Path(device).stat() + if not stat.S_ISBLK(info.st_mode): + raise ValueError("voting authority is not a block device") + return {"device": device, "backing": rows[0]["back-file"], + "major_minor": str(os.major(info.st_rdev))+":"+str(os.minor(info.st_rdev)), + "size": int(run(["blockdev", "--getsize64", device]).stdout)} + + def check_loop(self, record): + actual = self.loop_info(record["device"]) + if not exact_loop(*(record[k] for k in ("device", "backing", "major_minor", "size")), + *(actual[k] for k in ("device", "backing", "major_minor", "size"))): + raise ValueError("voting loop identity drift; refusing to use or detach it") + + def attach(self): + if self.state["loops"]: + for record in self.state["loops"]: + self.check_loop(record) + if len(self.state["loops"]) != 3: + raise ValueError("partial prior loop attach; preserve evidence") + return + for voting in self.state["voting"]: + path = self.host_path(voting["path"]) + info = path.stat() + if [info.st_dev, info.st_ino] != voting["identity"] or info.st_size != voting["size"]: + raise ValueError("voting backing identity drift") + if run(["losetup", "-j", str(path)]).stdout.strip(): + raise ValueError("backing already mapped outside this controller") + device = run(["losetup", "--find", "--show", "--direct-io=on", str(path)]).stdout.strip() + if not re.fullmatch(r"/dev/loop\d+", device): + raise ValueError("unexpected loop device path") + record = self.loop_info(device) + if record["backing"] != str(path) or record["size"] != voting["size"]: + raise ValueError("new loop does not map exact backing") + previous = Path(device).stat() + record["permissions"] = [previous.st_uid, previous.st_gid, stat.S_IMODE(previous.st_mode)] + self.state["loops"].append(record) + self.save() + os.chown(device, DATABASE_UID, DATABASE_UID) + os.chmod(device, 0o600) + + def running(self): + active = [] + for i in range(4): + result = run(["podman", "container", "inspect", self.container(i)], check=False) + if result.returncode == 0: + info = json.loads(result.stdout)[0] + if info["Config"]["Labels"].get("pgrac-demo") != self.args.name: + raise ValueError("container name belongs to another deployment") + if info["State"]["Running"]: + active.append(i) + return active + + def clean_controls(self): + clean = [] + for node in self.state["fixture"]["nodes"]: + result = run(["podman", "run", "--rm", "--network=none", "--user=10001:10001", + "--volume", str(self.storage)+":/demo:ro,z", "--entrypoint", "pg_controldata", + self.state["image"], node["pgdata"]]) + clean.append(bool(re.search(r"^Database cluster state:\s+shut down\s*$", result.stdout, re.M))) + return clean + + def sql(self, node, query, check=True, stdout=None, timeout=600): + return run(["podman", "exec", "-i", "-e", "PGOPTIONS=-c statement_timeout=600000", + self.container(node), "psql", "-XAtq", "-v", "ON_ERROR_STOP=1", + "-h", "127.0.0.1", "-p", str(self.state["port_base"]+node), + "-U", "pgrac", "postgres"], data=query, check=check, stdout=stdout, timeout=timeout) + + def healthy(self, node): + query = """SELECT (SELECT in_quorum FROM pg_cluster_quorum_state) + AND (SELECT count(*)=4 FROM pg_cluster_membership WHERE node_id BETWEEN 0 AND 3 + AND state='member' AND NOT removed AND presented_incarnation=last_admitted_incarnation + AND presented_incarnation>0) + AND (SELECT count(*)=3 FROM pg_cluster_state WHERE category='pcm' AND + ((key='resource_x_writer_path' AND value='target') OR (key='resource_x_gate_phase' AND value='open') + OR (key='resource_x_proof_readiness' AND value='AVAILABLE_PROOF_KIND')));""" + result = self.sql(node, query, check=False, timeout=20) + return result.returncode == 0 and result.stdout.strip() == "t" + + def up(self): + image, image_id = self.image() + if self.state is None: + self.preflight(starting=True) + self.initialize(image, image_id) + action = start_action(self.state["phase"], self.running(), + self.clean_controls() if self.state["phase"] == "CLEAN" else [], + image_id == self.state["image_id"]) + if action == "verify": + if not all(self.healthy(i) for i in range(4)): + raise ValueError("running deployment is not healthy") + self.status() + return + self.preflight(starting=True) + self.attach() + # Remove only stopped, identity-checked Pods. Never use force/replace. + for i in range(4): + pod = "pgrac-"+self.args.name+"-"+str(i) + if run(["podman", "pod", "exists", pod], check=False).returncode == 0: + info = json.loads(run(["podman", "pod", "inspect", pod]).stdout)[0] + if not removable_pod(info, self.args.name, i): + raise ValueError("Pod is not an owned stopped deployment; refusing to remove it") + run(["podman", "pod", "rm", pod]) + documents = pod_documents(self.args.name, image, str(self.storage), + [r["device"] for r in self.state["loops"]]) + manifest = self.root / "pods.yaml" + manifest.write_text("\n---\n".join(json.dumps(p, indent=2) for p in documents)+"\n") + self.state["phase"] = "STARTING" + self.state["start_log_offsets"] = self.log_offsets() + self.save() + run(["podman", "kube", "play", "--build=false", str(manifest)], stdout=sys.stdout) + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + if all(self.sql(i, "SELECT 1;", check=False, timeout=10).returncode == 0 for i in range(4)): + break + time.sleep(0.5) + else: + raise ValueError("four-node SQL startup incomplete; preserve data and logs") + if action == "first": + for i in range(4): + self.activate(i) + for _ in range(2): + self.activate(0, require_success=True) + while time.monotonic() < deadline: + if all(self.healthy(i) for i in range(4)): + break + time.sleep(0.5) + else: + raise ValueError("cluster target/open readiness not proven") + if action == "restart": + for i, text in enumerate(self.new_logs(self.state["start_log_offsets"])): + if not re.search(r"database system was shut down at", text) or re.search( + r"automatic recovery in progress|redo starts at|PANIC:|FATAL:", text): + raise ValueError("node%d did not demonstrate normal startup" % i) + self.state["phase"] = "READY" + self.save() + self.status() + + def activate(self, node, require_success=False): + deadline = time.monotonic()+120 + while time.monotonic() < deadline: + result = self.sql(node, "ALTER SYSTEM ENABLE RAC TWO_STAGE ROLLING UPDATES ALL;", + check=False, timeout=50) + if result.returncode == 0: + return + if not re.search(r"RF_DEFERRED|CONDITION_NOT_YET_MET|activation request was refused", result.stderr): + raise RuntimeError("activation rejected: "+result.stderr) + if not require_success: + return + time.sleep(0.2) + raise ValueError("initial activation did not complete") + + def log_offsets(self): + return [self.host_path(n["log"]).stat().st_size if self.host_path(n["log"]).exists() else 0 + for n in self.state["fixture"]["nodes"]] + + def new_logs(self, offsets): + result = [] + for node, offset in zip(self.state["fixture"]["nodes"], offsets): + with self.host_path(node["log"]).open("rb") as log: + if log.seek(0, 2) < offset: + raise ValueError("server log truncated during verification") + log.seek(offset) + result.append(log.read().decode("utf-8", errors="replace")) + return result + + def stop(self): + if self.state["phase"] == "CLEAN" and not self.running() and self.clean_controls() == [True]*4: + print("Already CLEAN; data retained.") + return + if self.state["phase"] != "READY" or self.running() != [0, 1, 2, 3]: + raise ValueError("not a complete ready deployment; no forced cleanup or crash recovery") + offsets = self.log_offsets() + self.state["phase"] = "STOPPING" + self.save() + for node in self.state["fixture"]["nodes"]: + run(["podman", "exec", self.container(node["id"]), "pg_ctl", "-D", node["pgdata"], + "-m", "fast", "-W", "stop"]) + deadline = time.monotonic()+600 + while self.running(): + if time.monotonic() >= deadline: + raise ValueError("normal shutdown incomplete; no force kill, data/devices retained") + time.sleep(0.5) + if self.clean_controls() != [True]*4: + raise ValueError("not all four durable controls are shut down") + for node, text in zip(self.state["fixture"]["nodes"], self.new_logs(offsets)): + if (self.host_path(node["pgdata"]+"/postmaster.pid").exists() or + "cluster normal-stop: protocol closed after shutdown checkpoint" not in text or + "database system is shut down" not in text or re.search(r"PANIC:|FATAL:", text)): + raise ValueError("normal-stop evidence incomplete; retain devices") + for record in reversed(self.state["loops"]): + self.check_loop(record) + uid, gid, mode = record["permissions"] + # Restore before detach: after detach this device number can be reused. + os.chown(record["device"], uid, gid) + os.chmod(record["device"], mode) + run(["losetup", "--detach", record["device"]]) + self.state["loops"] = [] + self.state["phase"] = "CLEAN" + self.save() + print("CLEAN: all four nodes shut down normally; original data retained.") + + def status(self): + print(json.dumps({"deployment": self.args.name, "phase": self.state["phase"], + "running_nodes": self.running(), "image": self.state["image"], + "source_revision": SOURCE_REVISION, "root": str(self.root), + "sql_ports": [self.state["port_base"]+i for i in range(4)]}, indent=2)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--name", default="demo") + parser.add_argument("--image", help="prebuilt image reference; defaults to frozen public demo") + parser.add_argument("--port-base", type=int, default=15432) + parser.add_argument("command", choices=("check", "pull", "up", "status", "stop", "verify", "init-data", "bench", "sql")) + parser.add_argument("--rows", type=int, default=10000) + parser.add_argument("--clients", type=int, default=2, help="connections per node") + parser.add_argument("--seconds", type=int, default=30) + parser.add_argument("--node", type=int, choices=range(4), default=0) + args = parser.parse_args() + if sys.platform != "linux" or os.geteuid() != 0: + parser.error("run with sudo on Linux; database processes run as non-root") + if not 1024 <= args.port_base <= 65000: + parser.error("port base must be in 1024..65000") + safe_root(BASE, args.name) + BASE.mkdir(mode=0o755, exist_ok=True) + if BASE.stat().st_uid != 0 or BASE.stat().st_mode & 0o022: + raise ValueError("deployment base must be root-owned and not group/world writable") + lock_path = BASE / ("."+args.name+".lock") + fd = os.open(lock_path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + demo = Demo(args) + if args.command == "check": + demo.preflight() + elif args.command == "pull": + print(json.dumps(dict(zip(("image", "image_id"), demo.image())))) + elif args.command == "up": + demo.up() + else: + if demo.state is None: + raise ValueError("deployment does not exist; run up first") + if args.command in ("status", "stop"): + getattr(demo, args.command)() + elif args.command == "sql": + result = demo.sql(args.node, sys.stdin.read(), stdout=sys.stdout) + return result.returncode + else: + from workload import execute + execute(demo, args) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (ValueError, RuntimeError, OSError, subprocess.TimeoutExpired, KeyError) as error: + print("REFUSED/FAILED: "+str(error), file=sys.stderr) + sys.exit(1) diff --git a/deploy/demo/tests/test_controller.py b/deploy/demo/tests/test_controller.py new file mode 100644 index 0000000000..376a6334c6 --- /dev/null +++ b/deploy/demo/tests/test_controller.py @@ -0,0 +1,58 @@ +"""Scoped teardown call-order checks. Author: SqlRush """ + +import importlib.machinery +import importlib.util +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +loader = importlib.machinery.SourceFileLoader("demo_controller", str(Path(__file__).resolve().parents[1] / "pgrac-demo")) +spec = importlib.util.spec_from_loader(loader.name, loader) +controller = importlib.util.module_from_spec(spec) +loader.exec_module(controller) + + +class Teardown(unittest.TestCase): + def deployment(self, root): + demo = object.__new__(controller.Demo) + demo.args = Mock(name="demo") + demo.args.name = "demo" + demo.state = {"phase": "READY", "fixture": {"nodes": [ + {"id": i, "pgdata": "/demo/node%d" % i} for i in range(4)]}, + "loops": [{"device": "/dev/loop7", "permissions": [0, 6, 0o660]}]} + demo.log_offsets = Mock(return_value=[0]*4) + demo.running = Mock(side_effect=[[0, 1, 2, 3], []]) + demo.clean_controls = Mock(return_value=[True]*4) + demo.new_logs = Mock(return_value=["cluster normal-stop: protocol closed after shutdown checkpoint\n" + "database system is shut down\n"]*4) + demo.host_path = lambda _path: root / "absent.pid" + demo.save = Mock() + return demo + + def test_permissions_restore_precedes_detach_after_clean_shutdown(self): + with tempfile.TemporaryDirectory() as temporary: + demo = self.deployment(Path(temporary)) + events = [] + demo.check_loop = Mock(side_effect=lambda record: events.append("identity")) + with patch.object(controller, "run", side_effect=lambda args: events.append(args)), \ + patch.object(controller.os, "chown", side_effect=lambda *args: events.append(("chown", args))), \ + patch.object(controller.os, "chmod", side_effect=lambda *args: events.append(("chmod", args))): + demo.stop() + self.assertEqual(events[-4:], ["identity", ("chown", ("/dev/loop7", 0, 6)), + ("chmod", ("/dev/loop7", 0o660)), + ["losetup", "--detach", "/dev/loop7"]]) + self.assertEqual(demo.state["phase"], "CLEAN") + + def test_identity_drift_prevents_permission_change_and_detach(self): + with tempfile.TemporaryDirectory() as temporary: + demo = self.deployment(Path(temporary)) + demo.check_loop = Mock(side_effect=ValueError("wrong backing")) + with patch.object(controller, "run") as run, patch.object(controller.os, "chown") as chown: + with self.assertRaises(ValueError): + demo.stop() + chown.assert_not_called() + self.assertFalse(any(call.args[0][0] == "losetup" for call in run.call_args_list)) + self.assertEqual(demo.state["phase"], "STOPPING") diff --git a/deploy/demo/tests/test_demo.py b/deploy/demo/tests/test_demo.py index 2020868577..ae5db3acee 100644 --- a/deploy/demo/tests/test_demo.py +++ b/deploy/demo/tests/test_demo.py @@ -4,7 +4,9 @@ """ import importlib.util +import io import json +import socket from pathlib import Path import sys import tempfile @@ -148,6 +150,44 @@ def test_four_pods_have_no_automatic_restart_or_privileged_database(self): self.assertEqual(devices, ["/dev/loop7", "/dev/loop8", "/dev/loop9"]) self.assertEqual(json.loads(json.dumps(pod)), pod) + def test_population_refuses_existing_data_and_unbounded_geometry(self): + self.assertTrue(callable(getattr(core, "population_rows", None)), "population guard missing") + self.assertEqual(core.population_rows(0, 10000), 10000) + for count, rows in ((1, 10000), (0, 0), (0, -1), (0, 1000001)): + with self.assertRaises(ValueError): + core.population_rows(count, rows) + + def test_row_snapshot_reads_all_rows_and_rejects_duplicate_keys(self): + self.assertTrue(callable(getattr(core, "row_snapshot", None)), "full-row verifier missing") + value = core.row_snapshot(io.BytesIO(b"1\t2\tx\n2\t4\ty\n")) + self.assertEqual((value["rows"], value["sum"]), (2, 6)) + for data in (b"1\t2\tx\n1\t4\ty\n", b"2\t3\tx\n1\t4\ty\n", b"1\tx\ty\n"): + with self.assertRaises(ValueError): + core.row_snapshot(io.BytesIO(data)) + + def test_port_check_allows_time_wait_but_rejects_a_listener(self): + self.assertTrue(callable(getattr(core, "check_port_free", None)), "port check missing") + with socket.socket() as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + port = server.getsockname()[1] + server.listen() + with self.assertRaises(OSError): + core.check_port_free(port) + with socket.create_connection(("127.0.0.1", port)) as client: + accepted, _ = server.accept() + accepted.close() + self.assertEqual(client.recv(1), b"") + core.check_port_free(port) + + def test_foreign_or_running_pod_is_not_removable(self): + self.assertTrue(callable(getattr(core, "removable_pod", None)), "pod owner guard missing") + owned = {"Name": "pgrac-demo-0", "State": "Exited", "Labels": {"pgrac-demo": "demo"}} + self.assertTrue(core.removable_pod(owned, "demo", 0)) + self.assertFalse(core.removable_pod(dict(owned, Labels={}), "demo", 0)) + self.assertFalse(core.removable_pod(dict(owned, Name="other"), "demo", 0)) + self.assertFalse(core.removable_pod(dict(owned, State="Running"), "demo", 0)) + if __name__ == "__main__": unittest.main() From 751ee289a68302906ac367d26f47337f1ee62842 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 20 Sep 2026 16:02:27 +0800 Subject: [PATCH 3/9] feat(demo): add initialized data and checked four-node benchmark Insert shared test rows only after cluster activation. Preserve client returns and full ordered row evidence, and fail reports on SQL errors, server errors or incorrect committed sums. --- deploy/demo/bench.sh | 5 ++ deploy/demo/init-data.sh | 5 ++ deploy/demo/sql/update.sql | 5 ++ deploy/demo/workload.py | 128 +++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100755 deploy/demo/bench.sh create mode 100755 deploy/demo/init-data.sh create mode 100644 deploy/demo/sql/update.sql create mode 100644 deploy/demo/workload.py diff --git a/deploy/demo/bench.sh b/deploy/demo/bench.sh new file mode 100755 index 0000000000..dacb3ac878 --- /dev/null +++ b/deploy/demo/bench.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Author: SqlRush +set -euo pipefail +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec "$script_dir/pgrac-demo" "$@" bench diff --git a/deploy/demo/init-data.sh b/deploy/demo/init-data.sh new file mode 100755 index 0000000000..22c9bf3575 --- /dev/null +++ b/deploy/demo/init-data.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Author: SqlRush +set -euo pipefail +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec "$script_dir/pgrac-demo" "$@" init-data diff --git a/deploy/demo/sql/update.sql b/deploy/demo/sql/update.sql new file mode 100644 index 0000000000..0763ad9c91 --- /dev/null +++ b/deploy/demo/sql/update.sql @@ -0,0 +1,5 @@ +-- One committed increment per transaction. Author: SqlRush +\set id random(1, :rows) +BEGIN; +UPDATE demo_account SET value=value+1 WHERE id=:id; +COMMIT; diff --git a/deploy/demo/workload.py b/deploy/demo/workload.py new file mode 100644 index 0000000000..49081968da --- /dev/null +++ b/deploy/demo/workload.py @@ -0,0 +1,128 @@ +"""Data initialization and measured demo workload. Author: SqlRush """ + +import json +from pathlib import Path +import re +import subprocess +import time + +from demo_lib import benchmark_verdict, parse_pgbench, population_rows, row_snapshot + + +def snapshots(demo, report, label): + result = [] + for i in range(4): + path = report / (label+"-node"+str(i)+".tsv") + with path.open("x") as output: + demo.sql(i, "COPY (SELECT id,value,pad FROM demo_account ORDER BY id) TO STDOUT;", + stdout=output) + with path.open("rb") as source: + result.append(row_snapshot(source)) + if any(item != result[0] for item in result): + raise ValueError("full ordered demo_account bytes differ between nodes") + return result + + +def summary(path, value): + (path / "summary.json").write_text(json.dumps(value, indent=2)+"\n") + print(json.dumps(value, indent=2)) + print("Reports retained: " + str(path)) + + +def execute(demo, args): + if demo.state["phase"] != "READY" or demo.running() != [0, 1, 2, 3]: + raise ValueError("all four nodes must be READY") + if not all(demo.healthy(i) for i in range(4)): + raise ValueError("cluster readiness check failed") + stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())+"-"+str(time.time_ns()) + report = demo.root / "reports" / (args.command+"-"+stamp) + report.mkdir(parents=True, mode=0o700) + offsets = demo.log_offsets() + result = {"verdict": "FAIL", "is_formal_pre": False, "command": args.command, + "source_revision": demo.state["source_revision"], "image_id": demo.state["image_id"], + "phase": "starting"} + try: + before = snapshots(demo, report, "before") + result["before"] = before + if args.command == "init-data": + rows = population_rows(before[0]["rows"], args.rows) + if demo.sql(0, "SELECT count(*) FROM demo_smoke;").stdout.strip() != "0": + raise ValueError("demo_smoke is not empty; refusing to overwrite prior data") + demo.sql(0, "BEGIN; INSERT INTO demo_account SELECT g,0,repeat('p',64) " + "FROM generate_series(1,%d) AS g; INSERT INTO demo_smoke VALUES(1,0); COMMIT;" % rows) + demo.sql(0, "VACUUM (FREEZE, ANALYZE) demo_account;") + demo.sql(0, "VACUUM (FREEZE, ANALYZE) demo_smoke;") + after = snapshots(demo, report, "after") + if after[0]["rows"] != rows or after[0]["sum"] != 0: + raise ValueError("initialized row count/value mismatch") + demo.state["rows"] = rows + demo.save() + result.update(after=after, initialized_rows=rows) + elif args.command == "verify": + if "rows" not in demo.state: + raise ValueError("run init-data before verify") + if before[0]["rows"] != demo.state["rows"]: + raise ValueError("row count differs from initialized dataset") + # A real committed cross-node write is checked independently of count/hash. + initial = int(demo.sql(0, "SELECT value FROM demo_smoke WHERE id=1;").stdout.strip()) + for i in range(4): + value = demo.sql(i, "UPDATE demo_smoke SET value=value+1 WHERE id=1 RETURNING value;").stdout.strip() + if value != str(initial+i+1): + raise ValueError("cross-node committed increment is incorrect") + if any(demo.sql(i, "SELECT value FROM demo_smoke WHERE id=1;").stdout.strip() + != str(initial+4) for i in range(4)): + raise ValueError("cross-node smoke readback is incorrect") + result["cross_node_increment"] = 4 + else: + if not 1 <= args.clients <= 16 or not 1 <= args.seconds <= 600: + raise ValueError("clients per node must be 1..16; seconds must be 1..600") + rows = demo.state.get("rows") + if not rows or rows != before[0]["rows"]: + raise ValueError("initialize test data before benchmarking") + # Consecutive keys are required: an UPDATE of an absent row is not work. + for i in range(4): + keys = demo.sql(i, "SELECT min(id)||':'||max(id)||':'||count(*) FROM demo_account;").stdout.strip() + if keys != "1:%d:%d" % (rows, rows): + raise ValueError("benchmark requires exact consecutive primary keys") + children, outputs = [], [] + started = time.monotonic() + for i in range(4): + log = (report / ("pgbench-node%d.log" % i)).open("x") + outputs.append(log) + children.append(subprocess.Popen([ + "podman", "exec", demo.container(i), "pgbench", "-h", "127.0.0.1", + "-p", str(demo.state["port_base"]+i), "-U", "pgrac", "-n", + "-c", str(args.clients), "-j", str(args.clients), "-T", str(args.seconds), + "-D", "rows="+str(rows), "-f", "/opt/pgrac/demo/sql/update.sql", "postgres"], + stdout=log, stderr=subprocess.STDOUT)) + # Natural completion only. No forced cancellation is hidden as success. + codes = [child.wait() for child in children] + elapsed = time.monotonic()-started + for output in outputs: + output.close() + (report / "client-return-codes.json").write_text(json.dumps(codes)+"\n") + nodes = [parse_pgbench((report / ("pgbench-node%d.log" % i)).read_text(), codes[i]) + for i in range(4)] + after = snapshots(demo, report, "after") + logs = demo.new_logs(offsets) + errors = [len(re.findall(r"(?:ERROR|FATAL|PANIC):", log)) for log in logs] + health = [demo.healthy(i) for i in range(4)] + for i, text in enumerate(logs): + (report / ("server-node%d.log" % i)).write_text(text) + result.update(benchmark_verdict(nodes, before, after, errors, health)) + result.update(after=after, clients_per_node=args.clients, duration_seconds=args.seconds, + wall_seconds=elapsed, wall_tps=sum(n["transactions"] for n in nodes)/elapsed, + server_errors=errors, healthy=health) + summary(report, result) + if result["verdict"] != "PASS": + raise ValueError("benchmark correctness FAILED; inspect retained report") + return + errors = [len(re.findall(r"(?:ERROR|FATAL|PANIC):", log)) for log in demo.new_logs(offsets)] + if any(errors) or not all(demo.healthy(i) for i in range(4)): + raise ValueError("server error or post-operation health failure") + result.update(verdict="PASS", phase="complete", server_errors=errors) + summary(report, result) + except Exception as error: + result.update(verdict="FAIL", failure=str(error)) + summary(report, result) + raise From 09644202e5ca258b2c6b2290cc7d260723e75ee6 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 20 Sep 2026 16:02:59 +0800 Subject: [PATCH 4/9] feat(demo): document installation and verify native image delivery Provide customer prerequisites, startup, data, benchmark and clean restart commands. Exercise both native architectures and require protected CI before publishing tested OCI images. --- .github/workflows/demo-image.yml | 121 ++++++++++++ deploy/demo/README.md | 252 +++++++++++++++++++++++++ deploy/demo/ci-smoke.sh | 26 +++ deploy/demo/release_gate.py | 28 +++ deploy/demo/tests/test_release_gate.py | 24 +++ docs/mvp/v0.131.0/README.md | 4 + 6 files changed, 455 insertions(+) create mode 100644 .github/workflows/demo-image.yml create mode 100644 deploy/demo/README.md create mode 100755 deploy/demo/ci-smoke.sh create mode 100644 deploy/demo/release_gate.py create mode 100644 deploy/demo/tests/test_release_gate.py diff --git a/.github/workflows/demo-image.yml b/.github/workflows/demo-image.yml new file mode 100644 index 0000000000..a5c89bf54b --- /dev/null +++ b/.github/workflows/demo-image.yml @@ -0,0 +1,121 @@ +# Author: SqlRush +name: Four-Pod demo image + +on: + pull_request: + branches: [main] + paths: ['deploy/demo/**', '.github/workflows/demo-image.yml'] + workflow_dispatch: + inputs: + publish: + description: 'Publish the tested image (main only, after required CI succeeds)' + type: boolean + default: false + +permissions: + contents: read + +jobs: + native: + name: Native demo (${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install -y podman python3 util-linux + sudo modprobe loop + - name: Build frozen product image natively + run: sudo podman build --network=host -t localhost/pgrac-demo:v0.131.0-demo.1 deploy/demo + - name: Four Pods, full rows, workload, normal restart, workload + run: deploy/demo/ci-smoke.sh localhost/pgrac-demo:v0.131.0-demo.1 + - name: Retain secret-free summaries + if: always() + run: | + mkdir -p demo-results + if sudo test -d /var/lib/pgrac-demo/ci-demo/reports; then + sudo find /var/lib/pgrac-demo/ci-demo/reports -name summary.json -type f \ + -exec cp --parents '{}' "$PWD/demo-results" \; + sudo chown -R "$(id -u):$(id -g)" demo-results + fi + - uses: actions/upload-artifact@v4 + if: always() + with: + name: demo-results-${{ matrix.arch }} + path: demo-results + retention-days: 14 + - name: Export the exact tested image + if: github.event_name == 'workflow_dispatch' && inputs.publish && github.ref == 'refs/heads/main' + run: | + sudo podman save --format oci-archive -o "$RUNNER_TEMP/demo-${{ matrix.arch }}.tar" \ + localhost/pgrac-demo:v0.131.0-demo.1 + sudo chmod 644 "$RUNNER_TEMP/demo-${{ matrix.arch }}.tar" + - uses: actions/upload-artifact@v4 + if: github.event_name == 'workflow_dispatch' && inputs.publish && github.ref == 'refs/heads/main' + with: + name: demo-image-${{ matrix.arch }} + path: ${{ runner.temp }}/demo-${{ matrix.arch }}.tar + retention-days: 2 + + publish: + if: github.event_name == 'workflow_dispatch' && inputs.publish && github.ref == 'refs/heads/main' + needs: native + runs-on: ubuntu-24.04 + permissions: + contents: read + checks: read + packages: write + env: + GH_TOKEN: ${{ github.token }} + IMAGE: ghcr.io/sqlrush/pgrac-demo + VERSION: v0.131.0-demo.1 + steps: + - uses: actions/checkout@v4 + - name: Require all protected CI checks on this exact main commit + run: | + gh api --paginate "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/check-runs?per_page=100" \ + --jq '.check_runs[] | {name,conclusion,started_at}' > checks.jsonl + python3 deploy/demo/release_gate.py checks.jsonl + - name: Tools + run: sudo apt-get update && sudo apt-get install -y podman + - uses: actions/download-artifact@v4 + with: + pattern: demo-image-* + path: images + merge-multiple: true + - name: Publish tested native images and multi-architecture index + env: + REGISTRY_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + printf '%s' "$REGISTRY_TOKEN" | podman login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + # Release names are never overwritten. A new delivery requires a new version. + if podman manifest inspect "docker://$IMAGE:$VERSION" >/dev/null 2>&1; then + echo 'Version already exists; refusing to overwrite it.' >&2 + exit 1 + fi + for arch in amd64 arm64; do + podman load -i "images/demo-$arch.tar" + podman tag localhost/pgrac-demo:v0.131.0-demo.1 "$IMAGE:$VERSION-$GITHUB_RUN_ID-$arch" + podman push "$IMAGE:$VERSION-$GITHUB_RUN_ID-$arch" + done + podman manifest create demo-release \ + "docker://$IMAGE:$VERSION-$GITHUB_RUN_ID-amd64" \ + "docker://$IMAGE:$VERSION-$GITHUB_RUN_ID-arm64" + podman manifest push --all --digestfile image-digest.txt demo-release "docker://$IMAGE:$VERSION" + printf 'Published %s@%s\n' "$IMAGE" "$(cat image-digest.txt)" >> "$GITHUB_STEP_SUMMARY" + podman logout ghcr.io + - uses: actions/upload-artifact@v4 + with: + name: demo-release-digest + path: image-digest.txt diff --git a/deploy/demo/README.md b/deploy/demo/README.md new file mode 100644 index 0000000000..038cb2f0f2 --- /dev/null +++ b/deploy/demo/README.md @@ -0,0 +1,252 @@ +# PGRAC 单机四 Pod 安装与演示手册 + +本包在**一台 Linux 主机**上运行四个独立 PGRAC 实例,四个 Pod 共同访问该主机的持久目录和三个演示投票设备。 +数据库版本为 **v0.131.0**,演示镜像版本为 **v0.131.0-demo.1**。 + +不需要编译数据库,不需要 Kubernetes,不需要购买共享 SAN,也不需要 GFS2。 +Git 下载的是部署脚本和 Pod 清单生成器;数据库镜像从 GHCR 下载。 +这是功能演示环境,不是四台物理机共享 LUN 的生产部署认证。 + +## 1. 安装前检查 + +建议专用 Linux 主机或 Linux 虚拟机:4 核以上、8 GiB 以上内存、20 GiB 以上可用磁盘。 +脚本硬检查至少 4 GiB **可用**内存、10 GiB 可用磁盘;这不是性能容量保证。 +需要 root/sudo、Podman、Python 3、util-linux,以及 Linux loop 设备。 +禁止放在容器套容器、rootless Podman 或不提供 loop 设备的受限环境中。 + +以下按 Ubuntu 24.04 安装: + +```bash +sudo apt-get update +sudo apt-get install -y podman python3 git util-linux +sudo modprobe loop +uname -m +podman --version +python3 --version +free -h +df -h /var/lib +findmnt -T /var/lib +test -c /dev/loop-control +``` + +Rocky Linux 9 的依赖命令: + +```bash +sudo dnf install -y podman python3 git util-linux +sudo modprobe loop +``` + +### 存储要求 + +- `/var/lib/pgrac-demo` 必须位于主机的本地 ext4、XFS 或 Btrfs 文件系统;不得用 NFS、CIFS、FUSE、对象存储或跨主机共享挂载替代。 +- **无需格式化磁盘、修改 fstab、配置 GFS2 或手工创建 loop 设备。**已有本地文件系统即可。 +- 主机没有独立磁盘时,直接使用 `/var/lib` 所在文件系统。若要换磁盘,请由管理员先完成正常挂载,部署开始后不要变更挂载或目录身份。 +- 工具只在 `/var/lib/pgrac-demo/<名称>` 下创建数据。已有同名非受管目录、符号链接、旧数据身份不匹配均拒绝,不会自动覆盖。 +- 用户表数据以普通文件保存在共享目录;投票介质是三个专属文件映射出来的 loop **块设备**,工具验证 direct I/O 后才交给四个 Pod。不是把业务数据库放到裸盘。 +- 三个投票设备仍在同一主机/磁盘上,没有三个独立故障域,也不提供存储高可用。 + +### 网络要求 + +默认只监听主机 `127.0.0.1`,不开放公网: + +| 用途 | 默认端口 | +|---|---| +| 四个节点 SQL | 15432–15435 | +| 节点间控制通信 | 15532–15535 | +| 节点间数据通信 | 15632–15639 | + +这些端口必须空闲。不需要开防火墙入站规则。不要把互联端口暴露给其他主机。 +`--port-base` 可在首次部署时改 SQL 基础端口,控制端口偏移 +100、数据端口偏移 +200;同一部署后续保持原值。 +所有 Pod 使用 host network,数据库进程使用 UID/GID 10001,**不是 privileged 容器**;部署工具使用 sudo 管理本演示专属设备。 + +## 2. 下载部署包与镜像 + +```bash +git clone --depth 1 https://github.com/sqlrush/pgrac.git +cd pgrac/deploy/demo +sudo ./pgrac-demo check +sudo ./pgrac-demo pull +``` + +预检输出应包含 `"check": "PASS"`;拉取输出包含镜像名和实际 `image_id`。 +默认镜像为 `ghcr.io/sqlrush/pgrac-demo:v0.131.0-demo.1`。 +不需要 GitHub 账号或 `podman login` 才能拉取公开镜像。 +同一部署会绑定实际镜像 ID;不能用重打标签悄悄给旧数据换版本。 + +离线客户现场:在相同 CPU 架构的联网 Linux 主机上准备: + +```bash +sudo podman pull ghcr.io/sqlrush/pgrac-demo:v0.131.0-demo.1 +sudo podman save --format oci-archive -o pgrac-demo-v0.131.0-demo.1.tar \ + ghcr.io/sqlrush/pgrac-demo:v0.131.0-demo.1 +sha256sum pgrac-demo-v0.131.0-demo.1.tar > pgrac-demo-v0.131.0-demo.1.tar.sha256 +``` + +把整个 `deploy/demo` 目录、镜像 tar、校验文件传到客户主机,安装依赖后执行: + +```bash +sha256sum -c pgrac-demo-v0.131.0-demo.1.tar.sha256 +sudo podman load -i pgrac-demo-v0.131.0-demo.1.tar +``` + +AMD64 镜像用于 x86_64,ARM64 镜像用于 aarch64;不要使用模拟运行的结果做性能比较。 + +## 3. 安装和启动四个节点 + +```bash +sudo ./pgrac-demo up +sudo ./pgrac-demo status +sudo podman pod ps +sudo podman ps +``` + +`up` 自动完成创建数据库目录、同构四节点初始化、投票设备配置、生成四份 Pod 定义、启动和集群就绪检查。 +第一次初始化只创建表结构;业务测试数据在下一步插入。 +启动成功时 `phase` 为 `READY`,`running_nodes` 为 `[0,1,2,3]`。 +再次执行 `up` 只验证现有正常集群,不重新建库。 + +```text +Linux 主机 + ├─ Pod node0 → SQL 127.0.0.1:15432 ┐ + ├─ Pod node1 → SQL 127.0.0.1:15433 ├─ 同一共享用户数据目录 + ├─ Pod node2 → SQL 127.0.0.1:15434 ┤ + 三个演示投票块设备 + └─ Pod node3 → SQL 127.0.0.1:15435 ┘ + 每节点有独立 PGDATA、WAL 和进程空间 +``` + +生成的可检查 Pod 清单: + +```bash +sudo less /var/lib/pgrac-demo/demo/pods.yaml +``` + +不要跳过控制器直接 `podman kube play`:设备编号、数据身份和正常停机记录必须先校验。 +Pod 清单为 JSON 形式的 YAML 文档,由代码按实际设备生成,不要求客户修改 `/dev/loopN`。 + +## 4. 初始化测试数据 + +默认初始化 10,000 行: + +```bash +sudo ./init-data.sh --rows 10000 +sudo ./pgrac-demo verify +``` + +`init-data.sh` 创建 `postgres.public.demo_account` 的测试数据,另初始化跨节点写入探针 `demo_smoke`。 +表结构已经由镜像初始化程序准备好: + +| 列 | 含义 | +|---|---| +| `id integer` | 连续主键,1 到指定行数 | +| `value bigint` | 初始为 0;每笔已提交压测事务加 1 | +| `pad text` | 固定 64 字符载荷,校验期间不得改变 | + +插入后执行 `VACUUM (FREEZE, ANALYZE)`,然后从四节点完整读取所有行并比较。 +可选行数为 1,000–1,000,000。**已有数据时初始化脚本拒绝执行,不 DROP/TRUNCATE。** +`verify` 做全行一致性和四节点顺序提交可见性检查;会使 `demo_smoke.value` 增加 4。 + +查询任意节点,不必安装宿主机 psql,也不必输出密码: + +```bash +printf '%s\n' 'SELECT count(*),sum(value) FROM demo_account;' | sudo ./pgrac-demo --node 0 sql +printf '%s\n' 'SELECT count(*),sum(value) FROM demo_account;' | sudo ./pgrac-demo --node 3 sql +printf '%s\n' 'SELECT * FROM pg_cluster_membership;' | sudo ./pgrac-demo --node 0 sql +``` + +不要在本演示环境执行任意 DDL 或 `pgbench -i`;这些不是本包的初始化流程。 + +## 5. 四节点压测 + +```bash +sudo ./bench.sh --clients 2 --seconds 30 +``` + +含义:每节点 2 条连接,四节点合计 8 条连接,业务发起窗口 30 秒。 +每笔事务随机选择已有主键,执行一次 `UPDATE value=value+1` 并 `COMMIT`。 +脚本为 [sql/update.sql](sql/update.sql),调用镜像内的 `pgbench`。 + +允许 `--clients 1..16`、`--seconds 1..600`。例如: + +```bash +sudo ./bench.sh --clients 4 --seconds 60 +``` + +业务窗口结束后等待已发出的事务自然结束,再进行全行对账和健康检查,因此**总耗时大于 `--seconds`**。 +脚本不会把强制取消或客户端错误算作成功,也不承诺不同硬件上的 TPS。 + +`summary.json` 中关键字段: + +| 字段 | 含义 | +|---|---| +| `verdict` | `PASS` 或 `FAIL`;不能只看 TPS | +| `nodes` | 每节点提交数、失败数、进程返回码和 TPS | +| `reported_tps_sum` | 四个 pgbench 报告 TPS 之和 | +| `wall_tps` | 总提交数 / 四客户端从启动到全部退出的墙钟时间 | +| `before` / `after` | 四节点行数、数值总和、完整数据及固定列摘要 | +| `server_errors` | 每节点本轮新增 ERROR/FATAL/PANIC 数 | +| `healthy` | 负载后四节点实际健康结果 | +| `image_id` / `source_revision` | 对应二进制镜像和源码身份 | +| `is_formal_pre` | 恒为 false:这是客户演示检查,不冒充正式 PRE | + +PASS 要求四节点均有提交、客户端零错误/零非零 RC、服务端零新增错误、完整行内容一致、固定列不变、`sum(value)` 增量等于成功提交总数、负载后健康检查通过。 +报告路径会直接打印,例如 `/var/lib/pgrac-demo/demo/reports/bench-.../`。 + +## 6. 正常停机与原数据重启 + +先停止外部业务,等待 `bench.sh` 结束,再执行: + +```bash +sudo ./pgrac-demo stop +sudo ./pgrac-demo status +sudo ./pgrac-demo up +sudo ./pgrac-demo verify +sudo ./bench.sh --clients 2 --seconds 30 +``` + +`stop` 同时请求四实例正常关闭,确认四份控制文件均已正常关闭及日志闭合后才释放本演示设备,成功输出 `CLEAN`。 +`up` 复用原数据,**不用再次执行 init-data.sh**。 +支持的是**四节点协调正常关闭/重启**;不保证随意单 Pod 重启、掉电、强杀后的恢复。 + +不要使用 `podman stop -a`、`podman rm -f`、`podman kube down --force`、`kill -9`,也不要删除 `postmaster.pid`。 +主机需要关机/重启或虚拟机需要休眠时,先确认 `stop` 成功且状态为 CLEAN。 +正常停机不成功时工具保留数据和设备,不会强杀、重置或悄悄恢复。 + +## 7. 日常检查与故障处理 + +```bash +sudo ./pgrac-demo status +sudo tail -n 100 /var/lib/pgrac-demo/demo/storage/server-log/node0.log +sudo find /var/lib/pgrac-demo/demo/reports -name summary.json -print +``` + +| 现象 | 处理 | +|---|---| +| 缺少 Podman/Python/loop-control | 按第 1 节安装、加载 loop;不要改成 privileged 容器绕过 | +| 端口已被监听 | 查 `sudo ss -ltnp`;选择空闲 `--port-base` 创建新部署,不杀其他服务 | +| 空间/内存不足 | 先正常停止本演示,扩容资源;不要删数据库文件 | +| 镜像 ID 与旧部署不符 | 使用原镜像;本包不支持原地升级 | +| INITIALIZING/STARTING/STOPPING 中断或缺节点 | 保留目录和日志,联系维护者;不把状态手工改为 CLEAN | +| 原数据非正常关闭 | 本包拒绝自动接管,不做崩溃恢复、不重建覆盖原目录 | +| 压测 FAIL | 保存 summary、四份 pgbench 日志和本轮服务器日志;TPS 不构成通过 | +| 校验需要较久 | SQL 验证上限 600 秒;超时返回失败,不会跳过校验 | + +每次安装自动生成随机 SCRAM 密码,文件权限 0600;凭据留在本机,不提交 Git、不放镜像、不输出到状态报告。 +不要公开打包整个 `storage`、bootstrap 日志或控制目录。提供报告前请检查是否含用户自行输入的 SQL/数据。 +没有自动删除/reset 命令;本包不会代替管理员清理旧场景。 + +## 8. 从源码重建镜像(可选) + +客户常规演示无需此步: + +```bash +sudo podman build --network=host -t localhost/pgrac-demo:v0.131.0-demo.1 . +sudo ./pgrac-demo --name source-demo --image localhost/pgrac-demo:v0.131.0-demo.1 up +sudo ./init-data.sh --name source-demo --rows 10000 +sudo ./bench.sh --name source-demo --clients 2 --seconds 30 +sudo ./pgrac-demo --name source-demo stop +``` + +构建固定下载公开稳定源码 `83d8c5a002643581f153058b7a766afe1ffa1eb0`,校验压缩包 SHA-256,镜像包含 PostgreSQL 版权声明。 +官方发布流程原生构建 AMD64/ARM64,并运行 `ci-smoke.sh` 的完整启动、数据、压测、正常重启链;只发布实际通过的镜像。 + +参考:[Podman kube play](https://docs.podman.io/en/latest/markdown/podman-kube-play.1.html)、[GHCR 镜像下载](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)、[数据库稳定版说明](../../docs/release-notes/v0.131.0.md)。 diff --git a/deploy/demo/ci-smoke.sh b/deploy/demo/ci-smoke.sh new file mode 100755 index 0000000000..a2356764e4 --- /dev/null +++ b/deploy/demo/ci-smoke.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Real native four-Pod lifecycle rehearsal. Author: SqlRush +set -euo pipefail +demo_dir=$(cd -- "$(dirname -- "$0")" && pwd) +demo_image=${1:?provide the locally built image reference} +demo_name=${2:-ci-demo} +demo=(sudo "$demo_dir/pgrac-demo" --name "$demo_name" --image "$demo_image") +python3 -m unittest discover -s "$demo_dir/tests" -v +"${demo[@]}" check +"${demo[@]}" up +"${demo[@]}" up +"${demo[@]}" init-data --rows 10000 +if "${demo[@]}" init-data --rows 10000; then + echo 'ERROR: repeated initialization unexpectedly accepted' >&2 + exit 1 +fi +"${demo[@]}" verify +"${demo[@]}" bench --clients 2 --seconds 30 +"${demo[@]}" stop +"${demo[@]}" up +"${demo[@]}" verify +"${demo[@]}" bench --clients 2 --seconds 30 +"${demo[@]}" stop +"${demo[@]}" stop +"${demo[@]}" status +echo 'Native four-Pod lifecycle rehearsal PASS (not formal PRE).' diff --git a/deploy/demo/release_gate.py b/deploy/demo/release_gate.py new file mode 100644 index 0000000000..8fae2cc345 --- /dev/null +++ b/deploy/demo/release_gate.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Publish only after protected CI succeeds. Author: SqlRush """ + +import json +from pathlib import Path +import sys + +REQUIRED = ( + "Validate (comment headers + format + tidy + commit msg)", + "Linux enable-cluster (unit + regress + smoke TAP)", + "Linux --disable-cluster regression", + "macOS build-only (compile guard)", + "Security (cppcheck + scan-build)", +) + + +def failures(checks): + latest = {} + for check in sorted(checks, key=lambda item: item.get("started_at") or ""): + latest[check["name"]] = check.get("conclusion") + return [name for name in REQUIRED if latest.get(name) != "success"] + + +if __name__ == "__main__": + pending = failures([json.loads(line) for line in Path(sys.argv[1]).read_text().splitlines()]) + if pending: + sys.exit("Image publication blocked by: " + ", ".join(pending)) + print("All protected checks succeeded on the queried commit.") diff --git a/deploy/demo/tests/test_release_gate.py b/deploy/demo/tests/test_release_gate.py new file mode 100644 index 0000000000..02e8cb2fa8 --- /dev/null +++ b/deploy/demo/tests/test_release_gate.py @@ -0,0 +1,24 @@ +"""Exact-commit publishing gate tests. Author: SqlRush """ + +import importlib.util +from pathlib import Path +import unittest + +MODULE = Path(__file__).resolve().parents[1] / "release_gate.py" +gate = None +if MODULE.exists(): + spec = importlib.util.spec_from_file_location("release_gate", MODULE) + gate = importlib.util.module_from_spec(spec) + spec.loader.exec_module(gate) + + +class ReleaseGate(unittest.TestCase): + def test_missing_failed_or_newer_running_check_blocks_publish(self): + self.assertIsNotNone(gate, "release gate missing") + good = [{"name": name, "conclusion": "success", "started_at": "2026-09-20T00:00:00Z"} + for name in gate.REQUIRED] + self.assertEqual(gate.failures(good), []) + self.assertTrue(gate.failures(good[:-1])) + for conclusion in (None, "failure", "cancelled", "skipped"): + changed = good + [dict(good[0], conclusion=conclusion, started_at="2026-09-20T01:00:00Z")] + self.assertTrue(gate.failures(changed)) diff --git a/docs/mvp/v0.131.0/README.md b/docs/mvp/v0.131.0/README.md index f0b0a22f21..33daf55ed8 100644 --- a/docs/mvp/v0.131.0/README.md +++ b/docs/mvp/v0.131.0/README.md @@ -15,6 +15,10 @@ UPDATE 耗时观测。稳定发布必须通过四轮正式正确性 PRE 与精 使用本机文件系统上的四个独立 PGDATA,共享业务数据;不是四台主机,不需要容器或 GFS2。 安装步骤沿用已演练示例,本轮没有重新运行 Quick Start 演练。 +免编译演示请用[单机四 Pod 安装与演示手册](../../../deploy/demo/README.md): +Podman 镜像、环境预检、初始化和压测脚本、正常停机与原数据重启。 +该独立演示包不是多机共享盘或生产高可用认证。 + 已有集群不得逐节点热替换:先全体正常关闭、保留原干净数据,再使用同版本二进制正常启动。 没有认证混版本、滚动升级、崩溃恢复、通用迁移或降级;异常退出现场不能当作干净数据接管。 From 53243b9eaf1850c2b5f479109e1d3e6ba54baa08 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 20 Sep 2026 16:11:57 +0800 Subject: [PATCH 5/9] ci(demo): preserve runtime evidence for failed startup The first native CI run reached Pod creation but not SQL readiness. Retain only runtime logs and summaries so the compatibility failure can be diagnosed without exporting bootstrap credentials or database contents. --- .github/workflows/demo-image.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/demo-image.yml b/.github/workflows/demo-image.yml index a5c89bf54b..af37979204 100644 --- a/.github/workflows/demo-image.yml +++ b/.github/workflows/demo-image.yml @@ -39,15 +39,23 @@ jobs: run: sudo podman build --network=host -t localhost/pgrac-demo:v0.131.0-demo.1 deploy/demo - name: Four Pods, full rows, workload, normal restart, workload run: deploy/demo/ci-smoke.sh localhost/pgrac-demo:v0.131.0-demo.1 - - name: Retain secret-free summaries + - name: Retain summaries and runtime-only failure evidence if: always() run: | mkdir -p demo-results + sudo podman ps -a > demo-results/containers.txt + for node in 0 1 2 3; do + sudo podman logs "pgrac-ci-demo-$node-db" > "demo-results/container-$node.log" 2>&1 || true + if sudo test -f "/var/lib/pgrac-demo/ci-demo/storage/server-log/node$node.log"; then + sudo cp "/var/lib/pgrac-demo/ci-demo/storage/server-log/node$node.log" "demo-results/server-$node.log" + fi + done + # Never export bootstrap logs, credentials, settings or entire PGDATA. if sudo test -d /var/lib/pgrac-demo/ci-demo/reports; then sudo find /var/lib/pgrac-demo/ci-demo/reports -name summary.json -type f \ -exec cp --parents '{}' "$PWD/demo-results" \; - sudo chown -R "$(id -u):$(id -g)" demo-results fi + sudo chown -R "$(id -u):$(id -g)" demo-results - uses: actions/upload-artifact@v4 if: always() with: From b4c3f01c78f043770d926fa5b9243edcc1336968 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 20 Sep 2026 16:23:01 +0800 Subject: [PATCH 6/9] ci(demo): isolate socket admission before database startup Compare the identical non-root capability-free TCP probe with runtime-set and entrypoint-set no-new-privileges. Preserve AppArmor denial evidence without changing host policy or database code. --- .github/workflows/demo-image.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/demo-image.yml b/.github/workflows/demo-image.yml index af37979204..25b657f235 100644 --- a/.github/workflows/demo-image.yml +++ b/.github/workflows/demo-image.yml @@ -37,12 +37,22 @@ jobs: sudo modprobe loop - name: Build frozen product image natively run: sudo podman build --network=host -t localhost/pgrac-demo:v0.131.0-demo.1 deploy/demo + - name: Diagnose restricted socket admission before database startup + run: | + set -euo pipefail + probe='import ctypes, pathlib, socket; print(pathlib.Path("/proc/self/attr/current").read_text(), flush=True); print([x for x in pathlib.Path("/proc/self/status").read_text().splitlines() if x.startswith(("NoNewPrivs:", "CapEff:", "Seccomp:"))], flush=True); c=ctypes.CDLL(None, use_errno=True); assert c.prctl(38, 1, 0, 0, 0)==0; print("NNP set inside entry point", flush=True); s=socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP); s.bind(("127.0.0.1", 0)); s.listen(); print("socket PASS", flush=True)' + sudo podman run --rm --network=host --cap-drop=ALL --user=10001:10001 \ + --entrypoint python3 localhost/pgrac-demo:v0.131.0-demo.1 -c "$probe" + sudo podman run --rm --network=host --cap-drop=ALL --user=10001:10001 \ + --security-opt=no-new-privileges --entrypoint python3 \ + localhost/pgrac-demo:v0.131.0-demo.1 -c "$probe" - name: Four Pods, full rows, workload, normal restart, workload run: deploy/demo/ci-smoke.sh localhost/pgrac-demo:v0.131.0-demo.1 - name: Retain summaries and runtime-only failure evidence if: always() run: | mkdir -p demo-results + sudo dmesg | grep 'apparmor="DENIED"' > demo-results/apparmor-denials.txt || true sudo podman ps -a > demo-results/containers.txt for node in 0 1 2 3; do sudo podman logs "pgrac-ci-demo-$node-db" > "demo-results/container-$node.log" 2>&1 || true From ed803faaa91e4e9badf2f6c67e1745dec1c88af7 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 20 Sep 2026 16:30:47 +0800 Subject: [PATCH 7/9] fix(demo): preserve process hardening across AppArmor admission Reproduce Ubuntu crun profile stacking with runtime-applied NNP. Set NNP at the trusted image entrypoint and managed exec boundaries, assert actual kernel flags, and verify running PostgreSQL processes before and after restart. Run kube play from owned storage to avoid its image-name probe colliding with the customer script. --- .github/workflows/demo-image.yml | 8 ++----- deploy/demo/Containerfile | 3 ++- deploy/demo/README.md | 5 +++++ deploy/demo/ci-smoke.sh | 21 ++++++++++++++++++ deploy/demo/demo_lib.py | 12 +++++++++- deploy/demo/entrypoint.py | 15 +++++++++---- deploy/demo/pgrac-demo | 33 ++++++++++++++++++---------- deploy/demo/tests/test_controller.py | 14 ++++++++++++ deploy/demo/tests/test_demo.py | 16 +++++++++++++- deploy/demo/workload.py | 6 ++--- 10 files changed, 105 insertions(+), 28 deletions(-) diff --git a/.github/workflows/demo-image.yml b/.github/workflows/demo-image.yml index 25b657f235..ff1f0874b7 100644 --- a/.github/workflows/demo-image.yml +++ b/.github/workflows/demo-image.yml @@ -37,15 +37,11 @@ jobs: sudo modprobe loop - name: Build frozen product image natively run: sudo podman build --network=host -t localhost/pgrac-demo:v0.131.0-demo.1 deploy/demo - - name: Diagnose restricted socket admission before database startup + - name: Prove restricted socket admission before database startup run: | set -euo pipefail - probe='import ctypes, pathlib, socket; print(pathlib.Path("/proc/self/attr/current").read_text(), flush=True); print([x for x in pathlib.Path("/proc/self/status").read_text().splitlines() if x.startswith(("NoNewPrivs:", "CapEff:", "Seccomp:"))], flush=True); c=ctypes.CDLL(None, use_errno=True); assert c.prctl(38, 1, 0, 0, 0)==0; print("NNP set inside entry point", flush=True); s=socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP); s.bind(("127.0.0.1", 0)); s.listen(); print("socket PASS", flush=True)' sudo podman run --rm --network=host --cap-drop=ALL --user=10001:10001 \ - --entrypoint python3 localhost/pgrac-demo:v0.131.0-demo.1 -c "$probe" - sudo podman run --rm --network=host --cap-drop=ALL --user=10001:10001 \ - --security-opt=no-new-privileges --entrypoint python3 \ - localhost/pgrac-demo:v0.131.0-demo.1 -c "$probe" + localhost/pgrac-demo:v0.131.0-demo.1 runtime-check - name: Four Pods, full rows, workload, normal restart, workload run: deploy/demo/ci-smoke.sh localhost/pgrac-demo:v0.131.0-demo.1 - name: Retain summaries and runtime-only failure evidence diff --git a/deploy/demo/Containerfile b/deploy/demo/Containerfile index 330f3b1b93..b48e2d5cfe 100644 --- a/deploy/demo/Containerfile +++ b/deploy/demo/Containerfile @@ -38,4 +38,5 @@ ENV PATH="/opt/pgrac/bin:$PATH" LANG=C.UTF-8 \ USER 10001:10001 WORKDIR /demo STOPSIGNAL SIGTERM -ENTRYPOINT ["python3", "/opt/pgrac/demo/entrypoint.py"] +# Set NNP after the container's AppArmor transition, before any database code. +ENTRYPOINT ["setpriv", "--no-new-privs", "python3", "/opt/pgrac/demo/entrypoint.py"] diff --git a/deploy/demo/README.md b/deploy/demo/README.md index 038cb2f0f2..6a5021ccd4 100644 --- a/deploy/demo/README.md +++ b/deploy/demo/README.md @@ -59,6 +59,11 @@ sudo modprobe loop `--port-base` 可在首次部署时改 SQL 基础端口,控制端口偏移 +100、数据端口偏移 +200;同一部署后续保持原值。 所有 Pod 使用 host network,数据库进程使用 UID/GID 10001,**不是 privileged 容器**;部署工具使用 sudo 管理本演示专属设备。 +镜像入口通过 `setpriv --no-new-privs` 禁止数据库及其子进程提权,并实际校验 +UID 10001、零 capabilities、seccomp 开启。此标志在容器 AppArmor 配置生效之后设置, +避免部分 Ubuntu 24.04/crun 组合的 TCP 权限缺陷;不关闭主机 AppArmor/SELinux。 +`up` 会在接触数据库目录前实际测试受限容器的 TCP socket;失败时先处理运行环境,不添加 privileged。 + ## 2. 下载部署包与镜像 ```bash diff --git a/deploy/demo/ci-smoke.sh b/deploy/demo/ci-smoke.sh index a2356764e4..cf9f4033d7 100755 --- a/deploy/demo/ci-smoke.sh +++ b/deploy/demo/ci-smoke.sh @@ -2,13 +2,33 @@ # Real native four-Pod lifecycle rehearsal. Author: SqlRush set -euo pipefail demo_dir=$(cd -- "$(dirname -- "$0")" && pwd) +cd -- "$demo_dir" demo_image=${1:?provide the locally built image reference} demo_name=${2:-ci-demo} demo=(sudo "$demo_dir/pgrac-demo" --name "$demo_name" --image "$demo_image") +check_process_security() { + for node in 0 1 2 3; do + sudo podman exec "pgrac-$demo_name-$node-db" setpriv --no-new-privs python3 -c ' +import pathlib, re, sys +sys.path.insert(0, "/opt/pgrac/demo") +from demo_lib import security_status +statuses = [] +for path in pathlib.Path("/proc").glob("[0-9]*/status"): + try: + text = path.read_text() + except FileNotFoundError: + continue + if re.search(r"^Name:\s+postgres$", text, re.M): + statuses.append(security_status(text, int(re.search(r"^Uid:\s+(\d+)", text, re.M)[1]))) +assert statuses, "no actual PostgreSQL process checked" +print("PostgreSQL process security PASS:", len(statuses))' + done +} python3 -m unittest discover -s "$demo_dir/tests" -v "${demo[@]}" check "${demo[@]}" up "${demo[@]}" up +check_process_security "${demo[@]}" init-data --rows 10000 if "${demo[@]}" init-data --rows 10000; then echo 'ERROR: repeated initialization unexpectedly accepted' >&2 @@ -18,6 +38,7 @@ fi "${demo[@]}" bench --clients 2 --seconds 30 "${demo[@]}" stop "${demo[@]}" up +check_process_security "${demo[@]}" verify "${demo[@]}" bench --clients 2 --seconds 30 "${demo[@]}" stop diff --git a/deploy/demo/demo_lib.py b/deploy/demo/demo_lib.py index e00a972119..f5fb52a32c 100644 --- a/deploy/demo/demo_lib.py +++ b/deploy/demo/demo_lib.py @@ -15,6 +15,15 @@ DATABASE_UID = 10001 +def security_status(text, uid): + fields = dict(re.findall(r"^(NoNewPrivs|CapEff|CapBnd|Seccomp):\s+(\S+)", text, re.M)) + if (uid != DATABASE_UID or fields.get("NoNewPrivs") != "1" + or fields.get("CapEff") != "0000000000000000" + or fields.get("CapBnd") != "0000000000000000" or fields.get("Seccomp") != "2"): + raise ValueError("require UID 10001, no-new-privileges, zero capabilities and seccomp") + return fields + + def check_port_free(port): # Match PostgreSQL's SO_REUSEADDR: old TIME_WAIT is not a live listener. with socket.socket() as sock: @@ -134,7 +143,8 @@ def pod_documents(name, image, storage, loops): "containers": [{"name": "db", "image": image, "imagePullPolicy": "Never", "args": ["node", str(i)], "securityContext": { "runAsUser": DATABASE_UID, "runAsGroup": DATABASE_UID, - "allowPrivilegeEscalation": False, + # The image's setpriv entry point applies NNP after + # AppArmor admission (Ubuntu crun stub compatibility). "capabilities": {"drop": ["ALL"]}}, "volumeMounts": mounts}], "volumes": volumes}}) return documents diff --git a/deploy/demo/entrypoint.py b/deploy/demo/entrypoint.py index e1c9aab185..460e3af08f 100644 --- a/deploy/demo/entrypoint.py +++ b/deploy/demo/entrypoint.py @@ -5,19 +5,26 @@ import os from pathlib import Path import signal +import socket import subprocess import sys -from demo_lib import storage_relative +from demo_lib import security_status, storage_relative def main(): - if os.getuid() != 10001: - raise ValueError("database image must run as UID 10001, never root") + security = security_status(Path("/proc/self/status").read_text(), os.getuid()) + if sys.argv[1:] == ["runtime-check"]: + # No data/device mount: prove kernel security and real TCP admission first. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) as sock: + sock.bind(("127.0.0.1", 0)) + sock.listen() + print(json.dumps({"runtime_check": "PASS", "uid": os.getuid(), "security": security})) + return 0 if sys.argv[1:] == ["bootstrap"]: os.execvp("perl", ["perl", "/opt/pgrac/demo/bootstrap.pl"]) if len(sys.argv) != 3 or sys.argv[1] != "node" or sys.argv[2] not in ("0", "1", "2", "3"): - raise ValueError("use bootstrap or node 0..3") + raise ValueError("use runtime-check, bootstrap or node 0..3") node = json.loads(Path("/demo/bootstrap.json").read_text())["nodes"][int(sys.argv[2])] storage_relative(node["pgdata"]) storage_relative(node["log"]) diff --git a/deploy/demo/pgrac-demo b/deploy/demo/pgrac-demo index 8546ea93ca..2d1904afed 100755 --- a/deploy/demo/pgrac-demo +++ b/deploy/demo/pgrac-demo @@ -23,9 +23,9 @@ BASE = Path("/var/lib/pgrac-demo") HERE = Path(__file__).resolve().parent -def run(args, *, check=True, data=None, stdout=None, timeout=180): +def run(args, *, check=True, data=None, stdout=None, timeout=180, cwd=None): result = subprocess.run(args, input=data, text=True, stdout=stdout or subprocess.PIPE, - stderr=subprocess.PIPE, timeout=timeout) + stderr=subprocess.PIPE, timeout=timeout, cwd=cwd) if check and result.returncode: raise RuntimeError("command failed: " + args[0] + " " + " ".join(args[1:3]) + "\n" + result.stderr[-3000:]) @@ -79,6 +79,10 @@ class Demo: def container(self, node): return "pgrac-" + self.args.name + "-" + str(node) + "-db" + def client_command(self, node, command, options=()): + return ["podman", "exec", *options, self.container(node), + "setpriv", "--no-new-privs", *command] + def image(self): image = self.args.image or (self.state["image"] if self.state else DEFAULT_IMAGE) if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:/@-]+", image): @@ -144,7 +148,7 @@ class Demo: path.chmod(0o600) os.chown(path, DATABASE_UID, DATABASE_UID) with (self.root / "bootstrap.log").open("x") as log: - run(["podman", "run", "--rm", "--network=host", "--user=10001:10001", + run(["podman", "run", "--rm", "--network=host", "--user=10001:10001", "--cap-drop=ALL", "--volume", str(self.storage)+":/demo:rw,z", image, "bootstrap"], stdout=log, timeout=900) fixture = json.loads((self.storage / "bootstrap.json").read_text()) @@ -226,17 +230,17 @@ class Demo: def clean_controls(self): clean = [] for node in self.state["fixture"]["nodes"]: - result = run(["podman", "run", "--rm", "--network=none", "--user=10001:10001", - "--volume", str(self.storage)+":/demo:ro,z", "--entrypoint", "pg_controldata", - self.state["image"], node["pgdata"]]) + result = run(["podman", "run", "--rm", "--network=none", "--user=10001:10001", "--cap-drop=ALL", + "--volume", str(self.storage)+":/demo:ro,z", "--entrypoint", "setpriv", + self.state["image"], "--no-new-privs", "pg_controldata", node["pgdata"]]) clean.append(bool(re.search(r"^Database cluster state:\s+shut down\s*$", result.stdout, re.M))) return clean def sql(self, node, query, check=True, stdout=None, timeout=600): - return run(["podman", "exec", "-i", "-e", "PGOPTIONS=-c statement_timeout=600000", - self.container(node), "psql", "-XAtq", "-v", "ON_ERROR_STOP=1", + return run(self.client_command(node, ["psql", "-XAtq", "-v", "ON_ERROR_STOP=1", "-h", "127.0.0.1", "-p", str(self.state["port_base"]+node), - "-U", "pgrac", "postgres"], data=query, check=check, stdout=stdout, timeout=timeout) + "-U", "pgrac", "postgres"], ["-i", "-e", "PGOPTIONS=-c statement_timeout=600000"]), + data=query, check=check, stdout=stdout, timeout=timeout) def healthy(self, node): query = """SELECT (SELECT in_quorum FROM pg_cluster_quorum_state) @@ -251,6 +255,8 @@ class Demo: def up(self): image, image_id = self.image() + run(["podman", "run", "--rm", "--network=host", "--user=10001:10001", "--cap-drop=ALL", + image, "runtime-check"], stdout=sys.stdout) if self.state is None: self.preflight(starting=True) self.initialize(image, image_id) @@ -279,7 +285,10 @@ class Demo: self.state["phase"] = "STARTING" self.state["start_log_offsets"] = self.log_offsets() self.save() - run(["podman", "kube", "play", "--build=false", str(manifest)], stdout=sys.stdout) + # Podman probes cwd//Containerfile even with --build=false. + # Never run in the downloaded script directory containing pgrac-demo. + run(["podman", "kube", "play", "--build=false", str(manifest)], + stdout=sys.stdout, cwd=self.root) deadline = time.monotonic() + 180 while time.monotonic() < deadline: if all(self.sql(i, "SELECT 1;", check=False, timeout=10).returncode == 0 for i in range(4)): @@ -345,8 +354,8 @@ class Demo: self.state["phase"] = "STOPPING" self.save() for node in self.state["fixture"]["nodes"]: - run(["podman", "exec", self.container(node["id"]), "pg_ctl", "-D", node["pgdata"], - "-m", "fast", "-W", "stop"]) + run(self.client_command(node["id"], ["pg_ctl", "-D", node["pgdata"], + "-m", "fast", "-W", "stop"])) deadline = time.monotonic()+600 while self.running(): if time.monotonic() >= deadline: diff --git a/deploy/demo/tests/test_controller.py b/deploy/demo/tests/test_controller.py index 376a6334c6..f90dd2e510 100644 --- a/deploy/demo/tests/test_controller.py +++ b/deploy/demo/tests/test_controller.py @@ -16,6 +16,20 @@ class Teardown(unittest.TestCase): + def test_runtime_command_uses_explicit_owned_working_directory(self): + with patch.object(controller.subprocess, "run", return_value=Mock(returncode=0)) as child: + controller.run(["podman", "kube", "play", "--build=false", "/owned/pods.yaml"], + cwd=Path("/owned")) + self.assertEqual(child.call_args.kwargs["cwd"], Path("/owned")) + + def test_clients_set_no_new_privileges_before_exec(self): + demo = object.__new__(controller.Demo) + demo.args = Mock() + demo.args.name = "demo" + self.assertEqual(demo.client_command(2, ["psql", "-X"], ["-i"]), + ["podman", "exec", "-i", "pgrac-demo-2-db", + "setpriv", "--no-new-privs", "psql", "-X"]) + def deployment(self, root): demo = object.__new__(controller.Demo) demo.args = Mock(name="demo") diff --git a/deploy/demo/tests/test_demo.py b/deploy/demo/tests/test_demo.py index ae5db3acee..2a24167da8 100644 --- a/deploy/demo/tests/test_demo.py +++ b/deploy/demo/tests/test_demo.py @@ -144,10 +144,24 @@ def test_four_pods_have_no_automatic_restart_or_privileged_database(self): self.assertEqual(db["args"], ["node", str(i)]) self.assertEqual(db["securityContext"]["runAsUser"], 10001) self.assertFalse(db["securityContext"].get("privileged", False)) - self.assertFalse(db["securityContext"]["allowPrivilegeEscalation"]) + # setpriv applies NNP after the container AppArmor transition. + # Runtime-applied NNP stacks Ubuntu's crun stub onto this profile. + self.assertNotIn("allowPrivilegeEscalation", db["securityContext"]) devices = [v["hostPath"]["path"] for v in spec["volumes"] if v.get("hostPath", {}).get("type") == "BlockDevice"] self.assertEqual(devices, ["/dev/loop7", "/dev/loop8", "/dev/loop9"]) + + def test_entrypoint_requires_actual_kernel_security_not_a_manifest_claim(self): + status = "NoNewPrivs:\t1\nCapEff:\t0000000000000000\nCapBnd:\t0000000000000000\nSeccomp:\t2\n" + self.assertEqual(core.security_status(status, 10001)["NoNewPrivs"], "1") + for invalid in (status.replace("NoNewPrivs:\t1", "NoNewPrivs:\t0"), + status.replace("CapEff:\t0000000000000000", "CapEff:\t0000000000000001"), + status.replace("CapBnd:\t0000000000000000", "CapBnd:\t0000000000000001"), + status.replace("Seccomp:\t2", "Seccomp:\t0"), ""): + with self.assertRaises(ValueError): + core.security_status(invalid, 10001) + with self.assertRaises(ValueError): + core.security_status(status, 0) self.assertEqual(json.loads(json.dumps(pod)), pod) def test_population_refuses_existing_data_and_unbounded_geometry(self): diff --git a/deploy/demo/workload.py b/deploy/demo/workload.py index 49081968da..07b3f7aa08 100644 --- a/deploy/demo/workload.py +++ b/deploy/demo/workload.py @@ -89,11 +89,11 @@ def execute(demo, args): for i in range(4): log = (report / ("pgbench-node%d.log" % i)).open("x") outputs.append(log) - children.append(subprocess.Popen([ - "podman", "exec", demo.container(i), "pgbench", "-h", "127.0.0.1", + children.append(subprocess.Popen(demo.client_command(i, [ + "pgbench", "-h", "127.0.0.1", "-p", str(demo.state["port_base"]+i), "-U", "pgrac", "-n", "-c", str(args.clients), "-j", str(args.clients), "-T", str(args.seconds), - "-D", "rows="+str(rows), "-f", "/opt/pgrac/demo/sql/update.sql", "postgres"], + "-D", "rows="+str(rows), "-f", "/opt/pgrac/demo/sql/update.sql", "postgres"]), stdout=log, stderr=subprocess.STDOUT)) # Natural completion only. No forced cancellation is hidden as success. codes = [child.wait() for child in children] From 256c37162a3926c7c55b3a5669f3b9bf3e975cc2 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 20 Sep 2026 16:38:00 +0800 Subject: [PATCH 8/9] fix(demo): use stable Podman pod inspect output on restart Podman 4 legacy pod inspect emits an object for one Pod while Podman 5 emits an array. Use inspect --type=pod, whose array output is stable, without changing exact ownership or stopped-state checks. Native Ubuntu reached clean shutdown before exposing this compatibility failure. --- deploy/demo/pgrac-demo | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/demo/pgrac-demo b/deploy/demo/pgrac-demo index 2d1904afed..783b450de3 100755 --- a/deploy/demo/pgrac-demo +++ b/deploy/demo/pgrac-demo @@ -274,7 +274,8 @@ class Demo: for i in range(4): pod = "pgrac-"+self.args.name+"-"+str(i) if run(["podman", "pod", "exists", pod], check=False).returncode == 0: - info = json.loads(run(["podman", "pod", "inspect", pod]).stdout)[0] + # Unlike legacy `pod inspect`, this is an array on Podman 4 and 5. + info = json.loads(run(["podman", "inspect", "--type=pod", pod]).stdout)[0] if not removable_pod(info, self.args.name, i): raise ValueError("Pod is not an owned stopped deployment; refusing to remove it") run(["podman", "pod", "rm", pod]) From 950aa9c77490674d31eb822bed0ce91692ec4904 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sun, 20 Sep 2026 16:41:29 +0800 Subject: [PATCH 9/9] fix(demo): wait for server readiness before SQL startup probes The retained normal-restart trace proves an early probe emitted the expected PostgreSQL starting-up FATAL, which strict startup validation refused. Wait for all four current-launch ready messages before SQL; keep the error gate unchanged and retain the failed attempt. --- deploy/demo/demo_lib.py | 5 +++++ deploy/demo/pgrac-demo | 8 ++++++-- deploy/demo/tests/test_demo.py | 9 ++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/deploy/demo/demo_lib.py b/deploy/demo/demo_lib.py index f5fb52a32c..284f47d05d 100644 --- a/deploy/demo/demo_lib.py +++ b/deploy/demo/demo_lib.py @@ -24,6 +24,11 @@ def security_status(text, uid): return fields +def sql_startup_ready(logs): + return len(logs) == 4 and all(re.search( + r"LOG:\s+database system is ready to accept connections\s*$", text, re.M) for text in logs) + + def check_port_free(port): # Match PostgreSQL's SO_REUSEADDR: old TIME_WAIT is not a live listener. with socket.socket() as sock: diff --git a/deploy/demo/pgrac-demo b/deploy/demo/pgrac-demo index 783b450de3..19564aa7b3 100755 --- a/deploy/demo/pgrac-demo +++ b/deploy/demo/pgrac-demo @@ -17,7 +17,7 @@ import time from demo_lib import (DATABASE_UID, DEFAULT_IMAGE, DEMO_VERSION, SOURCE_REVISION, check_port_free, exact_loop, pod_documents, removable_pod, - safe_root, start_action, storage_relative) + safe_root, sql_startup_ready, start_action, storage_relative) BASE = Path("/var/lib/pgrac-demo") HERE = Path(__file__).resolve().parent @@ -292,7 +292,11 @@ class Demo: stdout=sys.stdout, cwd=self.root) deadline = time.monotonic() + 180 while time.monotonic() < deadline: - if all(self.sql(i, "SELECT 1;", check=False, timeout=10).returncode == 0 for i in range(4)): + # A connection during startup itself generates a FATAL rejection. + # Wait for this launch's ready messages before actively probing SQL. + logs_exist = all(self.host_path(n["log"]).is_file() for n in self.state["fixture"]["nodes"]) + if (logs_exist and sql_startup_ready(self.new_logs(self.state["start_log_offsets"])) + and all(self.sql(i, "SELECT 1;", check=False, timeout=10).returncode == 0 for i in range(4))): break time.sleep(0.5) else: diff --git a/deploy/demo/tests/test_demo.py b/deploy/demo/tests/test_demo.py index 2a24167da8..8a774d8c08 100644 --- a/deploy/demo/tests/test_demo.py +++ b/deploy/demo/tests/test_demo.py @@ -150,6 +150,7 @@ def test_four_pods_have_no_automatic_restart_or_privileged_database(self): devices = [v["hostPath"]["path"] for v in spec["volumes"] if v.get("hostPath", {}).get("type") == "BlockDevice"] self.assertEqual(devices, ["/dev/loop7", "/dev/loop8", "/dev/loop9"]) + self.assertEqual(json.loads(json.dumps(pod)), pod) def test_entrypoint_requires_actual_kernel_security_not_a_manifest_claim(self): status = "NoNewPrivs:\t1\nCapEff:\t0000000000000000\nCapBnd:\t0000000000000000\nSeccomp:\t2\n" @@ -162,7 +163,13 @@ def test_entrypoint_requires_actual_kernel_security_not_a_manifest_claim(self): core.security_status(invalid, 10001) with self.assertRaises(ValueError): core.security_status(status, 0) - self.assertEqual(json.loads(json.dumps(pod)), pod) + + def test_sql_probe_waits_for_all_four_current_startup_logs(self): + ready = "2026-09-20 UTC [2] LOG: database system is ready to accept connections\n" + self.assertTrue(core.sql_startup_ready([ready]*4)) + self.assertFalse(core.sql_startup_ready([ready]*3)) + self.assertFalse(core.sql_startup_ready([ready]*3+["listening on IPv4 address\n"])) + self.assertFalse(core.sql_startup_ready([""]*4)) def test_population_refuses_existing_data_and_unbounded_geometry(self): self.assertTrue(callable(getattr(core, "population_rows", None)), "population guard missing")