diff --git a/.github/workflows/demo-image.yml b/.github/workflows/demo-image.yml new file mode 100644 index 0000000000..ff1f0874b7 --- /dev/null +++ b/.github/workflows/demo-image.yml @@ -0,0 +1,135 @@ +# Author: SqlRush +name: Four-Pod demo image + +on: + pull_request: + branches: [main] + paths: ['deploy/demo/**', '.github/workflows/demo-image.yml'] + workflow_dispatch: + inputs: + publish: + description: 'Publish the tested image (main only, after required CI succeeds)' + type: boolean + default: false + +permissions: + contents: read + +jobs: + native: + name: Native demo (${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - name: Dependencies + run: | + sudo apt-get update + sudo apt-get install -y podman python3 util-linux + sudo modprobe loop + - name: Build frozen product image natively + run: sudo podman build --network=host -t localhost/pgrac-demo:v0.131.0-demo.1 deploy/demo + - name: Prove restricted socket admission before database startup + run: | + set -euo pipefail + sudo podman run --rm --network=host --cap-drop=ALL --user=10001:10001 \ + localhost/pgrac-demo:v0.131.0-demo.1 runtime-check + - name: Four Pods, full rows, workload, normal restart, workload + run: deploy/demo/ci-smoke.sh localhost/pgrac-demo:v0.131.0-demo.1 + - name: Retain summaries and runtime-only failure evidence + if: always() + run: | + mkdir -p demo-results + sudo dmesg | grep 'apparmor="DENIED"' > demo-results/apparmor-denials.txt || true + sudo podman ps -a > demo-results/containers.txt + for node in 0 1 2 3; do + sudo podman logs "pgrac-ci-demo-$node-db" > "demo-results/container-$node.log" 2>&1 || true + if sudo test -f "/var/lib/pgrac-demo/ci-demo/storage/server-log/node$node.log"; then + sudo cp "/var/lib/pgrac-demo/ci-demo/storage/server-log/node$node.log" "demo-results/server-$node.log" + fi + done + # Never export bootstrap logs, credentials, settings or entire PGDATA. + if sudo test -d /var/lib/pgrac-demo/ci-demo/reports; then + sudo find /var/lib/pgrac-demo/ci-demo/reports -name summary.json -type f \ + -exec cp --parents '{}' "$PWD/demo-results" \; + fi + sudo chown -R "$(id -u):$(id -g)" demo-results + - uses: actions/upload-artifact@v4 + if: always() + with: + name: demo-results-${{ matrix.arch }} + path: demo-results + retention-days: 14 + - name: Export the exact tested image + if: github.event_name == 'workflow_dispatch' && inputs.publish && github.ref == 'refs/heads/main' + run: | + sudo podman save --format oci-archive -o "$RUNNER_TEMP/demo-${{ matrix.arch }}.tar" \ + localhost/pgrac-demo:v0.131.0-demo.1 + sudo chmod 644 "$RUNNER_TEMP/demo-${{ matrix.arch }}.tar" + - uses: actions/upload-artifact@v4 + if: github.event_name == 'workflow_dispatch' && inputs.publish && github.ref == 'refs/heads/main' + with: + name: demo-image-${{ matrix.arch }} + path: ${{ runner.temp }}/demo-${{ matrix.arch }}.tar + retention-days: 2 + + publish: + if: github.event_name == 'workflow_dispatch' && inputs.publish && github.ref == 'refs/heads/main' + needs: native + runs-on: ubuntu-24.04 + permissions: + contents: read + checks: read + packages: write + env: + GH_TOKEN: ${{ github.token }} + IMAGE: ghcr.io/sqlrush/pgrac-demo + VERSION: v0.131.0-demo.1 + steps: + - uses: actions/checkout@v4 + - name: Require all protected CI checks on this exact main commit + run: | + gh api --paginate "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/check-runs?per_page=100" \ + --jq '.check_runs[] | {name,conclusion,started_at}' > checks.jsonl + python3 deploy/demo/release_gate.py checks.jsonl + - name: Tools + run: sudo apt-get update && sudo apt-get install -y podman + - uses: actions/download-artifact@v4 + with: + pattern: demo-image-* + path: images + merge-multiple: true + - name: Publish tested native images and multi-architecture index + env: + REGISTRY_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + printf '%s' "$REGISTRY_TOKEN" | podman login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + # Release names are never overwritten. A new delivery requires a new version. + if podman manifest inspect "docker://$IMAGE:$VERSION" >/dev/null 2>&1; then + echo 'Version already exists; refusing to overwrite it.' >&2 + exit 1 + fi + for arch in amd64 arm64; do + podman load -i "images/demo-$arch.tar" + podman tag localhost/pgrac-demo:v0.131.0-demo.1 "$IMAGE:$VERSION-$GITHUB_RUN_ID-$arch" + podman push "$IMAGE:$VERSION-$GITHUB_RUN_ID-$arch" + done + podman manifest create demo-release \ + "docker://$IMAGE:$VERSION-$GITHUB_RUN_ID-amd64" \ + "docker://$IMAGE:$VERSION-$GITHUB_RUN_ID-arm64" + podman manifest push --all --digestfile image-digest.txt demo-release "docker://$IMAGE:$VERSION" + printf 'Published %s@%s\n' "$IMAGE" "$(cat image-digest.txt)" >> "$GITHUB_STEP_SUMMARY" + podman logout ghcr.io + - uses: actions/upload-artifact@v4 + with: + name: demo-release-digest + path: image-digest.txt diff --git a/deploy/demo/.containerignore b/deploy/demo/.containerignore new file mode 100644 index 0000000000..c9bf896833 --- /dev/null +++ b/deploy/demo/.containerignore @@ -0,0 +1,6 @@ +__pycache__ +tests +*.pyc +*.tar +*.tar.gz +reports diff --git a/deploy/demo/.gitignore b/deploy/demo/.gitignore new file mode 100644 index 0000000000..85b95e0ea7 --- /dev/null +++ b/deploy/demo/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +tests/__pycache__/ +*.pyc diff --git a/deploy/demo/Containerfile b/deploy/demo/Containerfile new file mode 100644 index 0000000000..b48e2d5cfe --- /dev/null +++ b/deploy/demo/Containerfile @@ -0,0 +1,42 @@ +# Single-host customer demo. Author: SqlRush +FROM docker.io/library/ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 AS build +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential bison flex pkg-config curl ca-certificates libreadline-dev \ + zlib1g-dev libssl-dev libicu-dev liblz4-dev libzstd-dev perl libipc-run-perl +WORKDIR /build +RUN curl --fail --location --retry 3 --output source.tar.gz \ + https://codeload.github.com/sqlrush/pgrac/tar.gz/83d8c5a002643581f153058b7a766afe1ffa1eb0 \ + && echo 'c5b37e5413db30d3a882da4eac9b52208e9dafeb7b5175d1d6c5caff3201d1f2 source.tar.gz' | sha256sum -c - \ + && tar -xzf source.tar.gz --strip-components=1 +RUN ./configure --prefix=/opt/pgrac --enable-cluster --enable-tap-tests \ + --with-openssl --with-icu --with-lz4 --with-zstd \ + && make -j4 && make install && make -C src/test/regress pg_regress \ + && mkdir -p /opt/pgrac/test /opt/pgrac/licenses \ + && cp -a src/test/perl /opt/pgrac/test/ \ + && cp src/test/regress/pg_regress /opt/pgrac/test/ \ + && cp COPYRIGHT /opt/pgrac/licenses/PostgreSQL-COPYRIGHT \ + && sha256sum /opt/pgrac/bin/postgres > /opt/pgrac/postgres.sha256 + +FROM docker.io/library/ubuntu:24.04@sha256:008173c23f95b170204355c12626cb5a965d779a7e1283b09e9cffbb1bf33ca3 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates libreadline8t64 zlib1g libssl3t64 libicu74 liblz4-1 libzstd1 \ + python3 perl libipc-run-perl procps util-linux \ + && groupadd --gid 10001 pgrac && useradd --uid 10001 --gid 10001 --create-home pgrac +COPY --from=build /opt/pgrac /opt/pgrac +COPY bootstrap.pl entrypoint.py demo_lib.py /opt/pgrac/demo/ +COPY sql /opt/pgrac/demo/sql +LABEL org.opencontainers.image.title="PGRAC single-host four-Pod demo" \ + org.opencontainers.image.source="https://github.com/sqlrush/pgrac" \ + org.opencontainers.image.version="v0.131.0-demo.1" \ + org.opencontainers.image.revision="83d8c5a002643581f153058b7a766afe1ffa1eb0" \ + org.opencontainers.image.licenses="PostgreSQL" +ENV PATH="/opt/pgrac/bin:$PATH" LANG=C.UTF-8 \ + PERL5LIB=/opt/pgrac/test/perl PG_REGRESS=/opt/pgrac/test/pg_regress \ + PG_TEST_NOCLEAN=1 PG_TEST_TIMEOUT_DEFAULT=180 PGRAC_STAGE8_HAPPY_PATH_ONLY=1 \ + TESTDATADIR=/demo/tap TESTLOGDIR=/demo/setup-log TEMP_CONFIG=/demo/seed.conf \ + PGPASSFILE=/demo/client.pgpass PGCONNECT_TIMEOUT=5 +USER 10001:10001 +WORKDIR /demo +STOPSIGNAL SIGTERM +# Set NNP after the container's AppArmor transition, before any database code. +ENTRYPOINT ["setpriv", "--no-new-privs", "python3", "/opt/pgrac/demo/entrypoint.py"] diff --git a/deploy/demo/README.md b/deploy/demo/README.md new file mode 100644 index 0000000000..6a5021ccd4 --- /dev/null +++ b/deploy/demo/README.md @@ -0,0 +1,257 @@ +# PGRAC 单机四 Pod 安装与演示手册 + +本包在**一台 Linux 主机**上运行四个独立 PGRAC 实例，四个 Pod 共同访问该主机的持久目录和三个演示投票设备。 +数据库版本为 **v0.131.0**，演示镜像版本为 **v0.131.0-demo.1**。 + +不需要编译数据库，不需要 Kubernetes，不需要购买共享 SAN，也不需要 GFS2。 +Git 下载的是部署脚本和 Pod 清单生成器；数据库镜像从 GHCR 下载。 +这是功能演示环境，不是四台物理机共享 LUN 的生产部署认证。 + +## 1. 安装前检查 + +建议专用 Linux 主机或 Linux 虚拟机：4 核以上、8 GiB 以上内存、20 GiB 以上可用磁盘。 +脚本硬检查至少 4 GiB **可用**内存、10 GiB 可用磁盘；这不是性能容量保证。 +需要 root/sudo、Podman、Python 3、util-linux，以及 Linux loop 设备。 +禁止放在容器套容器、rootless Podman 或不提供 loop 设备的受限环境中。 + +以下按 Ubuntu 24.04 安装： + +```bash +sudo apt-get update +sudo apt-get install -y podman python3 git util-linux +sudo modprobe loop +uname -m +podman --version +python3 --version +free -h +df -h /var/lib +findmnt -T /var/lib +test -c /dev/loop-control +``` + +Rocky Linux 9 的依赖命令： + +```bash +sudo dnf install -y podman python3 git util-linux +sudo modprobe loop +``` + +### 存储要求 + +- `/var/lib/pgrac-demo` 必须位于主机的本地 ext4、XFS 或 Btrfs 文件系统；不得用 NFS、CIFS、FUSE、对象存储或跨主机共享挂载替代。 +- **无需格式化磁盘、修改 fstab、配置 GFS2 或手工创建 loop 设备。**已有本地文件系统即可。 +- 主机没有独立磁盘时，直接使用 `/var/lib` 所在文件系统。若要换磁盘，请由管理员先完成正常挂载，部署开始后不要变更挂载或目录身份。 +- 工具只在 `/var/lib/pgrac-demo/<名称>` 下创建数据。已有同名非受管目录、符号链接、旧数据身份不匹配均拒绝，不会自动覆盖。 +- 用户表数据以普通文件保存在共享目录；投票介质是三个专属文件映射出来的 loop **块设备**，工具验证 direct I/O 后才交给四个 Pod。不是把业务数据库放到裸盘。 +- 三个投票设备仍在同一主机/磁盘上，没有三个独立故障域，也不提供存储高可用。 + +### 网络要求 + +默认只监听主机 `127.0.0.1`，不开放公网： + +| 用途 | 默认端口 | +|---|---| +| 四个节点 SQL | 15432–15435 | +| 节点间控制通信 | 15532–15535 | +| 节点间数据通信 | 15632–15639 | + +这些端口必须空闲。不需要开防火墙入站规则。不要把互联端口暴露给其他主机。 +`--port-base` 可在首次部署时改 SQL 基础端口，控制端口偏移 +100、数据端口偏移 +200；同一部署后续保持原值。 +所有 Pod 使用 host network，数据库进程使用 UID/GID 10001，**不是 privileged 容器**；部署工具使用 sudo 管理本演示专属设备。 + +镜像入口通过 `setpriv --no-new-privs` 禁止数据库及其子进程提权，并实际校验 +UID 10001、零 capabilities、seccomp 开启。此标志在容器 AppArmor 配置生效之后设置， +避免部分 Ubuntu 24.04/crun 组合的 TCP 权限缺陷；不关闭主机 AppArmor/SELinux。 +`up` 会在接触数据库目录前实际测试受限容器的 TCP socket；失败时先处理运行环境，不添加 privileged。 + +## 2. 下载部署包与镜像 + +```bash +git clone --depth 1 https://github.com/sqlrush/pgrac.git +cd pgrac/deploy/demo +sudo ./pgrac-demo check +sudo ./pgrac-demo pull +``` + +预检输出应包含 `"check": "PASS"`；拉取输出包含镜像名和实际 `image_id`。 +默认镜像为 `ghcr.io/sqlrush/pgrac-demo:v0.131.0-demo.1`。 +不需要 GitHub 账号或 `podman login` 才能拉取公开镜像。 +同一部署会绑定实际镜像 ID；不能用重打标签悄悄给旧数据换版本。 + +离线客户现场：在相同 CPU 架构的联网 Linux 主机上准备： + +```bash +sudo podman pull ghcr.io/sqlrush/pgrac-demo:v0.131.0-demo.1 +sudo podman save --format oci-archive -o pgrac-demo-v0.131.0-demo.1.tar \ + ghcr.io/sqlrush/pgrac-demo:v0.131.0-demo.1 +sha256sum pgrac-demo-v0.131.0-demo.1.tar > pgrac-demo-v0.131.0-demo.1.tar.sha256 +``` + +把整个 `deploy/demo` 目录、镜像 tar、校验文件传到客户主机，安装依赖后执行： + +```bash +sha256sum -c pgrac-demo-v0.131.0-demo.1.tar.sha256 +sudo podman load -i pgrac-demo-v0.131.0-demo.1.tar +``` + +AMD64 镜像用于 x86_64，ARM64 镜像用于 aarch64；不要使用模拟运行的结果做性能比较。 + +## 3. 安装和启动四个节点 + +```bash +sudo ./pgrac-demo up +sudo ./pgrac-demo status +sudo podman pod ps +sudo podman ps +``` + +`up` 自动完成创建数据库目录、同构四节点初始化、投票设备配置、生成四份 Pod 定义、启动和集群就绪检查。 +第一次初始化只创建表结构；业务测试数据在下一步插入。 +启动成功时 `phase` 为 `READY`，`running_nodes` 为 `[0,1,2,3]`。 +再次执行 `up` 只验证现有正常集群，不重新建库。 + +```text +Linux 主机 + ├─ Pod node0 → SQL 127.0.0.1:15432 ┐ + ├─ Pod node1 → SQL 127.0.0.1:15433 ├─ 同一共享用户数据目录 + ├─ Pod node2 → SQL 127.0.0.1:15434 ┤ + 三个演示投票块设备 + └─ Pod node3 → SQL 127.0.0.1:15435 ┘ + 每节点有独立 PGDATA、WAL 和进程空间 +``` + +生成的可检查 Pod 清单： + +```bash +sudo less /var/lib/pgrac-demo/demo/pods.yaml +``` + +不要跳过控制器直接 `podman kube play`：设备编号、数据身份和正常停机记录必须先校验。 +Pod 清单为 JSON 形式的 YAML 文档，由代码按实际设备生成，不要求客户修改 `/dev/loopN`。 + +## 4. 初始化测试数据 + +默认初始化 10,000 行： + +```bash +sudo ./init-data.sh --rows 10000 +sudo ./pgrac-demo verify +``` + +`init-data.sh` 创建 `postgres.public.demo_account` 的测试数据，另初始化跨节点写入探针 `demo_smoke`。 +表结构已经由镜像初始化程序准备好： + +| 列 | 含义 | +|---|---| +| `id integer` | 连续主键，1 到指定行数 | +| `value bigint` | 初始为 0；每笔已提交压测事务加 1 | +| `pad text` | 固定 64 字符载荷，校验期间不得改变 | + +插入后执行 `VACUUM (FREEZE, ANALYZE)`，然后从四节点完整读取所有行并比较。 +可选行数为 1,000–1,000,000。**已有数据时初始化脚本拒绝执行，不 DROP/TRUNCATE。** +`verify` 做全行一致性和四节点顺序提交可见性检查；会使 `demo_smoke.value` 增加 4。 + +查询任意节点，不必安装宿主机 psql，也不必输出密码： + +```bash +printf '%s\n' 'SELECT count(*),sum(value) FROM demo_account;' | sudo ./pgrac-demo --node 0 sql +printf '%s\n' 'SELECT count(*),sum(value) FROM demo_account;' | sudo ./pgrac-demo --node 3 sql +printf '%s\n' 'SELECT * FROM pg_cluster_membership;' | sudo ./pgrac-demo --node 0 sql +``` + +不要在本演示环境执行任意 DDL 或 `pgbench -i`；这些不是本包的初始化流程。 + +## 5. 四节点压测 + +```bash +sudo ./bench.sh --clients 2 --seconds 30 +``` + +含义：每节点 2 条连接，四节点合计 8 条连接，业务发起窗口 30 秒。 +每笔事务随机选择已有主键，执行一次 `UPDATE value=value+1` 并 `COMMIT`。 +脚本为 [sql/update.sql](sql/update.sql)，调用镜像内的 `pgbench`。 + +允许 `--clients 1..16`、`--seconds 1..600`。例如： + +```bash +sudo ./bench.sh --clients 4 --seconds 60 +``` + +业务窗口结束后等待已发出的事务自然结束，再进行全行对账和健康检查，因此**总耗时大于 `--seconds`**。 +脚本不会把强制取消或客户端错误算作成功，也不承诺不同硬件上的 TPS。 + +`summary.json` 中关键字段： + +| 字段 | 含义 | +|---|---| +| `verdict` | `PASS` 或 `FAIL`；不能只看 TPS | +| `nodes` | 每节点提交数、失败数、进程返回码和 TPS | +| `reported_tps_sum` | 四个 pgbench 报告 TPS 之和 | +| `wall_tps` | 总提交数 / 四客户端从启动到全部退出的墙钟时间 | +| `before` / `after` | 四节点行数、数值总和、完整数据及固定列摘要 | +| `server_errors` | 每节点本轮新增 ERROR/FATAL/PANIC 数 | +| `healthy` | 负载后四节点实际健康结果 | +| `image_id` / `source_revision` | 对应二进制镜像和源码身份 | +| `is_formal_pre` | 恒为 false：这是客户演示检查，不冒充正式 PRE | + +PASS 要求四节点均有提交、客户端零错误/零非零 RC、服务端零新增错误、完整行内容一致、固定列不变、`sum(value)` 增量等于成功提交总数、负载后健康检查通过。 +报告路径会直接打印，例如 `/var/lib/pgrac-demo/demo/reports/bench-.../`。 + +## 6. 正常停机与原数据重启 + +先停止外部业务，等待 `bench.sh` 结束，再执行： + +```bash +sudo ./pgrac-demo stop +sudo ./pgrac-demo status +sudo ./pgrac-demo up +sudo ./pgrac-demo verify +sudo ./bench.sh --clients 2 --seconds 30 +``` + +`stop` 同时请求四实例正常关闭，确认四份控制文件均已正常关闭及日志闭合后才释放本演示设备，成功输出 `CLEAN`。 +`up` 复用原数据，**不用再次执行 init-data.sh**。 +支持的是**四节点协调正常关闭/重启**；不保证随意单 Pod 重启、掉电、强杀后的恢复。 + +不要使用 `podman stop -a`、`podman rm -f`、`podman kube down --force`、`kill -9`，也不要删除 `postmaster.pid`。 +主机需要关机/重启或虚拟机需要休眠时，先确认 `stop` 成功且状态为 CLEAN。 +正常停机不成功时工具保留数据和设备，不会强杀、重置或悄悄恢复。 + +## 7. 日常检查与故障处理 + +```bash +sudo ./pgrac-demo status +sudo tail -n 100 /var/lib/pgrac-demo/demo/storage/server-log/node0.log +sudo find /var/lib/pgrac-demo/demo/reports -name summary.json -print +``` + +| 现象 | 处理 | +|---|---| +| 缺少 Podman/Python/loop-control | 按第 1 节安装、加载 loop；不要改成 privileged 容器绕过 | +| 端口已被监听 | 查 `sudo ss -ltnp`；选择空闲 `--port-base` 创建新部署，不杀其他服务 | +| 空间/内存不足 | 先正常停止本演示，扩容资源；不要删数据库文件 | +| 镜像 ID 与旧部署不符 | 使用原镜像；本包不支持原地升级 | +| INITIALIZING/STARTING/STOPPING 中断或缺节点 | 保留目录和日志，联系维护者；不把状态手工改为 CLEAN | +| 原数据非正常关闭 | 本包拒绝自动接管，不做崩溃恢复、不重建覆盖原目录 | +| 压测 FAIL | 保存 summary、四份 pgbench 日志和本轮服务器日志；TPS 不构成通过 | +| 校验需要较久 | SQL 验证上限 600 秒；超时返回失败，不会跳过校验 | + +每次安装自动生成随机 SCRAM 密码，文件权限 0600；凭据留在本机，不提交 Git、不放镜像、不输出到状态报告。 +不要公开打包整个 `storage`、bootstrap 日志或控制目录。提供报告前请检查是否含用户自行输入的 SQL/数据。 +没有自动删除/reset 命令；本包不会代替管理员清理旧场景。 + +## 8. 从源码重建镜像（可选） + +客户常规演示无需此步： + +```bash +sudo podman build --network=host -t localhost/pgrac-demo:v0.131.0-demo.1 . +sudo ./pgrac-demo --name source-demo --image localhost/pgrac-demo:v0.131.0-demo.1 up +sudo ./init-data.sh --name source-demo --rows 10000 +sudo ./bench.sh --name source-demo --clients 2 --seconds 30 +sudo ./pgrac-demo --name source-demo stop +``` + +构建固定下载公开稳定源码 `83d8c5a002643581f153058b7a766afe1ffa1eb0`，校验压缩包 SHA-256，镜像包含 PostgreSQL 版权声明。 +官方发布流程原生构建 AMD64/ARM64，并运行 `ci-smoke.sh` 的完整启动、数据、压测、正常重启链；只发布实际通过的镜像。 + +参考：[Podman kube play](https://docs.podman.io/en/latest/markdown/podman-kube-play.1.html)、[GHCR 镜像下载](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)、[数据库稳定版说明](../../docs/release-notes/v0.131.0.md)。 diff --git a/deploy/demo/bench.sh b/deploy/demo/bench.sh new file mode 100755 index 0000000000..dacb3ac878 --- /dev/null +++ b/deploy/demo/bench.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Author: SqlRush +set -euo pipefail +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec "$script_dir/pgrac-demo" "$@" bench diff --git a/deploy/demo/bootstrap.pl b/deploy/demo/bootstrap.pl new file mode 100644 index 0000000000..58d6fba0a3 --- /dev/null +++ b/deploy/demo/bootstrap.pl @@ -0,0 +1,94 @@ +#!/usr/bin/env perl +# New demo seed only; uses the frozen public initializer. +# Author: SqlRush +use strict; +use warnings; +use JSON::PP; +use Fcntl qw(O_CREAT O_EXCL O_WRONLY); +use File::Path qw(make_path); +use PostgreSQL::Test::ClusterQuad; +use Test::More; + +die "expected non-root demo account\n" unless $< == 10001; +sysopen(my $claim, '/demo/INITIALIZED', O_WRONLY | O_CREAT | O_EXCL, 0600) + or die "seed already claimed; preserve this directory\n"; +close($claim) or die $!; +open(my $pf, '<', '/demo/password') or die "password file missing\n"; +my $password = <$pf>; +close($pf); +chomp($password); +die "invalid password format\n" unless $password =~ /\A[0-9a-f]{64}\z/; +open(my $settings, '<', '/demo/settings.json') or die $!; +my $config = decode_json(do { local $/; <$settings> }); +close($settings); +my $base = $config->{port_base}; +die "invalid port base\n" unless defined $base && $base =~ /^\d+$/ && $base >= 1024 && $base <= 65000; + +my $quad; +eval { + $quad = PostgreSQL::Test::ClusterQuad->new_quad('pgrac_demo', + quorum_voting_disks => 3, shared_data => 1, shared_system_identifier => 1, + shared_system_identifier_seed_sql => + "CREATE TABLE demo_account(id integer PRIMARY KEY, value bigint NOT NULL, pad text NOT NULL) WITH (fillfactor=70); " . + "CREATE TABLE demo_smoke(id integer PRIMARY KEY, value bigint NOT NULL); " . + "ALTER ROLE pgrac PASSWORD '$password';", + extra_conf => [ + 'fsync = on', 'full_page_writes = on', 'synchronous_commit = on', + 'autovacuum = off', 'shared_buffers = 128MB', 'max_connections = 32', + 'max_parallel_workers_per_gather = 0', 'log_statement = none', + 'log_min_error_statement = panic', 'restart_after_crash = off', + 'cluster.clean_leave_enabled = on', 'cluster.read_scache = on', + 'cluster.online_join = on', 'cluster.quorum_poll_interval_ms = 2000', + 'cluster.write_fence_lease_ms = 60000', 'cluster.join_convergence_timeout_ms = 30000', + 'cluster.xid_striping = on', 'cluster.crossnode_runtime_visibility = on', + 'cluster.page_scn_shortcut = on', 'cluster.past_image = on', + 'cluster.crossnode_write_write = on', 'cluster.undo_gcs_coherence = on', + 'cluster.crossnode_cr_data_plane = on', 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.gcs_block_retransmit_max_retries = 8', + 'cluster.cssd_heartbeat_interval_ms = 2000', 'cluster.cssd_dead_deadband_factor = 10', + 'cluster.undo_buffers = 8192', 'cluster.pcm_grd_max_entries = 32768', + 'cluster.ges_dedup_max_entries = 16384', 'cluster.gcs_block_dedup_max_entries = 16384', + ]); +}; +if ($@) { + my $error = "$@"; + $error =~ s/\Q$password\E/[REDACTED]/g; + die $error; +} +make_path('/demo/server-log', '/demo/sockets', $quad->shared_data_root . '/pg_undo'); +my $peers = "[cluster]\nname = pgrac_demo\n\n"; +for my $i (0 .. 3) { + $peers .= "[node.$i]\ninterconnect_addr = 127.0.0.1:" . ($base + 100 + $i) . + "\ndata_addr = 127.0.0.1:" . ($base + 200 + 2 * $i) . "\n\n"; +} +my @nodes; +for my $i (0 .. 3) { + my $node = $quad->node($i); + my $dir = $node->data_dir; + $node->append_conf('postgresql.conf', + "port = " . ($base + $i) . "\nlisten_addresses = '127.0.0.1'\n" . + "unix_socket_directories = '/demo/sockets'\nunix_socket_permissions = 0700\n" . + "cluster_name = 'pgrac_demo_node$i'\n" . + "cluster.voting_disks = '/dev/pgrac-vote0,/dev/pgrac-vote1,/dev/pgrac-vote2'\n" . + "cluster.voting_disk_size_bytes = 525824\n"); + open(my $pc, '>', "$dir/pgrac.conf") or die $!; + print {$pc} $peers; + close($pc) or die $!; + open(my $hba, '>', "$dir/pg_hba.conf") or die $!; + print {$hba} "local all all peer\nhost all all 127.0.0.1/32 scram-sha-256\n"; + close($hba) or die $!; + push @nodes, {id => $i, pgdata => $dir, port => $base + $i, + log => "/demo/server-log/node$i.log"}; +} +open(my $pass, '>', '/demo/client.pgpass') or die $!; +print {$pass} "127.0.0.1:*:postgres:pgrac:$password\n"; +close($pass) or die $!; +chmod(0600, '/demo/client.pgpass') or die $!; +open(my $out, '>', '/demo/bootstrap.json') or die $!; +print {$out} JSON::PP->new->canonical->pretty->encode({ + nodes => \@nodes, shared_data => $quad->shared_data_root, + voting => [$quad->voting_disk_paths], source_revision => + '83d8c5a002643581f153058b7a766afe1ffa1eb0'}); +close($out) or die $!; +ok(1, 'homogeneous demo seed prepared; no four-node workload started'); +done_testing(); diff --git a/deploy/demo/ci-smoke.sh b/deploy/demo/ci-smoke.sh new file mode 100755 index 0000000000..cf9f4033d7 --- /dev/null +++ b/deploy/demo/ci-smoke.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Real native four-Pod lifecycle rehearsal. Author: SqlRush +set -euo pipefail +demo_dir=$(cd -- "$(dirname -- "$0")" && pwd) +cd -- "$demo_dir" +demo_image=${1:?provide the locally built image reference} +demo_name=${2:-ci-demo} +demo=(sudo "$demo_dir/pgrac-demo" --name "$demo_name" --image "$demo_image") +check_process_security() { + for node in 0 1 2 3; do + sudo podman exec "pgrac-$demo_name-$node-db" setpriv --no-new-privs python3 -c ' +import pathlib, re, sys +sys.path.insert(0, "/opt/pgrac/demo") +from demo_lib import security_status +statuses = [] +for path in pathlib.Path("/proc").glob("[0-9]*/status"): + try: + text = path.read_text() + except FileNotFoundError: + continue + if re.search(r"^Name:\s+postgres$", text, re.M): + statuses.append(security_status(text, int(re.search(r"^Uid:\s+(\d+)", text, re.M)[1]))) +assert statuses, "no actual PostgreSQL process checked" +print("PostgreSQL process security PASS:", len(statuses))' + done +} +python3 -m unittest discover -s "$demo_dir/tests" -v +"${demo[@]}" check +"${demo[@]}" up +"${demo[@]}" up +check_process_security +"${demo[@]}" init-data --rows 10000 +if "${demo[@]}" init-data --rows 10000; then + echo 'ERROR: repeated initialization unexpectedly accepted' >&2 + exit 1 +fi +"${demo[@]}" verify +"${demo[@]}" bench --clients 2 --seconds 30 +"${demo[@]}" stop +"${demo[@]}" up +check_process_security +"${demo[@]}" verify +"${demo[@]}" bench --clients 2 --seconds 30 +"${demo[@]}" stop +"${demo[@]}" stop +"${demo[@]}" status +echo 'Native four-Pod lifecycle rehearsal PASS (not formal PRE).' diff --git a/deploy/demo/demo_lib.py b/deploy/demo/demo_lib.py new file mode 100644 index 0000000000..284f47d05d --- /dev/null +++ b/deploy/demo/demo_lib.py @@ -0,0 +1,183 @@ +"""Pure safety and evidence policies for the single-host Podman demo. + +Author: SqlRush +""" + +import hashlib +import math +from pathlib import Path, PurePosixPath +import re +import socket + +SOURCE_REVISION = "83d8c5a002643581f153058b7a766afe1ffa1eb0" +DEMO_VERSION = "v0.131.0-demo.1" +DEFAULT_IMAGE = "ghcr.io/sqlrush/pgrac-demo:" + DEMO_VERSION +DATABASE_UID = 10001 + + +def security_status(text, uid): + fields = dict(re.findall(r"^(NoNewPrivs|CapEff|CapBnd|Seccomp):\s+(\S+)", text, re.M)) + if (uid != DATABASE_UID or fields.get("NoNewPrivs") != "1" + or fields.get("CapEff") != "0000000000000000" + or fields.get("CapBnd") != "0000000000000000" or fields.get("Seccomp") != "2"): + raise ValueError("require UID 10001, no-new-privileges, zero capabilities and seccomp") + return fields + + +def sql_startup_ready(logs): + return len(logs) == 4 and all(re.search( + r"LOG:\s+database system is ready to accept connections\s*$", text, re.M) for text in logs) + + +def check_port_free(port): + # Match PostgreSQL's SO_REUSEADDR: old TIME_WAIT is not a live listener. + with socket.socket() as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", port)) + sock.listen() + + +def removable_pod(info, name, node): + return (info.get("Name") == "pgrac-%s-%d" % (name, node) + and info.get("Labels", {}).get("pgrac-demo") == name + and info.get("State") in ("Exited", "Stopped", "Created")) + + +def validate_name(name): + if not re.fullmatch(r"[a-z][a-z0-9-]{0,31}", name): + raise ValueError("name must be 1..32 lowercase letters/digits/hyphens, starting with a letter") + return name + + +def safe_root(base, name): + validate_name(name) + path = Path(base) / name + if not path.is_absolute() or ".." in path.parts: + raise ValueError("storage root must be an absolute, non-traversing path") + for part in [path, *path.parents]: + if part.is_symlink(): + raise ValueError("symlink in storage root: " + str(part)) + return path + + +def storage_relative(path): + if not isinstance(path, str) or not path.startswith("/demo/"): + raise ValueError("path is outside the deployment storage") + parsed = PurePosixPath(path) + if str(parsed) != path or ".." in parsed.parts or len(parsed.parts) < 3: + raise ValueError("non-canonical storage path") + return Path(*parsed.parts[2:]) + + +def start_action(state, running, clean, image_matches): + if not image_matches: + raise ValueError("image differs from initialized deployment; in-place upgrade is not supported") + if state == "READY" and sorted(running) == [0, 1, 2, 3]: + return "verify" + if state == "INITIALIZED" and not running: + return "first" + if state == "CLEAN" and not running and clean == [True] * 4: + return "restart" + raise ValueError("deployment is not a complete running or proven clean cluster; preserve evidence") + + +def exact_loop(device, backing, major_minor, size, actual_device, actual_backing, + actual_major_minor, actual_size): + return ((device, backing, major_minor, size) == + (actual_device, actual_backing, actual_major_minor, actual_size)) + + +def parse_pgbench(text, rc): + values = {} + for key, pattern in ( + ("transactions", r"^number of transactions actually processed: (\d+)(?:/\d+)?\s*$"), + ("failed", r"^number of failed transactions: (\d+) \([^\n]+\)\s*$"), + ("tps", r"^tps = ([0-9.]+) \([^\n]+\)\s*$")): + matches = re.findall(pattern, text, flags=re.MULTILINE) + if len(matches) != 1: + raise ValueError("missing or ambiguous pgbench field: " + key) + values[key] = float(matches[0]) if key == "tps" else int(matches[0]) + if not math.isfinite(values["tps"]): + raise ValueError("non-finite throughput") + values["rc"] = rc + return values + + +def benchmark_verdict(nodes, before, after, server_errors, healthy): + reasons = [] + if len(nodes) != 4 or any(n["rc"] != 0 or n["failed"] != 0 or n["transactions"] <= 0 + for n in nodes): + reasons.append("CLIENT_ERROR_OR_MISSING_NODE") + if len(server_errors) != 4 or any(server_errors): + reasons.append("SERVER_ERROR_OR_MISSING_LOG") + if healthy != [True] * 4: + reasons.append("HEALTH_NOT_PROVEN") + total = sum(n["transactions"] for n in nodes) + for label, snapshots in (("BEFORE", before), ("AFTER", after)): + if len(snapshots) != 4 or any(s != snapshots[0] for s in snapshots): + reasons.append(label + "_ROW_MISMATCH") + if len(before) == 4 and len(after) == 4: + if (before[0]["rows"] <= 0 or before[0]["rows"] != after[0]["rows"] + or before[0]["key_payload_sha256"] != after[0]["key_payload_sha256"]): + reasons.append("KEY_PAYLOAD_OR_COUNT_CHANGED") + if after[0]["sum"] - before[0]["sum"] != total: + reasons.append("COMMITTED_SUM_MISMATCH") + return {"verdict": "FAIL" if reasons else "PASS", "reasons": reasons, + "transactions": total, "reported_tps_sum": sum(n["tps"] for n in nodes), + "nodes": nodes, "is_formal_pre": False} + + +def pod_documents(name, image, storage, loops): + validate_name(name) + if len(loops) != 3 or any(not re.fullmatch(r"/dev/loop\d+", d) for d in loops): + raise ValueError("exactly three Linux loop devices are required") + documents = [] + for i in range(4): + volumes = [{"name": "storage", "hostPath": {"path": storage, "type": "Directory"}}, + {"name": "shm", "emptyDir": {"medium": "Memory", "sizeLimit": "256Mi"}}] + mounts = [{"name": "storage", "mountPath": "/demo"}, + {"name": "shm", "mountPath": "/dev/shm"}] + for j, device in enumerate(loops): + volumes.append({"name": "vote" + str(j), "hostPath": { + "path": device, "type": "BlockDevice"}}) + mounts.append({"name": "vote" + str(j), "mountPath": "/dev/pgrac-vote" + str(j)}) + documents.append({"apiVersion": "v1", "kind": "Pod", "metadata": { + "name": "pgrac-" + name + "-" + str(i), + "labels": {"app": "pgrac-demo", "pgrac-demo": name}}, "spec": { + "hostNetwork": True, "restartPolicy": "Never", "terminationGracePeriodSeconds": 600, + "containers": [{"name": "db", "image": image, "imagePullPolicy": "Never", + "args": ["node", str(i)], "securityContext": { + "runAsUser": DATABASE_UID, "runAsGroup": DATABASE_UID, + # The image's setpriv entry point applies NNP after + # AppArmor admission (Ubuntu crun stub compatibility). + "capabilities": {"drop": ["ALL"]}}, + "volumeMounts": mounts}], "volumes": volumes}}) + return documents + + +def population_rows(existing, requested): + if existing != 0: + raise ValueError("demo_account already contains data; initialization never truncates/replaces it") + if not isinstance(requested, int) or not 1000 <= requested <= 1000000: + raise ValueError("rows must be in 1000..1000000") + return requested + + +def row_snapshot(stream): + full = hashlib.sha256() + immutable = hashlib.sha256() + rows = total = previous = 0 + for line in stream: + columns = line.rstrip(b"\n").split(b"\t") + if len(columns) != 3: + raise ValueError("unexpected COPY row shape") + key, value = int(columns[0]), int(columns[1]) + if key <= previous or value < 0: + raise ValueError("duplicate/unsorted key or negative counter") + previous = key + rows += 1 + total += value + full.update(line) + immutable.update(columns[0]+b"\t"+columns[2]+b"\n") + return {"rows": rows, "sum": total, "sha256": full.hexdigest(), + "key_payload_sha256": immutable.hexdigest()} diff --git a/deploy/demo/entrypoint.py b/deploy/demo/entrypoint.py new file mode 100644 index 0000000000..460e3af08f --- /dev/null +++ b/deploy/demo/entrypoint.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Non-root image entry point. Author: SqlRush """ + +import json +import os +from pathlib import Path +import signal +import socket +import subprocess +import sys + +from demo_lib import security_status, storage_relative + + +def main(): + security = security_status(Path("/proc/self/status").read_text(), os.getuid()) + if sys.argv[1:] == ["runtime-check"]: + # No data/device mount: prove kernel security and real TCP admission first. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) as sock: + sock.bind(("127.0.0.1", 0)) + sock.listen() + print(json.dumps({"runtime_check": "PASS", "uid": os.getuid(), "security": security})) + return 0 + if sys.argv[1:] == ["bootstrap"]: + os.execvp("perl", ["perl", "/opt/pgrac/demo/bootstrap.pl"]) + if len(sys.argv) != 3 or sys.argv[1] != "node" or sys.argv[2] not in ("0", "1", "2", "3"): + raise ValueError("use runtime-check, bootstrap or node 0..3") + node = json.loads(Path("/demo/bootstrap.json").read_text())["nodes"][int(sys.argv[2])] + storage_relative(node["pgdata"]) + storage_relative(node["log"]) + with open(node["log"], "ab", buffering=0) as log: + process = subprocess.Popen(["postgres", "-D", node["pgdata"]], stdout=log, stderr=log) + + def stop(_sig, _frame): + # SIGINT is PostgreSQL's normal fast shutdown, not immediate/kill. + if process.poll() is None: + process.send_signal(signal.SIGINT) + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + return process.wait() + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (ValueError, OSError, KeyError) as error: + print("demo entry point refused: " + str(error), file=sys.stderr) + sys.exit(1) diff --git a/deploy/demo/init-data.sh b/deploy/demo/init-data.sh new file mode 100755 index 0000000000..22c9bf3575 --- /dev/null +++ b/deploy/demo/init-data.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Author: SqlRush +set -euo pipefail +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec "$script_dir/pgrac-demo" "$@" init-data diff --git a/deploy/demo/pgrac-demo b/deploy/demo/pgrac-demo new file mode 100755 index 0000000000..19564aa7b3 --- /dev/null +++ b/deploy/demo/pgrac-demo @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +"""Single-host four-Pod demo lifecycle. Author: SqlRush """ + +import argparse +import fcntl +import json +import os +from pathlib import Path +import re +import secrets +import shutil +import stat +import subprocess +import sys +import tempfile +import time + +from demo_lib import (DATABASE_UID, DEFAULT_IMAGE, DEMO_VERSION, SOURCE_REVISION, + check_port_free, exact_loop, pod_documents, removable_pod, + safe_root, sql_startup_ready, start_action, storage_relative) + +BASE = Path("/var/lib/pgrac-demo") +HERE = Path(__file__).resolve().parent + + +def run(args, *, check=True, data=None, stdout=None, timeout=180, cwd=None): + result = subprocess.run(args, input=data, text=True, stdout=stdout or subprocess.PIPE, + stderr=subprocess.PIPE, timeout=timeout, cwd=cwd) + if check and result.returncode: + raise RuntimeError("command failed: " + args[0] + " " + " ".join(args[1:3]) + + "\n" + result.stderr[-3000:]) + return result + + +def save_json(path, value): + with tempfile.NamedTemporaryFile(mode="w", dir=path.parent, prefix=".manifest-", + delete=False) as output: + json.dump(value, output, indent=2) + output.write("\n") + output.flush() + os.fsync(output.fileno()) + temporary = output.name + os.replace(temporary, path) + fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +class Demo: + def __init__(self, args): + self.args = args + self.root = safe_root(BASE, args.name) + self.storage = self.root / "storage" + self.control = self.root / "control.json" + self.state = json.loads(self.control.read_text()) if self.control.exists() else None + if self.state: + if self.control.is_symlink() or self.control.stat().st_uid != 0: + raise ValueError("controller manifest must be a root-owned regular file") + if self.state["name"] != args.name or self.state["version"] != DEMO_VERSION: + raise ValueError("deployment identity/version mismatch") + current = self.storage.stat() + if [current.st_dev, current.st_ino] != self.state["storage_identity"]: + raise ValueError("storage directory identity changed") + + def save(self): + save_json(self.control, self.state) + + def host_path(self, path): + result = self.storage / storage_relative(path) + for part in [result, *result.parents]: + if part == self.root: + break + if part.is_symlink(): + raise ValueError("symlink in managed storage path") + return result + + def container(self, node): + return "pgrac-" + self.args.name + "-" + str(node) + "-db" + + def client_command(self, node, command, options=()): + return ["podman", "exec", *options, self.container(node), + "setpriv", "--no-new-privs", *command] + + def image(self): + image = self.args.image or (self.state["image"] if self.state else DEFAULT_IMAGE) + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:/@-]+", image): + raise ValueError("invalid image reference") + if run(["podman", "image", "exists", image], check=False).returncode: + run(["podman", "pull", image], stdout=sys.stdout, timeout=900) + metadata = json.loads(run(["podman", "image", "inspect", image]).stdout)[0] + labels = metadata.get("Labels") or metadata.get("Config", {}).get("Labels") or {} + if (labels.get("org.opencontainers.image.revision") != SOURCE_REVISION or + labels.get("org.opencontainers.image.version") != DEMO_VERSION): + raise ValueError("image is not this frozen demo/source version") + return image, metadata["Id"] + + def preflight(self, starting=False): + if sys.platform != "linux" or os.geteuid() != 0: + raise ValueError("use sudo on a Linux host (rootful Podman; databases remain non-root)") + for name in ("podman", "losetup", "blockdev", "findmnt"): + if not shutil.which(name): + raise ValueError("missing dependency: " + name) + version = run(["podman", "--version"]).stdout.strip() + if not Path("/dev/loop-control").exists(): + raise ValueError("/dev/loop-control missing; load the Linux loop module first") + parent = self.storage if self.storage.exists() else BASE + fs = run(["findmnt", "-n", "-o", "FSTYPE", "-T", str(parent)]).stdout.strip() + if fs not in ("ext4", "xfs", "btrfs"): + raise ValueError("local ext4/XFS/Btrfs required; rejected filesystem: " + fs) + free = shutil.disk_usage(parent).free + mem = dict(re.findall(r"^(\w+):\s+(\d+) kB", Path("/proc/meminfo").read_text(), re.M)) + if free < 10 * 1024**3 or int(mem["MemAvailable"]) < 4 * 1024**2: + raise ValueError("need at least 10 GiB free disk and 4 GiB available memory") + if starting: + base = self.state["port_base"] if self.state else self.args.port_base + for port in [*(base+i for i in range(4)), *(base+100+i for i in range(4)), + *(base+200+i for i in range(8))]: + check_port_free(port) + print(json.dumps({"check": "PASS", "runtime": version, "architecture": os.uname().machine, + "filesystem": fs, "free_disk_GiB": round(free/1024**3, 1), + "available_memory_GiB": round(int(mem["MemAvailable"])/1024**2, 1)})) + + def initialize(self, image, image_id): + if self.root.exists(): + raise ValueError("directory already exists without a valid deployment; not overwriting") + self.root.mkdir(mode=0o700) + self.storage.mkdir(mode=0o700) + os.chown(self.storage, DATABASE_UID, DATABASE_UID) + identity = self.storage.stat() + self.state = {"name": self.args.name, "version": DEMO_VERSION, "source_revision": SOURCE_REVISION, + "image": image, "image_id": image_id, "phase": "INITIALIZING", "loops": [], + "port_base": self.args.port_base, "storage_identity": [identity.st_dev, identity.st_ino]} + self.save() + for dirname in ("tap", "setup-log"): + path = self.storage / dirname + path.mkdir(mode=0o700) + os.chown(path, DATABASE_UID, DATABASE_UID) + files = {"password": secrets.token_hex(32) + "\n", + "settings.json": json.dumps({"port_base": self.args.port_base}), + "seed.conf": "fsync=on\nfull_page_writes=on\nsynchronous_commit=on\n" + "log_statement=none\nlog_min_error_statement=panic\nlisten_addresses=''\n"} + for name, content in files.items(): + path = self.storage / name + with path.open("x") as output: + output.write(content) + path.chmod(0o600) + os.chown(path, DATABASE_UID, DATABASE_UID) + with (self.root / "bootstrap.log").open("x") as log: + run(["podman", "run", "--rm", "--network=host", "--user=10001:10001", "--cap-drop=ALL", + "--volume", str(self.storage)+":/demo:rw,z", image, "bootstrap"], + stdout=log, timeout=900) + fixture = json.loads((self.storage / "bootstrap.json").read_text()) + if fixture["source_revision"] != SOURCE_REVISION or len(fixture["nodes"]) != 4: + raise ValueError("invalid bootstrap identity") + for node in fixture["nodes"]: + self.host_path(node["pgdata"]) + self.host_path(node["log"]) + self.host_path(fixture["shared_data"]) + self.state["fixture"] = fixture + voting = [] + for path in fixture["voting"]: + info = self.host_path(path).stat() + if not stat.S_ISREG(info.st_mode) or info.st_size != 525824: + raise ValueError("invalid voting backing") + voting.append({"path": path, "identity": [info.st_dev, info.st_ino], "size": info.st_size}) + if len(voting) != 3: + raise ValueError("three voting backings required") + self.state["voting"] = voting + self.state["phase"] = "INITIALIZED" + self.save() + + def loop_info(self, device): + rows = json.loads(run(["losetup", "--json", "--output", "NAME,BACK-FILE,DIO", device]).stdout)["loopdevices"] + if len(rows) != 1 or rows[0]["dio"] not in (1, True, "1"): + raise ValueError("loop device lacks exact mapping/direct I/O") + info = Path(device).stat() + if not stat.S_ISBLK(info.st_mode): + raise ValueError("voting authority is not a block device") + return {"device": device, "backing": rows[0]["back-file"], + "major_minor": str(os.major(info.st_rdev))+":"+str(os.minor(info.st_rdev)), + "size": int(run(["blockdev", "--getsize64", device]).stdout)} + + def check_loop(self, record): + actual = self.loop_info(record["device"]) + if not exact_loop(*(record[k] for k in ("device", "backing", "major_minor", "size")), + *(actual[k] for k in ("device", "backing", "major_minor", "size"))): + raise ValueError("voting loop identity drift; refusing to use or detach it") + + def attach(self): + if self.state["loops"]: + for record in self.state["loops"]: + self.check_loop(record) + if len(self.state["loops"]) != 3: + raise ValueError("partial prior loop attach; preserve evidence") + return + for voting in self.state["voting"]: + path = self.host_path(voting["path"]) + info = path.stat() + if [info.st_dev, info.st_ino] != voting["identity"] or info.st_size != voting["size"]: + raise ValueError("voting backing identity drift") + if run(["losetup", "-j", str(path)]).stdout.strip(): + raise ValueError("backing already mapped outside this controller") + device = run(["losetup", "--find", "--show", "--direct-io=on", str(path)]).stdout.strip() + if not re.fullmatch(r"/dev/loop\d+", device): + raise ValueError("unexpected loop device path") + record = self.loop_info(device) + if record["backing"] != str(path) or record["size"] != voting["size"]: + raise ValueError("new loop does not map exact backing") + previous = Path(device).stat() + record["permissions"] = [previous.st_uid, previous.st_gid, stat.S_IMODE(previous.st_mode)] + self.state["loops"].append(record) + self.save() + os.chown(device, DATABASE_UID, DATABASE_UID) + os.chmod(device, 0o600) + + def running(self): + active = [] + for i in range(4): + result = run(["podman", "container", "inspect", self.container(i)], check=False) + if result.returncode == 0: + info = json.loads(result.stdout)[0] + if info["Config"]["Labels"].get("pgrac-demo") != self.args.name: + raise ValueError("container name belongs to another deployment") + if info["State"]["Running"]: + active.append(i) + return active + + def clean_controls(self): + clean = [] + for node in self.state["fixture"]["nodes"]: + result = run(["podman", "run", "--rm", "--network=none", "--user=10001:10001", "--cap-drop=ALL", + "--volume", str(self.storage)+":/demo:ro,z", "--entrypoint", "setpriv", + self.state["image"], "--no-new-privs", "pg_controldata", node["pgdata"]]) + clean.append(bool(re.search(r"^Database cluster state:\s+shut down\s*$", result.stdout, re.M))) + return clean + + def sql(self, node, query, check=True, stdout=None, timeout=600): + return run(self.client_command(node, ["psql", "-XAtq", "-v", "ON_ERROR_STOP=1", + "-h", "127.0.0.1", "-p", str(self.state["port_base"]+node), + "-U", "pgrac", "postgres"], ["-i", "-e", "PGOPTIONS=-c statement_timeout=600000"]), + data=query, check=check, stdout=stdout, timeout=timeout) + + def healthy(self, node): + query = """SELECT (SELECT in_quorum FROM pg_cluster_quorum_state) + AND (SELECT count(*)=4 FROM pg_cluster_membership WHERE node_id BETWEEN 0 AND 3 + AND state='member' AND NOT removed AND presented_incarnation=last_admitted_incarnation + AND presented_incarnation>0) + AND (SELECT count(*)=3 FROM pg_cluster_state WHERE category='pcm' AND + ((key='resource_x_writer_path' AND value='target') OR (key='resource_x_gate_phase' AND value='open') + OR (key='resource_x_proof_readiness' AND value='AVAILABLE_PROOF_KIND')));""" + result = self.sql(node, query, check=False, timeout=20) + return result.returncode == 0 and result.stdout.strip() == "t" + + def up(self): + image, image_id = self.image() + run(["podman", "run", "--rm", "--network=host", "--user=10001:10001", "--cap-drop=ALL", + image, "runtime-check"], stdout=sys.stdout) + if self.state is None: + self.preflight(starting=True) + self.initialize(image, image_id) + action = start_action(self.state["phase"], self.running(), + self.clean_controls() if self.state["phase"] == "CLEAN" else [], + image_id == self.state["image_id"]) + if action == "verify": + if not all(self.healthy(i) for i in range(4)): + raise ValueError("running deployment is not healthy") + self.status() + return + self.preflight(starting=True) + self.attach() + # Remove only stopped, identity-checked Pods. Never use force/replace. + for i in range(4): + pod = "pgrac-"+self.args.name+"-"+str(i) + if run(["podman", "pod", "exists", pod], check=False).returncode == 0: + # Unlike legacy `pod inspect`, this is an array on Podman 4 and 5. + info = json.loads(run(["podman", "inspect", "--type=pod", pod]).stdout)[0] + if not removable_pod(info, self.args.name, i): + raise ValueError("Pod is not an owned stopped deployment; refusing to remove it") + run(["podman", "pod", "rm", pod]) + documents = pod_documents(self.args.name, image, str(self.storage), + [r["device"] for r in self.state["loops"]]) + manifest = self.root / "pods.yaml" + manifest.write_text("\n---\n".join(json.dumps(p, indent=2) for p in documents)+"\n") + self.state["phase"] = "STARTING" + self.state["start_log_offsets"] = self.log_offsets() + self.save() + # Podman probes cwd//Containerfile even with --build=false. + # Never run in the downloaded script directory containing pgrac-demo. + run(["podman", "kube", "play", "--build=false", str(manifest)], + stdout=sys.stdout, cwd=self.root) + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + # A connection during startup itself generates a FATAL rejection. + # Wait for this launch's ready messages before actively probing SQL. + logs_exist = all(self.host_path(n["log"]).is_file() for n in self.state["fixture"]["nodes"]) + if (logs_exist and sql_startup_ready(self.new_logs(self.state["start_log_offsets"])) + and all(self.sql(i, "SELECT 1;", check=False, timeout=10).returncode == 0 for i in range(4))): + break + time.sleep(0.5) + else: + raise ValueError("four-node SQL startup incomplete; preserve data and logs") + if action == "first": + for i in range(4): + self.activate(i) + for _ in range(2): + self.activate(0, require_success=True) + while time.monotonic() < deadline: + if all(self.healthy(i) for i in range(4)): + break + time.sleep(0.5) + else: + raise ValueError("cluster target/open readiness not proven") + if action == "restart": + for i, text in enumerate(self.new_logs(self.state["start_log_offsets"])): + if not re.search(r"database system was shut down at", text) or re.search( + r"automatic recovery in progress|redo starts at|PANIC:|FATAL:", text): + raise ValueError("node%d did not demonstrate normal startup" % i) + self.state["phase"] = "READY" + self.save() + self.status() + + def activate(self, node, require_success=False): + deadline = time.monotonic()+120 + while time.monotonic() < deadline: + result = self.sql(node, "ALTER SYSTEM ENABLE RAC TWO_STAGE ROLLING UPDATES ALL;", + check=False, timeout=50) + if result.returncode == 0: + return + if not re.search(r"RF_DEFERRED|CONDITION_NOT_YET_MET|activation request was refused", result.stderr): + raise RuntimeError("activation rejected: "+result.stderr) + if not require_success: + return + time.sleep(0.2) + raise ValueError("initial activation did not complete") + + def log_offsets(self): + return [self.host_path(n["log"]).stat().st_size if self.host_path(n["log"]).exists() else 0 + for n in self.state["fixture"]["nodes"]] + + def new_logs(self, offsets): + result = [] + for node, offset in zip(self.state["fixture"]["nodes"], offsets): + with self.host_path(node["log"]).open("rb") as log: + if log.seek(0, 2) < offset: + raise ValueError("server log truncated during verification") + log.seek(offset) + result.append(log.read().decode("utf-8", errors="replace")) + return result + + def stop(self): + if self.state["phase"] == "CLEAN" and not self.running() and self.clean_controls() == [True]*4: + print("Already CLEAN; data retained.") + return + if self.state["phase"] != "READY" or self.running() != [0, 1, 2, 3]: + raise ValueError("not a complete ready deployment; no forced cleanup or crash recovery") + offsets = self.log_offsets() + self.state["phase"] = "STOPPING" + self.save() + for node in self.state["fixture"]["nodes"]: + run(self.client_command(node["id"], ["pg_ctl", "-D", node["pgdata"], + "-m", "fast", "-W", "stop"])) + deadline = time.monotonic()+600 + while self.running(): + if time.monotonic() >= deadline: + raise ValueError("normal shutdown incomplete; no force kill, data/devices retained") + time.sleep(0.5) + if self.clean_controls() != [True]*4: + raise ValueError("not all four durable controls are shut down") + for node, text in zip(self.state["fixture"]["nodes"], self.new_logs(offsets)): + if (self.host_path(node["pgdata"]+"/postmaster.pid").exists() or + "cluster normal-stop: protocol closed after shutdown checkpoint" not in text or + "database system is shut down" not in text or re.search(r"PANIC:|FATAL:", text)): + raise ValueError("normal-stop evidence incomplete; retain devices") + for record in reversed(self.state["loops"]): + self.check_loop(record) + uid, gid, mode = record["permissions"] + # Restore before detach: after detach this device number can be reused. + os.chown(record["device"], uid, gid) + os.chmod(record["device"], mode) + run(["losetup", "--detach", record["device"]]) + self.state["loops"] = [] + self.state["phase"] = "CLEAN" + self.save() + print("CLEAN: all four nodes shut down normally; original data retained.") + + def status(self): + print(json.dumps({"deployment": self.args.name, "phase": self.state["phase"], + "running_nodes": self.running(), "image": self.state["image"], + "source_revision": SOURCE_REVISION, "root": str(self.root), + "sql_ports": [self.state["port_base"]+i for i in range(4)]}, indent=2)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--name", default="demo") + parser.add_argument("--image", help="prebuilt image reference; defaults to frozen public demo") + parser.add_argument("--port-base", type=int, default=15432) + parser.add_argument("command", choices=("check", "pull", "up", "status", "stop", "verify", "init-data", "bench", "sql")) + parser.add_argument("--rows", type=int, default=10000) + parser.add_argument("--clients", type=int, default=2, help="connections per node") + parser.add_argument("--seconds", type=int, default=30) + parser.add_argument("--node", type=int, choices=range(4), default=0) + args = parser.parse_args() + if sys.platform != "linux" or os.geteuid() != 0: + parser.error("run with sudo on Linux; database processes run as non-root") + if not 1024 <= args.port_base <= 65000: + parser.error("port base must be in 1024..65000") + safe_root(BASE, args.name) + BASE.mkdir(mode=0o755, exist_ok=True) + if BASE.stat().st_uid != 0 or BASE.stat().st_mode & 0o022: + raise ValueError("deployment base must be root-owned and not group/world writable") + lock_path = BASE / ("."+args.name+".lock") + fd = os.open(lock_path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + demo = Demo(args) + if args.command == "check": + demo.preflight() + elif args.command == "pull": + print(json.dumps(dict(zip(("image", "image_id"), demo.image())))) + elif args.command == "up": + demo.up() + else: + if demo.state is None: + raise ValueError("deployment does not exist; run up first") + if args.command in ("status", "stop"): + getattr(demo, args.command)() + elif args.command == "sql": + result = demo.sql(args.node, sys.stdin.read(), stdout=sys.stdout) + return result.returncode + else: + from workload import execute + execute(demo, args) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (ValueError, RuntimeError, OSError, subprocess.TimeoutExpired, KeyError) as error: + print("REFUSED/FAILED: "+str(error), file=sys.stderr) + sys.exit(1) diff --git a/deploy/demo/release_gate.py b/deploy/demo/release_gate.py new file mode 100644 index 0000000000..8fae2cc345 --- /dev/null +++ b/deploy/demo/release_gate.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Publish only after protected CI succeeds. Author: SqlRush """ + +import json +from pathlib import Path +import sys + +REQUIRED = ( + "Validate (comment headers + format + tidy + commit msg)", + "Linux enable-cluster (unit + regress + smoke TAP)", + "Linux --disable-cluster regression", + "macOS build-only (compile guard)", + "Security (cppcheck + scan-build)", +) + + +def failures(checks): + latest = {} + for check in sorted(checks, key=lambda item: item.get("started_at") or ""): + latest[check["name"]] = check.get("conclusion") + return [name for name in REQUIRED if latest.get(name) != "success"] + + +if __name__ == "__main__": + pending = failures([json.loads(line) for line in Path(sys.argv[1]).read_text().splitlines()]) + if pending: + sys.exit("Image publication blocked by: " + ", ".join(pending)) + print("All protected checks succeeded on the queried commit.") diff --git a/deploy/demo/sql/update.sql b/deploy/demo/sql/update.sql new file mode 100644 index 0000000000..0763ad9c91 --- /dev/null +++ b/deploy/demo/sql/update.sql @@ -0,0 +1,5 @@ +-- One committed increment per transaction. Author: SqlRush +\set id random(1, :rows) +BEGIN; +UPDATE demo_account SET value=value+1 WHERE id=:id; +COMMIT; diff --git a/deploy/demo/tests/test_controller.py b/deploy/demo/tests/test_controller.py new file mode 100644 index 0000000000..f90dd2e510 --- /dev/null +++ b/deploy/demo/tests/test_controller.py @@ -0,0 +1,72 @@ +"""Scoped teardown call-order checks. Author: SqlRush """ + +import importlib.machinery +import importlib.util +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +loader = importlib.machinery.SourceFileLoader("demo_controller", str(Path(__file__).resolve().parents[1] / "pgrac-demo")) +spec = importlib.util.spec_from_loader(loader.name, loader) +controller = importlib.util.module_from_spec(spec) +loader.exec_module(controller) + + +class Teardown(unittest.TestCase): + def test_runtime_command_uses_explicit_owned_working_directory(self): + with patch.object(controller.subprocess, "run", return_value=Mock(returncode=0)) as child: + controller.run(["podman", "kube", "play", "--build=false", "/owned/pods.yaml"], + cwd=Path("/owned")) + self.assertEqual(child.call_args.kwargs["cwd"], Path("/owned")) + + def test_clients_set_no_new_privileges_before_exec(self): + demo = object.__new__(controller.Demo) + demo.args = Mock() + demo.args.name = "demo" + self.assertEqual(demo.client_command(2, ["psql", "-X"], ["-i"]), + ["podman", "exec", "-i", "pgrac-demo-2-db", + "setpriv", "--no-new-privs", "psql", "-X"]) + + def deployment(self, root): + demo = object.__new__(controller.Demo) + demo.args = Mock(name="demo") + demo.args.name = "demo" + demo.state = {"phase": "READY", "fixture": {"nodes": [ + {"id": i, "pgdata": "/demo/node%d" % i} for i in range(4)]}, + "loops": [{"device": "/dev/loop7", "permissions": [0, 6, 0o660]}]} + demo.log_offsets = Mock(return_value=[0]*4) + demo.running = Mock(side_effect=[[0, 1, 2, 3], []]) + demo.clean_controls = Mock(return_value=[True]*4) + demo.new_logs = Mock(return_value=["cluster normal-stop: protocol closed after shutdown checkpoint\n" + "database system is shut down\n"]*4) + demo.host_path = lambda _path: root / "absent.pid" + demo.save = Mock() + return demo + + def test_permissions_restore_precedes_detach_after_clean_shutdown(self): + with tempfile.TemporaryDirectory() as temporary: + demo = self.deployment(Path(temporary)) + events = [] + demo.check_loop = Mock(side_effect=lambda record: events.append("identity")) + with patch.object(controller, "run", side_effect=lambda args: events.append(args)), \ + patch.object(controller.os, "chown", side_effect=lambda *args: events.append(("chown", args))), \ + patch.object(controller.os, "chmod", side_effect=lambda *args: events.append(("chmod", args))): + demo.stop() + self.assertEqual(events[-4:], ["identity", ("chown", ("/dev/loop7", 0, 6)), + ("chmod", ("/dev/loop7", 0o660)), + ["losetup", "--detach", "/dev/loop7"]]) + self.assertEqual(demo.state["phase"], "CLEAN") + + def test_identity_drift_prevents_permission_change_and_detach(self): + with tempfile.TemporaryDirectory() as temporary: + demo = self.deployment(Path(temporary)) + demo.check_loop = Mock(side_effect=ValueError("wrong backing")) + with patch.object(controller, "run") as run, patch.object(controller.os, "chown") as chown: + with self.assertRaises(ValueError): + demo.stop() + chown.assert_not_called() + self.assertFalse(any(call.args[0][0] == "losetup" for call in run.call_args_list)) + self.assertEqual(demo.state["phase"], "STOPPING") diff --git a/deploy/demo/tests/test_demo.py b/deploy/demo/tests/test_demo.py new file mode 100644 index 0000000000..8a774d8c08 --- /dev/null +++ b/deploy/demo/tests/test_demo.py @@ -0,0 +1,214 @@ +"""Behavior tests for customer demo safety and evidence. + +Author: SqlRush +""" + +import importlib.util +import io +import json +import socket +from pathlib import Path +import sys +import tempfile +import unittest + +MODULE = Path(__file__).resolve().parents[1] / "demo_lib.py" +core = None +if MODULE.exists(): + spec = importlib.util.spec_from_file_location("demo_lib", MODULE) + core = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = core + spec.loader.exec_module(core) + + +class DemoPolicies(unittest.TestCase): + def setUp(self): + self.assertIsNotNone(core, "demo safety policy is not implemented") + + def test_names_cannot_escape_storage_or_become_options(self): + for name in ("../data", "", "-rf", "a/b", "a b", "x;id", "X", "a" * 33): + with self.subTest(name=name), self.assertRaises(ValueError): + core.validate_name(name) + self.assertEqual(core.validate_name("customer-01"), "customer-01") + + def test_symlink_parent_or_target_is_rejected_without_touching_it(self): + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + real = base / "real" + real.mkdir() + link = base / "link" + link.symlink_to(real, target_is_directory=True) + with self.assertRaises(ValueError): + core.safe_root(link, "demo") + (real / "demo").symlink_to(base, target_is_directory=True) + with self.assertRaises(ValueError): + core.safe_root(real, "demo") + self.assertTrue((real / "demo").is_symlink()) + + def test_container_paths_cannot_escape_owned_storage(self): + for path in ("/etc/passwd", "/demo/../other", "/demo", "/demo/a//b"): + with self.assertRaises(ValueError): + core.storage_relative(path) + self.assertEqual(core.storage_relative("/demo/tap/node/pgdata"), + Path("tap/node/pgdata")) + + def test_repeat_up_does_not_initialize_or_restart(self): + self.assertEqual(core.start_action("READY", [0, 1, 2, 3], [], True), "verify") + + def test_only_clean_four_member_shutdown_allows_restart(self): + self.assertEqual(core.start_action("CLEAN", [], [True] * 4, True), "restart") + for state, running, clean in (("CLEAN", [], [True, True, False, True]), + ("READY", [0, 1, 2], []), + ("STARTING", [], [True] * 4), + ("INITIALIZING", [], []), + ("CLEAN", [1], [True] * 4)): + with self.subTest(state=state), self.assertRaises(ValueError): + core.start_action(state, running, clean, True) + + def test_new_image_cannot_silently_take_over_old_data(self): + with self.assertRaises(ValueError): + core.start_action("CLEAN", [], [True] * 4, False) + + def test_first_start_is_distinct_from_restart(self): + self.assertEqual(core.start_action("INITIALIZED", [], [], True), "first") + + def test_reused_loop_device_is_not_owned(self): + self.assertTrue(core.exact_loop("/dev/loop7", "/owned/vote0", "7:7", 525824, + "/dev/loop7", "/owned/vote0", "7:7", 525824)) + self.assertFalse(core.exact_loop("/dev/loop7", "/owned/vote0", "7:7", 525824, + "/dev/loop7", "/other/vote0", "7:7", 525824)) + + def test_pgbench_output_preserves_failed_transactions_and_rc(self): + text = ("number of transactions actually processed: 123\n" + "number of failed transactions: 2 (1.6%)\n" + "tps = 12.300000 (without initial connection time)\n") + self.assertEqual(core.parse_pgbench(text, 3), + {"transactions": 123, "failed": 2, "tps": 12.3, "rc": 3}) + + def test_missing_or_duplicate_benchmark_fields_are_not_zero_errors(self): + for text in ("", "tps = 5.0\n", "number of transactions actually processed: 12\n", + self.output() + self.output()): + with self.assertRaises(ValueError): + core.parse_pgbench(text, 0) + + @staticmethod + def output(): + return ("number of transactions actually processed: 10\n" + "number of failed transactions: 0 (0.0%)\n" + "tps = 1.000000 (without initial connection time)\n") + + @staticmethod + def snapshots(total=0, digest="aaa"): + return [{"rows": 100, "sum": total, "sha256": digest, + "key_payload_sha256": "bbb"} for _ in range(4)] + + def test_benchmark_pass_requires_actual_committed_sum_and_full_rows(self): + nodes = [core.parse_pgbench(self.output(), 0) for _ in range(4)] + result = core.benchmark_verdict(nodes, self.snapshots(), self.snapshots(40, "ccc"), + [0] * 4, [True] * 4) + self.assertEqual(result["verdict"], "PASS") + self.assertEqual(result["transactions"], 40) + self.assertEqual(result["reported_tps_sum"], 4.0) + self.assertFalse(result["is_formal_pre"]) + + def test_errors_bad_sums_missing_nodes_and_server_errors_fail(self): + nodes = [core.parse_pgbench(self.output(), 0) for _ in range(4)] + cases = [([dict(nodes[0], rc=2)] + nodes[1:], self.snapshots(40, "ccc"), [0]*4), + ([dict(nodes[0], failed=1)] + nodes[1:], self.snapshots(40, "ccc"), [0]*4), + (nodes, self.snapshots(39, "ccc"), [0]*4), + (nodes[:3], self.snapshots(40, "ccc"), [0]*4), + (nodes, self.snapshots(40, "ccc"), [0, 1, 0, 0])] + for clients, after, errors in cases: + with self.subTest(clients=clients, errors=errors): + result = core.benchmark_verdict(clients, self.snapshots(), after, + errors, [True] * 4) + self.assertEqual(result["verdict"], "FAIL") + self.assertTrue(result["reasons"]) + + def test_equal_totals_do_not_hide_cross_node_different_rows(self): + nodes = [core.parse_pgbench(self.output(), 0) for _ in range(4)] + after = self.snapshots(40, "ccc") + after[3]["sha256"] = "ddd" + self.assertEqual(core.benchmark_verdict(nodes, self.snapshots(), after, + [0]*4, [True]*4)["verdict"], "FAIL") + + def test_four_pods_have_no_automatic_restart_or_privileged_database(self): + manifests = core.pod_documents("demo", "localhost/pgrac:test", "/owned/storage", + ["/dev/loop7", "/dev/loop8", "/dev/loop9"]) + self.assertEqual(len(manifests), 4) + self.assertEqual(len({p["metadata"]["name"] for p in manifests}), 4) + for i, pod in enumerate(manifests): + spec = pod["spec"] + self.assertEqual(spec["restartPolicy"], "Never") + db = spec["containers"][0] + self.assertEqual(db["args"], ["node", str(i)]) + self.assertEqual(db["securityContext"]["runAsUser"], 10001) + self.assertFalse(db["securityContext"].get("privileged", False)) + # setpriv applies NNP after the container AppArmor transition. + # Runtime-applied NNP stacks Ubuntu's crun stub onto this profile. + self.assertNotIn("allowPrivilegeEscalation", db["securityContext"]) + devices = [v["hostPath"]["path"] for v in spec["volumes"] + if v.get("hostPath", {}).get("type") == "BlockDevice"] + self.assertEqual(devices, ["/dev/loop7", "/dev/loop8", "/dev/loop9"]) + self.assertEqual(json.loads(json.dumps(pod)), pod) + + def test_entrypoint_requires_actual_kernel_security_not_a_manifest_claim(self): + status = "NoNewPrivs:\t1\nCapEff:\t0000000000000000\nCapBnd:\t0000000000000000\nSeccomp:\t2\n" + self.assertEqual(core.security_status(status, 10001)["NoNewPrivs"], "1") + for invalid in (status.replace("NoNewPrivs:\t1", "NoNewPrivs:\t0"), + status.replace("CapEff:\t0000000000000000", "CapEff:\t0000000000000001"), + status.replace("CapBnd:\t0000000000000000", "CapBnd:\t0000000000000001"), + status.replace("Seccomp:\t2", "Seccomp:\t0"), ""): + with self.assertRaises(ValueError): + core.security_status(invalid, 10001) + with self.assertRaises(ValueError): + core.security_status(status, 0) + + def test_sql_probe_waits_for_all_four_current_startup_logs(self): + ready = "2026-09-20 UTC [2] LOG: database system is ready to accept connections\n" + self.assertTrue(core.sql_startup_ready([ready]*4)) + self.assertFalse(core.sql_startup_ready([ready]*3)) + self.assertFalse(core.sql_startup_ready([ready]*3+["listening on IPv4 address\n"])) + self.assertFalse(core.sql_startup_ready([""]*4)) + + def test_population_refuses_existing_data_and_unbounded_geometry(self): + self.assertTrue(callable(getattr(core, "population_rows", None)), "population guard missing") + self.assertEqual(core.population_rows(0, 10000), 10000) + for count, rows in ((1, 10000), (0, 0), (0, -1), (0, 1000001)): + with self.assertRaises(ValueError): + core.population_rows(count, rows) + + def test_row_snapshot_reads_all_rows_and_rejects_duplicate_keys(self): + self.assertTrue(callable(getattr(core, "row_snapshot", None)), "full-row verifier missing") + value = core.row_snapshot(io.BytesIO(b"1\t2\tx\n2\t4\ty\n")) + self.assertEqual((value["rows"], value["sum"]), (2, 6)) + for data in (b"1\t2\tx\n1\t4\ty\n", b"2\t3\tx\n1\t4\ty\n", b"1\tx\ty\n"): + with self.assertRaises(ValueError): + core.row_snapshot(io.BytesIO(data)) + + def test_port_check_allows_time_wait_but_rejects_a_listener(self): + self.assertTrue(callable(getattr(core, "check_port_free", None)), "port check missing") + with socket.socket() as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + port = server.getsockname()[1] + server.listen() + with self.assertRaises(OSError): + core.check_port_free(port) + with socket.create_connection(("127.0.0.1", port)) as client: + accepted, _ = server.accept() + accepted.close() + self.assertEqual(client.recv(1), b"") + core.check_port_free(port) + + def test_foreign_or_running_pod_is_not_removable(self): + self.assertTrue(callable(getattr(core, "removable_pod", None)), "pod owner guard missing") + owned = {"Name": "pgrac-demo-0", "State": "Exited", "Labels": {"pgrac-demo": "demo"}} + self.assertTrue(core.removable_pod(owned, "demo", 0)) + self.assertFalse(core.removable_pod(dict(owned, Labels={}), "demo", 0)) + self.assertFalse(core.removable_pod(dict(owned, Name="other"), "demo", 0)) + self.assertFalse(core.removable_pod(dict(owned, State="Running"), "demo", 0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/demo/tests/test_release_gate.py b/deploy/demo/tests/test_release_gate.py new file mode 100644 index 0000000000..02e8cb2fa8 --- /dev/null +++ b/deploy/demo/tests/test_release_gate.py @@ -0,0 +1,24 @@ +"""Exact-commit publishing gate tests. Author: SqlRush """ + +import importlib.util +from pathlib import Path +import unittest + +MODULE = Path(__file__).resolve().parents[1] / "release_gate.py" +gate = None +if MODULE.exists(): + spec = importlib.util.spec_from_file_location("release_gate", MODULE) + gate = importlib.util.module_from_spec(spec) + spec.loader.exec_module(gate) + + +class ReleaseGate(unittest.TestCase): + def test_missing_failed_or_newer_running_check_blocks_publish(self): + self.assertIsNotNone(gate, "release gate missing") + good = [{"name": name, "conclusion": "success", "started_at": "2026-09-20T00:00:00Z"} + for name in gate.REQUIRED] + self.assertEqual(gate.failures(good), []) + self.assertTrue(gate.failures(good[:-1])) + for conclusion in (None, "failure", "cancelled", "skipped"): + changed = good + [dict(good[0], conclusion=conclusion, started_at="2026-09-20T01:00:00Z")] + self.assertTrue(gate.failures(changed)) diff --git a/deploy/demo/workload.py b/deploy/demo/workload.py new file mode 100644 index 0000000000..07b3f7aa08 --- /dev/null +++ b/deploy/demo/workload.py @@ -0,0 +1,128 @@ +"""Data initialization and measured demo workload. Author: SqlRush """ + +import json +from pathlib import Path +import re +import subprocess +import time + +from demo_lib import benchmark_verdict, parse_pgbench, population_rows, row_snapshot + + +def snapshots(demo, report, label): + result = [] + for i in range(4): + path = report / (label+"-node"+str(i)+".tsv") + with path.open("x") as output: + demo.sql(i, "COPY (SELECT id,value,pad FROM demo_account ORDER BY id) TO STDOUT;", + stdout=output) + with path.open("rb") as source: + result.append(row_snapshot(source)) + if any(item != result[0] for item in result): + raise ValueError("full ordered demo_account bytes differ between nodes") + return result + + +def summary(path, value): + (path / "summary.json").write_text(json.dumps(value, indent=2)+"\n") + print(json.dumps(value, indent=2)) + print("Reports retained: " + str(path)) + + +def execute(demo, args): + if demo.state["phase"] != "READY" or demo.running() != [0, 1, 2, 3]: + raise ValueError("all four nodes must be READY") + if not all(demo.healthy(i) for i in range(4)): + raise ValueError("cluster readiness check failed") + stamp = time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())+"-"+str(time.time_ns()) + report = demo.root / "reports" / (args.command+"-"+stamp) + report.mkdir(parents=True, mode=0o700) + offsets = demo.log_offsets() + result = {"verdict": "FAIL", "is_formal_pre": False, "command": args.command, + "source_revision": demo.state["source_revision"], "image_id": demo.state["image_id"], + "phase": "starting"} + try: + before = snapshots(demo, report, "before") + result["before"] = before + if args.command == "init-data": + rows = population_rows(before[0]["rows"], args.rows) + if demo.sql(0, "SELECT count(*) FROM demo_smoke;").stdout.strip() != "0": + raise ValueError("demo_smoke is not empty; refusing to overwrite prior data") + demo.sql(0, "BEGIN; INSERT INTO demo_account SELECT g,0,repeat('p',64) " + "FROM generate_series(1,%d) AS g; INSERT INTO demo_smoke VALUES(1,0); COMMIT;" % rows) + demo.sql(0, "VACUUM (FREEZE, ANALYZE) demo_account;") + demo.sql(0, "VACUUM (FREEZE, ANALYZE) demo_smoke;") + after = snapshots(demo, report, "after") + if after[0]["rows"] != rows or after[0]["sum"] != 0: + raise ValueError("initialized row count/value mismatch") + demo.state["rows"] = rows + demo.save() + result.update(after=after, initialized_rows=rows) + elif args.command == "verify": + if "rows" not in demo.state: + raise ValueError("run init-data before verify") + if before[0]["rows"] != demo.state["rows"]: + raise ValueError("row count differs from initialized dataset") + # A real committed cross-node write is checked independently of count/hash. + initial = int(demo.sql(0, "SELECT value FROM demo_smoke WHERE id=1;").stdout.strip()) + for i in range(4): + value = demo.sql(i, "UPDATE demo_smoke SET value=value+1 WHERE id=1 RETURNING value;").stdout.strip() + if value != str(initial+i+1): + raise ValueError("cross-node committed increment is incorrect") + if any(demo.sql(i, "SELECT value FROM demo_smoke WHERE id=1;").stdout.strip() + != str(initial+4) for i in range(4)): + raise ValueError("cross-node smoke readback is incorrect") + result["cross_node_increment"] = 4 + else: + if not 1 <= args.clients <= 16 or not 1 <= args.seconds <= 600: + raise ValueError("clients per node must be 1..16; seconds must be 1..600") + rows = demo.state.get("rows") + if not rows or rows != before[0]["rows"]: + raise ValueError("initialize test data before benchmarking") + # Consecutive keys are required: an UPDATE of an absent row is not work. + for i in range(4): + keys = demo.sql(i, "SELECT min(id)||':'||max(id)||':'||count(*) FROM demo_account;").stdout.strip() + if keys != "1:%d:%d" % (rows, rows): + raise ValueError("benchmark requires exact consecutive primary keys") + children, outputs = [], [] + started = time.monotonic() + for i in range(4): + log = (report / ("pgbench-node%d.log" % i)).open("x") + outputs.append(log) + children.append(subprocess.Popen(demo.client_command(i, [ + "pgbench", "-h", "127.0.0.1", + "-p", str(demo.state["port_base"]+i), "-U", "pgrac", "-n", + "-c", str(args.clients), "-j", str(args.clients), "-T", str(args.seconds), + "-D", "rows="+str(rows), "-f", "/opt/pgrac/demo/sql/update.sql", "postgres"]), + stdout=log, stderr=subprocess.STDOUT)) + # Natural completion only. No forced cancellation is hidden as success. + codes = [child.wait() for child in children] + elapsed = time.monotonic()-started + for output in outputs: + output.close() + (report / "client-return-codes.json").write_text(json.dumps(codes)+"\n") + nodes = [parse_pgbench((report / ("pgbench-node%d.log" % i)).read_text(), codes[i]) + for i in range(4)] + after = snapshots(demo, report, "after") + logs = demo.new_logs(offsets) + errors = [len(re.findall(r"(?:ERROR|FATAL|PANIC):", log)) for log in logs] + health = [demo.healthy(i) for i in range(4)] + for i, text in enumerate(logs): + (report / ("server-node%d.log" % i)).write_text(text) + result.update(benchmark_verdict(nodes, before, after, errors, health)) + result.update(after=after, clients_per_node=args.clients, duration_seconds=args.seconds, + wall_seconds=elapsed, wall_tps=sum(n["transactions"] for n in nodes)/elapsed, + server_errors=errors, healthy=health) + summary(report, result) + if result["verdict"] != "PASS": + raise ValueError("benchmark correctness FAILED; inspect retained report") + return + errors = [len(re.findall(r"(?:ERROR|FATAL|PANIC):", log)) for log in demo.new_logs(offsets)] + if any(errors) or not all(demo.healthy(i) for i in range(4)): + raise ValueError("server error or post-operation health failure") + result.update(verdict="PASS", phase="complete", server_errors=errors) + summary(report, result) + except Exception as error: + result.update(verdict="FAIL", failure=str(error)) + summary(report, result) + raise diff --git a/docs/mvp/v0.131.0/README.md b/docs/mvp/v0.131.0/README.md index f0b0a22f21..33daf55ed8 100644 --- a/docs/mvp/v0.131.0/README.md +++ b/docs/mvp/v0.131.0/README.md @@ -15,6 +15,10 @@ UPDATE 耗时观测。稳定发布必须通过四轮正式正确性 PRE 与精 使用本机文件系统上的四个独立 PGDATA，共享业务数据；不是四台主机，不需要容器或 GFS2。 安装步骤沿用已演练示例，本轮没有重新运行 Quick Start 演练。 +免编译演示请用[单机四 Pod 安装与演示手册](../../../deploy/demo/README.md)： +Podman 镜像、环境预检、初始化和压测脚本、正常停机与原数据重启。 +该独立演示包不是多机共享盘或生产高可用认证。 + 已有集群不得逐节点热替换：先全体正常关闭、保留原干净数据，再使用同版本二进制正常启动。 没有认证混版本、滚动升级、崩溃恢复、通用迁移或降级；异常退出现场不能当作干净数据接管。