From 5ea2375f98f8f7fad5d1e087babcfc96a80d00b8 Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Thu, 8 Oct 2026 00:25:34 +0100 Subject: [PATCH 1/9] feat: stage every cold append before the serializer and claim at commit --- ci/journey.sh | 64 +++--- docker/entrypoint.sh | 9 +- docs/architecture.md | 36 ++-- docs/architecture_decoupled.md | 19 +- docs/changelog.md | 10 + docs/compaction.md | 13 +- docs/formal/Bakery_async_samenode.cfg | 36 ++++ docs/formal/Bakery_async_single.cfg | 34 +++ docs/formal/README.md | 125 +++++++---- docs/installation.md | 8 +- extension/coldfront/Makefile | 3 +- extension/coldfront/coldfront--1.0.sql | 196 ++++++++++-------- extension/coldfront/src/coldfront.c | 102 +++++++++ .../expected/async_write_before_claim.out | 152 ++++++++++++++ .../test/sql/async_write_before_claim.sql | 77 +++++++ 15 files changed, 698 insertions(+), 186 deletions(-) create mode 100644 docs/formal/Bakery_async_samenode.cfg create mode 100644 docs/formal/Bakery_async_single.cfg create mode 100644 extension/coldfront/test/expected/async_write_before_claim.out create mode 100644 extension/coldfront/test/sql/async_write_before_claim.sql diff --git a/ci/journey.sh b/ci/journey.sh index 03a15e2..7f1e090 100755 --- a/ci/journey.sh +++ b/ci/journey.sh @@ -561,7 +561,7 @@ EOSQL # ─────────────────────────────────────────────────────────────────────────── # Decoupled concurrency / no-409 — parallel cold writers to ONE iceberg-only # table must all land. Vanilla serializes them with the local advisory-lock -# bakery (_exec_iceberg_with_claim, v_armed=false → pg_advisory_xact_lock); +# bakery (_exec_iceberg_with_claim; _bakery_armed() false → pg_advisory_xact_lock); # without it, concurrent Iceberg commits race Lakekeeper's assert-ref-snapshot # precondition and 409 (CatalogCommitConflict), which aborts the losing # transaction. (Standing rule: multi-writer no-409 probe in vanilla. The bakery @@ -2399,14 +2399,15 @@ SQL # Story 6g: the compactor claims the reference cold writes claim, on a table # create_iceberg_table() made (TC-183) and on one adoption registered (TC-184), # as on the archiver's. Every registration stores one spelling, each part -# quoted, and it is the one the compactor builds, so a cold write holding the -# table's claim makes the compactor's claim wait. Each step then reads the table -# under its claim, so it works from the write it waited for: expiry (TC-183), -# compaction (TC-184), and orphan cleanup (TC-185), which with no age window -# would otherwise delete that write's files. +# quoted, and it is the one the compactor builds. An append's claim is taken at +# its COMMIT, so a transaction with a staged cold write holds nothing the +# compactor waits on: expiry (TC-183), compaction (TC-184) and orphan cleanup +# (TC-185) run beside the open write, and the write still lands on the result, +# its staged file kept by the --orphan-age window that protects every +# in-flight write. # ─────────────────────────────────────────────────────────────────────────── story_compactor_claim_matches_writes() { - step "6g. The compactor waits for a cold write's claim on created and adopted tables" + step "6g. The compactor runs beside an open cold write on created and adopted tables" require_compactor || return local col='[{"name":"id","type":"bigint"}]' t for t in cf_cw cf_ad cf_or; do @@ -2417,22 +2418,22 @@ story_compactor_claim_matches_writes() { assert_eq "TC-183: the archiver, create_iceberg_table() and adoption register one spelling" \ '"ice"."public"."cf_ad" "ice"."public"."cf_cw" "ice"."public"."events"' \ "$(q "$HOST" "SELECT string_agg(iceberg_table, ' ' ORDER BY relname) FROM coldfront.tiered_views WHERE schema_name = 'public' AND relname IN ('cf_ad', 'cf_cw', 'events');")" - compactor_waits_for_write TC-183 cf_cw 2 'expired [1-9][0-9]* snapshot' \ + compactor_runs_beside_write TC-183 cf_cw 2 'expired [1-9][0-9]* snapshot' \ --expire-snapshots --expire-older-than 0s --expire-retain-last 1 --expire-keep-files - compactor_waits_for_write TC-184 cf_ad 5 'compacted: [0-9]+ files' - compactor_waits_for_write TC-185 cf_or 1 'deleted [0-9]+ orphan' --orphans --orphan-age 0s + compactor_runs_beside_write TC-184 cf_ad 5 'compacted: [0-9]+ files' + compactor_runs_beside_write TC-185 cf_or 1 'deleted [0-9]+ orphan' --orphans for t in cf_cw cf_ad cf_or; do q "$HOST" "SELECT coldfront.drop_iceberg_table('public','$t', true);" >/dev/null 2>&1 done } -# compactor_waits_for_write [flags...]: after -# single-row cold writes to public.
, one more holds the table's claim for -# 8 s while the compactor runs with . The compactor's claim waits on the -# lock the write holds; once the write commits, the compactor succeeds, its -# output matches , and every write is still there. -compactor_waits_for_write() { - local tc=$1 t=$2 n=$3 pat=$4 w c ec i waited=0 +# compactor_runs_beside_write
[flags...]: after +# single-row cold writes to public.
, one more stays open for 8 s +# with its append staged and its claim due at COMMIT, while the compactor runs +# with . The compactor finishes while the write is still open, its output +# matches , and every write is still there once the write commits. +compactor_runs_beside_write() { + local tc=$1 t=$2 n=$3 pat=$4 w c ec i open=0 shift 4 for i in $(seq 1 "$n"); do echo "INSERT INTO $t VALUES ($i);"; done | qf "$HOST" >/dev/null 2>&1 qf "$HOST" >/dev/null 2>&1 <"$TMPD/cf.$tc" 2>&1 & c=$! - for i in $(seq 1 24); do - if [ "$(q "$HOST" "SELECT count(*) FROM pg_stat_activity WHERE wait_event_type = 'Lock' AND query LIKE '%coldfront._claim_iceberg_external%' AND pid <> pg_backend_pid();")" = 1 ]; then - waited=1; break - fi - sleep 0.25 - done wait "$c"; ec=$? + open=$(q "$HOST" "SELECT count(*) FROM pg_stat_activity WHERE application_name = 'cf_${tc}_writer' AND wait_event = 'PgSleep';") wait "$w" - assert_eq "$tc: the compactor's claim waits for the cold write's" 1 "$waited" - assert_eq "$tc: the compactor finishes once the write commits" 0 "$ec" + assert_eq "$tc: the compactor finished while the cold write was still open" 1 "$open" + assert_eq "$tc: the compactor succeeded" 0 "$ec" [ "$ec" = 0 ] || tail -5 "$TMPD/cf.$tc" assert_eq "$tc: the compactor did its work" 1 "$(grep -c -E "$pat" "$TMPD/cf.$tc")" assert_eq "$tc: every write is still there" "$((n + 1))" "$(q "$HOST" "SELECT count(*) FROM $t;")" @@ -3503,7 +3499,7 @@ story_mesh() { assert_eq "write from peer $p1 visible on db1" "1" "$(q "$HOST" "SELECT count(*) FROM iceonly WHERE status='from_peer';")" # R-A bakery under multi-node contention: concurrent cold writers on db1 AND - # a peer to the SAME Iceberg table must both land. Here v_armed is true + # a peer to the SAME Iceberg table must both land. Here _bakery_armed() is true # (snowflake.node + loopback_dsn set), so this exercises the Ricart-Agrawala # claim protocol across nodes — not the local advisory lock — to avoid 409. rm -f $TMPD/ra.* 2>/dev/null @@ -3883,7 +3879,8 @@ story_mesh_claim_failures() { ref_t=$(q "$HOST" "SELECT iceberg_table FROM coldfront.tiered_views WHERE relname = 'cf_ct';") # TC-174: a claim times out while another session holds the table's claim - # key. The session holds no advisory lock afterwards, and its next write + # key. An append's claim is taken at its COMMIT, which is where the timeout + # fires. The session holds no advisory lock afterwards, and its next write # clears what the timed-out attempt and an orphan planted on another table # left on the node. q "$HOST" "INSERT INTO coldfront.claims (iceberg_table, ticket) VALUES ('${ref_t}_tc174', snowflake.nextval());" >/dev/null @@ -3897,7 +3894,9 @@ SQL sleep 1 out=$(sess "$HOST" 2>&1 <"$TMPD/cf.175a" 2>&1 </dev/null a_commit=$(sed -n 's/^a_commit=//p' "$TMPD/cf.175a") - assert_eq "TC-175 a second write to the same table keeps the transaction's claim" "1" "$(later "$p_done" "$a_commit")" + assert_eq "TC-175 a peer's write lands while the transaction is still open" "0" "$(later "$p_done" "$a_commit")" assert_eq "TC-175 no write errored" "0" "$(cat "$TMPD"/cf.175? | grep -c ERROR)" assert_eq "TC-175 all three writes landed" "3" "$(q "$HOST" "SELECT count(*) FROM cf_ct WHERE id BETWEEN 1751 AND 1753;")" # TC-176: one transaction writes cf_cu, then cf_ct; a peer's write to cf_cu - # arrives between the two and stays behind the cf_cu claim until COMMIT. + # arrives between the two and lands at once, as in TC-175, and every write + # on both tables lands. sess "$HOST" >"$TMPD/cf.176a" 2>&1 </dev/null a_commit=$(sed -n 's/^a_commit=//p' "$TMPD/cf.176a") - assert_eq "TC-176 a write to a second table keeps the transaction's claim on the first" "1" "$(later "$p_done" "$a_commit")" + assert_eq "TC-176 a peer's write to the first table lands while the transaction is still open" "0" "$(later "$p_done" "$a_commit")" assert_eq "TC-176 no write errored" "0" "$(cat "$TMPD"/cf.176? | grep -c ERROR)" assert_eq "TC-176 all three writes landed" "3" \ "$(q "$HOST" "SELECT (SELECT count(*) FROM cf_cu WHERE id IN (1761, 1763)) + (SELECT count(*) FROM cf_ct WHERE id = 1762);")" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 5848c3d..6f77951 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -62,10 +62,11 @@ shared_preload_libraries = '${PRELOAD}' # postgres_scanner + any non-bundled deps auto-install; it does NOT clobber the # pre-placed iceberg/avro (DuckDB skips install for extensions already present). # allow_unsigned ON so the locally-built (unsigned) extension loads; autoload ON -# so ATTACH (TYPE ICEBERG, ...) lazily LOADs it; iceberg_async_parquet ON so the -# mesh bakery uploads parquet in the background and serializes only the commit -# POST (safe because the patch refreshes parent_snapshot_id at commit). Vanilla -# ignores the flag (advisory lock, claim-first). +# so ATTACH (TYPE ICEBERG, ...) lazily LOADs it; iceberg_async_parquet ON so +# every cold write uploads its parquet before it takes the serializer (the +# bakery claim on a mesh node, the advisory lock on a vanilla one) and only the +# commit POST is serialized (safe because the patch refreshes +# parent_snapshot_id at commit). # # iceberg_bakery_patch = on ASSERTS that the duckdb-iceberg in THIS image carries # the bakery-aware-commit-refresh patch (it does — the Dockerfile git-applies it). diff --git a/docs/architecture.md b/docs/architecture.md index 796f568..3a7460c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -350,26 +350,28 @@ ColdFront coordinates concurrent writes across the cluster as follows: ### Cold-Write Strategy: Stock vs Patched duckdb-iceberg -Most cold writes run through `_exec_iceberg_with_claim`; the tiered `INSERT`'s -cold sink and the cross-tier move take the claim through `_take_iceberg_claim` -themselves. What differs is *when* the bakery ticket is held. The async -ordering is used only on a mesh node with the bakery enabled, and only when -**both** `coldfront.iceberg_async_parquet` (default `off`) and the build marker -`coldfront.iceberg_bakery_patch` (asserting the loaded duckdb-iceberg includes -the patch) are on (`coldfront._iceberg_async_active()`); a vanilla node always -takes its lock before uploading. The following table shows the two orderings: - -| Ordering | duckdb-iceberg | Behavior | +Every cold write runs through `_exec_iceberg_with_claim`. What differs is +*when* the serializer (the bakery ticket on a mesh node, the advisory lock on a +vanilla node) is held. The async ordering is used on every topology, for +appends, when **both** `coldfront.iceberg_async_parquet` (default `off`) and +the build marker `coldfront.iceberg_bakery_patch` (asserting the loaded +duckdb-iceberg includes the patch) are on (`coldfront._iceberg_async_active()`). +The following table shows the two orderings: + +| Ordering | Applies to | Behavior | |---|---|---| -| stock (default) | stock upstream | The writer takes the bakery ticket first, *then* uploads parquet and commits inside the ticket. This ordering is correct on an unpatched binary, but the whole parquet upload happens under the lock, so concurrent writers serialize on upload and commit. | -| async (both GUCs `on`) | patched (`iceberg-bakery-aware-commit-refresh-v15.patch`) | The writer uploads parquet in the background *first*, then takes the ticket only for the Lakekeeper commit. Concurrent writers' uploads overlap; only the short commit POST is serialized. | +| stock (default) | every write on stock upstream duckdb-iceberg; a `DELETE`, `UPDATE` or `MERGE` (the cross-tier move and `vector_train`'s assignment included) on every build | The writer takes the serializer first, *then* uploads parquet and commits inside it. On an unpatched binary this is what keeps a second writer from capturing a stale parent. A write that references rows the table holds needs it on every build: its position deletes name data files by path, and a compaction or another such write committing between its scan and its commit would rewrite or remove those files, after which the deletes apply to nothing and the rows come back. | +| async (both GUCs `on`) | appends (`INSERT`, `COPY`, the archiver's exports, the tiered `INSERT`'s cold sink) on the patched build (`iceberg-bakery-aware-commit-refresh-v15.patch`) | The statement uploads its parquet and queues the claim; the extension's transaction callback takes it at PRE_COMMIT, just before pg_duckdb's own PRE_COMMIT callback commits the Iceberg transaction, and releases it at COMMIT. Uploads overlap on one node and across nodes, an open transaction blocks no other writer, and only the short commit POST is serialized. A new data file conflicts with nothing committed in between. | The code path and the application-visible behavior are identical, so the GUCs -are purely a performance knob. The patch relocates parent-snapshot stamping -from upload time into PG's pre-commit phase (inside the bakery ticket, against -a freshly-fetched table), so overlapping uploads cannot commit a stale parent. -Async requested without the build marker downgrades safely to the stock -ordering, noted once per session with a server LOG line - never a silent 409. +are purely a performance knob. The patch refreshes the table from the catalog +in PG's pre-commit phase (inside the serializer) and re-derives every pending +snapshot of the transaction from that head, so overlapping uploads and several +batches alike cannot commit a stale parent. A staged file is referenced by no +snapshot until its transaction commits, which is what the compactor's +`--orphan-age` window is for. Async requested without the build marker +downgrades safely to the stock ordering, noted once per session with a server +LOG line - never a silent 409. The Docker image ships the patched binary and sets both GUCs on (`docker/entrypoint.sh`); bare-metal users on a stock binary leave both `off` and lose only the upload overlap. See diff --git a/docs/architecture_decoupled.md b/docs/architecture_decoupled.md index f19e710..952d472 100644 --- a/docs/architecture_decoupled.md +++ b/docs/architecture_decoupled.md @@ -598,10 +598,21 @@ pointing at the same Lakekeeper endpoint and S3 bucket: that ticket, emitting any acks the node had been holding back. This is the stock ordering. With `coldfront.iceberg_async_parquet` and - `coldfront.iceberg_bakery_patch` both on, as the shipped image sets them, a - mesh writer stages the Parquet upload through `duckdb.raw_query` first and - takes the claim afterwards; only the commit POST runs under the claim, with - the parent snapshot re-stamped by the bakery-aware patch. With + `coldfront.iceberg_bakery_patch` both on, as the shipped image sets them, an + `INSERT` stages its Parquet upload through `duckdb.raw_query` inside the + statement and the C transaction callback takes the claim at PRE_COMMIT, + just before pg_duckdb's own PRE_COMMIT callback commits the Iceberg + transaction; only the commit POST runs under the claim, with the parent + snapshot re-stamped by the bakery-aware patch, and an open transaction + holds no claim. PostgreSQL switches `statement_timeout` off before commit + processing, so the callback runs the timer again, at the statement's own + deadline, while it waits for the claim: a wait that outlives the setting + fails with the statement-timeout error and the transaction rolls back. A + `DELETE`, `UPDATE` or `MERGE` references rows the table + already holds, so it takes the stock ordering on every build: its position + deletes name data files, and a compaction or another such write committing + between its scan and its commit would leave them pointing at nothing. A + vanilla node orders its advisory lock the same way. With `coldfront.iceberg_async_parquet` on and `coldfront.iceberg_bakery_patch` off, `coldfront._iceberg_async_active()` returns false, and the writer keeps the stock ordering and logs the downgrade once per session. diff --git a/docs/changelog.md b/docs/changelog.md index 5bd1499..9f01251 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -80,6 +80,16 @@ this project adheres to ### Changed +- Every append to the cold tier (`INSERT`, `COPY`, the archiver's exports, the + tiered `INSERT`'s cold sink) uses the async ordering wherever the patched + duckdb-iceberg is loaded, on a single node as on a mesh: the statement + uploads its Parquet, and the serializer (the bakery claim, or the node's + advisory lock) is taken at the transaction's commit, so it wraps only the + catalog commit and an open transaction blocks no other writer; + `statement_timeout` bounds the wait for it. A `DELETE`, + `UPDATE` or `MERGE`, the cross-tier move and `coldfront.vector_train` keep + the claim-first ordering on every build, since their position deletes name + the data files a concurrent compaction could rewrite. - The mesh bakery no longer needs the `dblink` extension. Claims, acks, releases and orphan reaping run over a libpq loopback connection that the extension opens from `coldfront.loopback_dsn`, which must name a unix socket. diff --git a/docs/compaction.md b/docs/compaction.md index ba79027..55cd321 100644 --- a/docs/compaction.md +++ b/docs/compaction.md @@ -108,13 +108,16 @@ requested steps, each under a bakery claim on that table: table; the 168 h default does, and `0s` is for a table nothing is reading. - Orphan removal deletes files under the table location that no retained snapshot references, which covers files left by an interrupted write or by - `--expire-keep-files`. The `--orphan-age` window keeps a concurrent writer's - freshly-staged files from being removed. + `--expire-keep-files`. A cold write's data files are staged before its + transaction commits and referenced by no snapshot until then, so the + `--orphan-age` window is what keeps them: keep it longer than the longest + transaction that writes the table. The 72 h default does. Each mutating step holds the bakery claim across its catalog commit and -releases it when its PostgreSQL transaction commits, so it cannot interleave -with a cold write to the same table. Snapshot maintenance is the engine's job, -not the catalog's: Lakekeeper does no Iceberg snapshot or orphan maintenance. +releases it when its PostgreSQL transaction commits, so its commit cannot +interleave with a cold write's commit to the same table. Snapshot maintenance +is the engine's job, not the catalog's: Lakekeeper does no Iceberg snapshot or +orphan maintenance. ## Requirements diff --git a/docs/formal/Bakery_async_samenode.cfg b/docs/formal/Bakery_async_samenode.cfg new file mode 100644 index 0000000..bb8747f --- /dev/null +++ b/docs/formal/Bakery_async_samenode.cfg @@ -0,0 +1,36 @@ +\* TLC config for Bakery.tla: the patched async ordering with SEVERAL cold +\* writers per node. Writers a1 and a2 share node nA and b1 is on node nB, as in +\* Bakery_samenode.cfg; AsyncParquet and RestampPatch are on, as in +\* Bakery_async.cfg. Every writer stages its parquet before it takes the claim, +\* so a1 and a2 stage concurrently; the node-local advisory lock then admits one +\* of them to the bakery, and the commit POST re-stamps parent_snapshot_id under +\* the claim. The cold sink, which stages several batches before its one claim, +\* is the same writer: the model's Stage covers every upload a transaction makes +\* before it claims. +\* +\* TLC is EXPECTED to report no error: all four safety invariants hold. Without +\* SameNodeLock this topology violates NoLakekeeperConflict exactly as +\* Bakery_samenode_race.cfg does, so the check is not vacuous. No crashes. + +CONSTANTS + Writers = {a1, a2, b1} + NodeParts = {{a1, a2}, {b1}} + SameNodeLock = TRUE + Reaper = FALSE + NodeRetries = FALSE + MaxTickets = 3 + MaxIcebergLen = 5 + MaxCrashes = 0 + AsyncParquet = TRUE + RestampPatch = TRUE + SafeAcks = TRUE + AdoptClaims = FALSE + +SPECIFICATION Spec + +INVARIANTS + NoAckOutlivesClaim + NoLakekeeperConflict + RollbackNoIceberg + UniqueTickets + TicketOrderPreserved diff --git a/docs/formal/Bakery_async_single.cfg b/docs/formal/Bakery_async_single.cfg new file mode 100644 index 0000000..3bfd38a --- /dev/null +++ b/docs/formal/Bakery_async_single.cfg @@ -0,0 +1,34 @@ +\* TLC config for Bakery.tla: the patched async ordering on a SINGLE node with +\* three concurrent cold writers (NodeParts = {{a1, a2, a3}}). A node without +\* Spock has no peer and no claim row: coldfront._take_iceberg_claim takes the +\* per-table advisory transaction lock alone, which is SameNodeLock here. The +\* claim the model still inserts has no peer to see it and changes nothing a +\* same-node writer does, so this is the vanilla deployment: stage outside the +\* lock, lock, re-stamp parent_snapshot_id under the lock, commit, release. +\* With one node the Applier and the Poker never run. +\* +\* TLC is EXPECTED to report no error: all four safety invariants hold. No +\* crashes. + +CONSTANTS + Writers = {a1, a2, a3} + NodeParts = {{a1, a2, a3}} + SameNodeLock = TRUE + Reaper = FALSE + NodeRetries = FALSE + MaxTickets = 3 + MaxIcebergLen = 5 + MaxCrashes = 0 + AsyncParquet = TRUE + RestampPatch = TRUE + SafeAcks = TRUE + AdoptClaims = FALSE + +SPECIFICATION Spec + +INVARIANTS + NoAckOutlivesClaim + NoLakekeeperConflict + RollbackNoIceberg + UniqueTickets + TicketOrderPreserved diff --git a/docs/formal/README.md b/docs/formal/README.md index 1bec5b3..b2692ff 100644 --- a/docs/formal/README.md +++ b/docs/formal/README.md @@ -23,6 +23,8 @@ an explicit applier. The following table describes its files and their roles: | `Bakery.tla` | This file is the PlusCal source. It models the real Spock world: each node has its OWN local `claims[nd]` view, and INSERTs propagate via an explicit `Applier`. The model does not assume `synchronous_commit = remote_apply`. Coordination is Lamport's 1978 distributed mutual exclusion algorithm with Ricart-Agrawala's (1981; henceforth abbreviated R-A) deferred-reply optimization: peers ack each claim immediately unless they have a pending claim with smaller ticket, in which case they defer the ack until they release their own claim. The file also models the `coldfront.iceberg_async_parquet` flag (constants `AsyncParquet`/`RestampPatch`): the `Stage` label stages parquet OUTSIDE the claim (async ordering); `Prepare` captures the `parent_snapshot_id` UNDER the claim (stock at stage time, patched async re-stamped at the commit POST); the conditional commit at `Decide` asserts against it. Defer/drain atomicity is modeled too (constant `SafeAcks`): the apply-time defer DECISION and its WRITE (`ApplyDecide`/`ApplyEmit`), and the release drain's FORWARD and DELETE (`DrainForward`/`DrainDelete`), are SEPARATE steps, faithful to the non-atomic SQL. `SafeAcks=FALSE` lets them race (a deferral written behind a just-released claim is deleted unforwarded / orphaned - a dropped ack); `SafeAcks=TRUE` is the safe implementation: an atomic re-check of R-A's own defer rule against the claim, i.e. `SELECT … FOR UPDATE` on the claim row in `coldfront._on_claim_apply`. The orphan reaper is modeled as well (constant `Reaper`): a same-node claim whose writer has crashed holds no advisory lock, so `BeginClaim` deletes it in the step that inserts its own claim; the `Applier` does the same on its defer branch under `HoldsTableLock(nd)` (a live writer holds `coldfront_iceberg:
` from `WaitAcks` through `DrainDelete`, because the release runs in the COMMIT callback before PostgreSQL drops the xact lock); and a `Poker` process models the waiter's periodic no-op UPDATE of its own claim row, which re-runs the peer's defer branch and is what unwedges a waiter whose peer's holder died after deferring to it. Constant `NodeRetries` restricts the `Crasher` to writers whose node still has an unstarted writer (the node is touched again); FALSE allows the crash after which only the poke reaches the node. The `Crasher` cannot split `Release`/`DrainForward`/`DrainDelete`, which are one loopback transaction in the real code. | | `Bakery.cfg` | This TLC config checks 3 writers with no crashes against all four safety invariants, in stock ordering (`AsyncParquet=FALSE`, the default, with parquet staged inside the claim). The config passes: the bakery makes `NoLakekeeperConflict` and `TicketOrderPreserved` hold even with realistic asymmetric apply. | | `Bakery_async.cfg` | This config checks the patched async ordering (`AsyncParquet=TRUE, RestampPatch=TRUE`) that the DuckDB 1.5.x (duckdb15) image runs: parquet is staged outside the claim, and the bakery-aware patch re-stamps `parent_snapshot_id` at the commit POST under the claim. All four safety invariants HOLD; the test is non-vacuous, because it shares the stock config's under-claim window from `Prepare` to `Decide`, which R-A keeps empty. | +| `Bakery_async_samenode.cfg` | This config checks the patched async ordering with several cold writers per node (`NodeParts={{a1,a2},{b1}}`, `SameNodeLock=TRUE`): the writers on one node stage their parquet concurrently, the node-local advisory lock admits one of them to the bakery at a time, and the commit POST re-stamps the parent under the claim. All four safety invariants HOLD. The cold sink, which stages several batches before its one claim, is this writer: `Stage` covers every upload a transaction makes before it claims. | +| `Bakery_async_single.cfg` | This config checks the patched async ordering on one node with three writers (`NodeParts={{a1,a2,a3}}`), the vanilla deployment: `coldfront._take_iceberg_claim` takes the per-table advisory transaction lock alone there, which is `SameNodeLock`; the claim row the model inserts has no peer to see it and changes nothing a same-node writer does. With one node the `Applier` and the `Poker` never run. All four safety invariants HOLD. | | `Bakery_race.cfg` | This config checks the pre-patch async race (`AsyncParquet=TRUE, RestampPatch=FALSE`): async ordering WITHOUT the bakery-aware patch, where the stale tentative parent from the pre-claim stage is used at the POST. `NoLakekeeperConflict` is EXPECTED to be violated, which is the formal proof that the patch is mandatory for the async ordering. | | `Bakery_crash.cfg` | This config checks 3 writers with a budget of 1 crash, in stock ordering (crash safety is ordering-independent). The safety invariants still hold, and survivors proceed, because the dead-peer escape treats a crashed node as already acked. This config checks safety only; stranding under a crash appears only with same-node writers (`Bakery_wedge.cfg`). | | `Bakery_live.cfg` | This config checks the defer/drain race (`SafeAcks=FALSE`), the non-atomic implementation. `EventualProgress` is EXPECTED to be VIOLATED: a deferred ack written behind a just-released claim is dropped, stranding the min-ticket holder at `WaitAcks` forever, which is the N-writer wedge this race produces. The four safety invariants still HOLD (a dropped ack is a liveness failure, not a wrong commit). The config declares no `SYMMETRY`, because symmetry reduction is unsound with liveness checking. | @@ -116,6 +118,14 @@ java -cp $TLA tlc2.TLC -workers auto -deadlock -config Bakery_async.cfg Bakery.t # bakery-aware patch is mandatory for the async path. java -cp $TLA tlc2.TLC -workers auto -deadlock -config Bakery_race.cfg Bakery.tla +# n. This run checks the patched async ordering with several writers per node +# (the cold sink's shape, several uploads before one claim). All hold. +java -cp $TLA tlc2.TLC -workers auto -deadlock -config Bakery_async_samenode.cfg Bakery.tla + +# o. This run checks the patched async ordering on a single node (the vanilla +# deployment's advisory lock). All hold. +java -cp $TLA tlc2.TLC -workers auto -deadlock -config Bakery_async_single.cfg Bakery.tla + # d. This run has a budget of 1 crash. Safety still holds; survivors proceed, # because a crashed node counts as already acked. java -cp $TLA tlc2.TLC -workers auto -deadlock -config Bakery_crash.cfg Bakery.tla @@ -210,6 +220,39 @@ hold. The check is non-vacuous: it shares the stock config's under-claim ticket defers its ack until it releases, so two writers never both clear `WaitAcks`). This is the ordering the DuckDB 1.5.x (duckdb15) image runs. +### `Bakery_async_samenode.cfg` (Patched Async, Several Writers per Node) + +The patched-async config with two writers on one node reports a clean check +with the following output: + +```text +Model checking completed. No error has been found. +568475 states generated, 194964 distinct states found, 0 states left on queue. +``` + +Two writers on one node stage their parquet concurrently, outside the claim; +the node-local advisory lock (`SameNodeLock`) then admits one of them to the +bakery at a time, and each re-stamps the parent under the claim. This is the +cold sink's shape: several uploads, then one claim. Without the lock this +topology violates `NoLakekeeperConflict` as `Bakery_samenode_race.cfg` does, so +the check is non-vacuous. + +### `Bakery_async_single.cfg` (Patched Async, Single Node) + +The single-node patched-async config reports a clean check with the following +output: + +```text +Model checking completed. No error has been found. +208013 states generated, 72608 distinct states found, 0 states left on queue. +``` + +Three writers on one node, with no peer: the vanilla deployment, whose +serializer is the per-table advisory transaction lock (`SameNodeLock`) alone. +Each writer stages outside the lock, takes it, re-stamps the parent under it +and commits. The claim row the model still inserts has no peer to see it; the +`Applier` and the `Poker` never run. + ### `Bakery_race.cfg` (Pre-Patch Async - EXPECTED FAILURE) The pre-patch async config is expected to fail the check, reporting the @@ -281,26 +324,38 @@ faithfully because they affect protocol correctness: row on the originator; the node's next claim deletes it (the `acks` CTE in `_insert_claim`). -- The `coldfront.iceberg_async_parquet` flag's two mesh orderings in - [_exec_iceberg_with_claim](https://github.com/pgEdge/ColdFront/blob/main/extension/coldfront/coldfront--1.0.sql): - stock (claim → stage+commit under the claim) and patched async (stage - parquet outside the claim → claim → re-stamp `parent_snapshot_id` at the - commit POST under the claim). The safety-critical invariant - the parent - snapshot the conditional commit checks is taken UNDER the held claim - is - captured at `Prepare` for both; the `AsyncParquet`/`RestampPatch` constants - select the ordering and whether the bakery-aware patch is present. In the - code, `coldfront._iceberg_async_active()` selects the async ordering only - when the build marker `coldfront.iceberg_bakery_patch` is also on. With the - flag alone, the writer keeps the stock ordering and logs the downgrade once - per session, so the `Bakery_race.cfg` combination runs only where the marker - is set on a stock binary. +- The `coldfront.iceberg_async_parquet` flag's two orderings in + [_exec_iceberg_with_claim](https://github.com/pgEdge/ColdFront/blob/main/extension/coldfront/coldfront--1.0.sql), + on a mesh node and a vanilla node alike: stock (claim → stage+commit under + the claim) and patched async (stage parquet outside the claim → claim → + re-stamp `parent_snapshot_id` at the commit POST under the claim). The + safety-critical invariant - the parent snapshot the conditional commit + checks is taken UNDER the held claim - is captured at `Prepare` for both; the + `AsyncParquet`/`RestampPatch` constants select the ordering and whether the + bakery-aware patch is present. `Stage` stands for every upload the + transaction makes before it claims, so the cold sink's batches and a + transaction's several `INSERT` statements are one `Stage`, and `BeginClaim` + is the C `XactCallback` taking the queued claims at `XACT_EVENT_PRE_COMMIT`, + before pg_duckdb's own PRE_COMMIT callback commits the Iceberg transaction + (coldfront registers its callback after pg_duckdb, and PostgreSQL calls the + latest registered first). The async ordering is taken by appends alone: a + `DELETE`, `UPDATE` or `MERGE` references data files by path, which the model + does not represent, and takes the stock ordering so its scan and its commit + see one head. In the code, `coldfront._iceberg_async_active()` selects the + async ordering only when the build marker `coldfront.iceberg_bakery_patch` is + also on. With the flag alone, the writer takes the stock ordering and logs + the downgrade once per session, so the `Bakery_race.cfg` combination runs + only where the marker is set on a stock binary. - The bakery's min-ticket spin in [_claim_iceberg_lock](https://github.com/pgEdge/ColdFront/blob/main/extension/coldfront/coldfront--1.0.sql). - The deferred release, in which pg_duckdb commits the Iceberg transaction at `XACT_EVENT_PRE_COMMIT` and coldfront's XactCallback `DELETE`s the claim at `XACT_EVENT_COMMIT`, so the commit always precedes the release whatever order the two callbacks were registered in. The model represents this as the - iceberg append at `Decide` followed by the claim `DELETE` at `Release`. + iceberg append at `Decide` followed by the claim `DELETE` at `Release`. The + async ordering's deferred claim, taken by the same callback at PRE_COMMIT, + rests on the registration order: coldfront's callback runs first, so the + claim is held when pg_duckdb's commit runs. - pg_duckdb's iceberg `ROLLBACK` on PG ABORT (no append on the rollback branch), which the `RollbackNoIceberg` property requires. @@ -363,21 +418,20 @@ view. A partition-column `UPDATE` that crosses the cutoff is rewritten to `coldfront._cross_tier_move`, which relocates rows between tiers. Its hot-tier work is plain PostgreSQL (heap `INSERT`/`UPDATE`/`DELETE` - no Iceberg, no -claim). Its cold-tier work is **one** `duckdb.raw_query` issued through the -**unchanged** path: a single `DELETE`-set plus `INSERT`-set in one DuckDB -MetaTransaction - one Iceberg snapshot, one conditional commit POST to -Lakekeeper - under **one** `_claim_iceberg_lock` held to transaction end -(released by the C `XactCallback`). The move is therefore the **same -stock-ordering single claimant** the cold writer is: one conditional commit -(identical parent-snapshot conflict shape to the append modeled at `Decide`) -under the held claim. The move forces `iceberg_async_parquet = off` -(`AsyncParquet = FALSE`, `Bakery.cfg`) - the `DELETE`+`INSERT` bundle is not -pg_duckdb's single deferred POST that the async re-stamp patch wraps. Each move -takes exactly **one** claim (a second claim on the same table would reap the -first, which `_take_iceberg_claim` prevents by reusing the transaction's -claim), so the move never holds two tickets. The move adds no new protocol -primitive, so it is covered by the existing model and configs; no dedicated -config is needed. +claim). Its cold-tier work is **one** statement issued through the +**unchanged** `_exec_iceberg_with_claim`: a single `DELETE`-set plus +`INSERT`-set in one DuckDB MetaTransaction - one Iceberg snapshot, one +conditional commit POST to Lakekeeper - under **one** claim held to +transaction end (released by the C `XactCallback`). The move is therefore the +**same stock-ordering single claimant** the cold writer is: one conditional +commit (identical parent-snapshot conflict shape to the append modeled at +`Decide`) under the held claim, which it takes before it reads the rows it +relocates, since the bundle removes rows by key and re-adds the values read +(`AsyncParquet = FALSE`, `Bakery.cfg`). Each move takes exactly **one** claim (a second +claim on the same table would reap the first, which `_take_iceberg_claim` +prevents by reusing the transaction's claim), so the move never holds two +tickets. The move adds no new protocol primitive, so it is covered by the +existing model and configs; no dedicated config is needed. ### Partition Detach Fan-Out @@ -436,12 +490,13 @@ models are only as useful as their fidelity. `MaxTickets = 6, MaxIcebergLen = 5, |Writers| = 3` is the default bound (`MaxCrashes` is 1 in the crash configs and 0 elsewhere; the same-node configs -use `MaxTickets = 3` with writers `{a1, a2, b1}`, and the adopt configs use -`MaxTickets = 2` and `MaxIcebergLen = 4` with writers `{a1, b1}`). Every config -checks in seconds on a modern laptop. `SYMMETRY` on `Writers` (up to a ~6× -reduction at 3 writers) is per-config: the liveness configs (`Bakery_live.cfg`, -`Bakery_fixed.cfg`) run without it (symmetry reduction is unsound with liveness -checking), and the same-node configs run without it (`a1`/`a2`/`b1` are not +and the single-node config use `MaxTickets = 3` with writers `{a1, a2, b1}` +and `{a1, a2, a3}`, and the adopt configs use `MaxTickets = 2` and +`MaxIcebergLen = 4` with writers `{a1, b1}`). Every config checks in seconds on +a modern laptop. `SYMMETRY` on `Writers` (up to a ~6× reduction at 3 writers) +is per-config: the liveness configs (`Bakery_live.cfg`, `Bakery_fixed.cfg`) +run without it (symmetry reduction is unsound with liveness checking), and the +same-node and single-node configs run without it (`a1`/`a2`/`b1` are not interchangeable under `NodeParts`), and so do the two adopt configs; the other configs declare it. diff --git a/docs/installation.md b/docs/installation.md index 72d721b..9deaf81 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -73,10 +73,10 @@ The patched extensions are unsigned, so `duckdb.allow_unsigned_extensions` must be on. Keeping `duckdb.autoinstall_known_extensions` off stops DuckDB from downloading an unpatched upstream copy when an extension file is missing. -The two `coldfront.iceberg_*` settings take effect only on a Spock mesh, where -a node then uploads Parquet files outside the bakery claim and serializes only -the catalog commit. The packaged duckdb-iceberg includes the patch that this -ordering requires, and the +The two `coldfront.iceberg_*` settings make a node upload the Parquet files of +an `INSERT` or `COPY` outside its write lock (the bakery claim on a Spock mesh, +an advisory lock on a single node) and serialize only the catalog commit. The +packaged duckdb-iceberg includes the patch that this ordering requires, and the [Distributed Setup](usage.md#distributed-setup-3-node-mesh-decoupled-mode) section of the Using ColdFront guide describes the mesh settings. diff --git a/extension/coldfront/Makefile b/extension/coldfront/Makefile index 24c65a1..29ad732 100644 --- a/extension/coldfront/Makefile +++ b/extension/coldfront/Makefile @@ -29,7 +29,8 @@ REGRESS = load_order update_unregistered_view update_heap_table \ type_map hot_guard adopt_type_map adopt_read_only adopt_iceberg_table \ cold_write_json_agg duckdb_temp_dir \ vector_centroids vector_type_map vector_cold_render array_cold_render vector_param_render vector_ops vector_probe vector_status vector_multicolumn \ - duckdb_keyword_column partition_clause failed_txn_block tiered_insert_single_pass tiered_view_no_trigger copy_tiered_view param_in_subquery merge_on_view delete_without_where update_from_without_alias settings_registered ensure_replicated read_inner_product + duckdb_keyword_column partition_clause failed_txn_block tiered_insert_single_pass tiered_view_no_trigger copy_tiered_view param_in_subquery merge_on_view delete_without_where update_from_without_alias settings_registered ensure_replicated read_inner_product \ + async_write_before_claim REGRESS_OPTS = --inputdir=test --outputdir=test PG_CONFIG ?= pg_config diff --git a/extension/coldfront/coldfront--1.0.sql b/extension/coldfront/coldfront--1.0.sql index dbe6f19..0d0cac4 100644 --- a/extension/coldfront/coldfront--1.0.sql +++ b/extension/coldfront/coldfront--1.0.sql @@ -512,12 +512,14 @@ BEGIN -- UPDATE to SELECT _cross_tier_move(...), which serialises -- cold rows via _move_row_literal. '_cross_tier_move', '_move_row_literal', '_move_pg_row_literal', - '_enqueue_release', + '_enqueue_release', '_enqueue_claim', -- cold-write preconditions + serializer gate wrappers (the - -- standby refusal, the armed-predicate, the claim-or-advisory - -- acquire); app-role cold paths call these SECURITY INVOKER - -- wrappers, which delegate to the DEFINER primitives below. - '_reject_on_standby', '_bakery_armed', '_take_iceberg_claim', + -- standby refusal, the armed-predicate, the Spock-without- + -- bakery refusal, the claim-or-advisory acquire); app-role + -- cold paths call these SECURITY INVOKER wrappers, which + -- delegate to the DEFINER primitives below. + '_reject_on_standby', '_bakery_armed', '_refuse_spock_without_bakery', + '_take_iceberg_claim', -- R-A bakery coordination (mesh cold writes); SECURITY -- DEFINER, so the app role just needs EXECUTE — the -- spock/loopback/pg_stat_replication access happens as owner. @@ -905,11 +907,13 @@ $$; -- column, the DEFAULT expression for an omitted column that has one, NULL -- otherwise. The sink renders and batches and does nothing else: the step -- function renders each row as a DuckDB VALUES tuple and flushes every --- coldfront.cold_write_batch_size rows as one INSERT under the table's claim, --- and the final function flushes the rest and returns the row count. The --- claim is taken once per table per transaction (_take_iceberg_claim), the --- same sequence as every other cold write. The state is NULL until the first --- row, so an aggregate over no cold rows writes nothing and returns 0. +-- coldfront.cold_write_batch_size rows as one INSERT through +-- _exec_iceberg_with_claim, and the final function flushes the rest and +-- returns the row count. Each batch is an append, so in the async ordering +-- every batch is staged inside the statement and the table's one claim is +-- taken at the transaction's commit; in the stock ordering the claim comes +-- before the first batch. The state is NULL until the first row, so an +-- aggregate over no cold rows writes nothing and returns 0. -- ============================================================================ CREATE TYPE coldfront._cold_sink_state AS ( iceberg text, -- the DuckDB ref the batches are written to @@ -965,15 +969,14 @@ BEGIN END; $$; --- One batch to Iceberg, under the table's claim. +-- One batch to Iceberg, in the ordering every cold write uses. CREATE FUNCTION coldfront._cold_sink_flush(s coldfront._cold_sink_state) RETURNS coldfront._cold_sink_state LANGUAGE plpgsql AS $$ BEGIN IF cardinality(s.buf) > 0 THEN - PERFORM coldfront._take_iceberg_claim(s.iceberg); - PERFORM duckdb.raw_query(format('INSERT INTO %s VALUES %s', - s.iceberg, array_to_string(s.buf, ', '))); + PERFORM coldfront._exec_iceberg_with_claim(s.iceberg, + format('INSERT INTO %s VALUES %s', s.iceberg, array_to_string(s.buf, ', '))); s.buf := '{}'; END IF; RETURN s; @@ -1060,7 +1063,7 @@ CREATE AGGREGATE coldfront._cold_sink(text, text, jsonb) ( -- statement — so this runs at top level, in the user's one transaction. -- -- It is the mixed-tier-update shape (hot tier = plain PG, cold tier = one --- duckdb.raw_query under one bakery claim), with rows routed by (current tier, +-- DuckDB statement through _exec_iceberg_with_claim), with rows routed by (current tier, -- e vs cutoff) into four disjoint cases handled separately: -- stay-hot hot, e>=cut : in-place UPDATE of the hot heap. -- hot→cold hot, e 0 THEN cold_sql := format('DELETE FROM %s WHERE (%s) IN (%s)', v_iceberg, v_pk_list, array_to_string(del_arr, ', ')); END IF; @@ -1289,7 +1294,7 @@ BEGIN cold_sql := cold_sql || format('INSERT INTO %s VALUES %s', v_iceberg, array_to_string(ins_arr, ', ')); END IF; IF cold_sql <> '' THEN - PERFORM duckdb.raw_query(cold_sql); + PERFORM coldfront._exec_iceberg_with_claim(v_iceberg, cold_sql); END IF; END; $fn$; @@ -1740,7 +1745,10 @@ $$; -- -- The table's claim is held from the sample to the commit, as the compactor holds -- it across its read and rewrite: a cold write landing in between would carry an --- assignment against the centroids this call replaces. +-- assignment against the centroids this call replaces. An append staged in the +-- async ordering assigns its rows before it claims, so one staged during the +-- training commits after it with the generation it replaced; a retrain picks +-- those rows up. CREATE PROCEDURE coldfront.vector_train( p_schema text, p_table text, @@ -2217,7 +2225,10 @@ $$; -- nothing, and the expression yields NULL: unassigned, which a probe reads -- through the null arm of its predicate rather than missing. A retrain cannot -- interleave with a cold write, since vector_train holds the table's claim from --- its sample to its commit and every cold write serialises on that same claim. +-- its sample to its commit and every cold write serialises on that same claim; +-- an append staged in the async ordering assigns against the generation live +-- when it stages, so one staged during a retrain lands with the generation the +-- retrain replaced, and the next retrain reassigns it. -- -- The formula itself is _vec_nearest_expr, shared with vector_train, which scores -- the generation it has just trained from its own session: one formula, so a @@ -4045,6 +4056,17 @@ CREATE FUNCTION coldfront._enqueue_release(p_ticket bigint) RETURNS void LANGUAGE c AS 'coldfront', 'coldfront_enqueue_release'; +-- C-bridge for the async ordering's appends: queue the table's claim for the +-- same XactCallback to take at XACT_EVENT_PRE_COMMIT, through +-- _take_iceberg_claim. coldfront registers its callback after pg_duckdb and +-- PostgreSQL calls the latest registered first, so the claim is held when +-- pg_duckdb commits the Iceberg transaction at its own PRE_COMMIT and released +-- at COMMIT as above. An ABORT drops the queue: nothing was committed, and +-- nothing was claimed. +CREATE FUNCTION coldfront._enqueue_claim(p_iceberg_table text) +RETURNS void +LANGUAGE c AS 'coldfront', 'coldfront_enqueue_claim'; + -- coldfront._reject_on_standby: refuse a cold-tier mutation on a physical standby. -- PostgreSQL's read-only enforcement covers only its own writes; a cold write leaves -- PG entirely (DuckDB to object storage), so the cold tier relies on this explicit @@ -4096,20 +4118,31 @@ BEGIN concat_ws(E'\n', NULLIF(current_setting('coldfront._claimed'), ''), p_iceberg_ref), true); ELSE - -- A node that runs Spock is a mesh node, and a mesh node serializes cold - -- writes through the bakery alone: without its settings, refuse rather - -- than take the node-local lock, which peers cannot see. - IF EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'spock') THEN - RAISE EXCEPTION 'coldfront: this node runs Spock, so a cold write needs the bakery, and % is not set', - CASE WHEN NULLIF(current_setting('coldfront.loopback_dsn'), '') IS NULL - THEN 'coldfront.loopback_dsn' ELSE 'snowflake.node' END - USING HINT = 'Set coldfront.loopback_dsn to this server''s unix-socket DSN (e.g. ''host=/var/run/postgresql dbname=coldfront user=coldfront'') and snowflake.node to this node''s id.'; - END IF; + PERFORM coldfront._refuse_spock_without_bakery(); PERFORM pg_advisory_xact_lock(hashtext('coldfront_iceberg:' || p_iceberg_ref)); END IF; END; $$; +-- A node that runs Spock is a mesh node, and a mesh node serializes cold writes +-- through the bakery alone: without its settings, refuse rather than take the +-- node-local lock, which peers cannot see. _take_iceberg_claim calls this where +-- it would take the lock, and _exec_iceberg_with_claim calls it before it +-- stages an append whose claim comes at commit, so the refusal names the +-- statement rather than the COMMIT. +CREATE FUNCTION coldfront._refuse_spock_without_bakery() RETURNS void +LANGUAGE plpgsql AS $$ +BEGIN + IF NOT coldfront._bakery_armed() + AND EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'spock') THEN + RAISE EXCEPTION 'coldfront: this node runs Spock, so a cold write needs the bakery, and % is not set', + CASE WHEN NULLIF(current_setting('coldfront.loopback_dsn'), '') IS NULL + THEN 'coldfront.loopback_dsn' ELSE 'snowflake.node' END + USING HINT = 'Set coldfront.loopback_dsn to this server''s unix-socket DSN (e.g. ''host=/var/run/postgresql dbname=coldfront user=coldfront'') and snowflake.node to this node''s id.'; + END IF; +END; +$$; + -- Is the async-parquet upload ordering BOTH requested AND safe to use? -- coldfront.iceberg_async_parquet asks to stage parquet OUTSIDE the bakery claim -- (writers overlap on S3); that is correct ONLY when the loaded duckdb-iceberg @@ -4134,53 +4167,48 @@ $$; -- chokepoint every cold write -- routes through, in every deployment. -- --- The serializer is fundamentally a per-Iceberg-table mutex that scales by --- deployment, chosen by the v_armed gate: --- --- * Multi-node mesh (v_armed): the Ricart-Agrawala bakery. The upload ordering --- adapts to the loaded duckdb-iceberg via coldfront.iceberg_async_parquet — --- NEVER a 409 either way, only overlap-vs-serialized upload: --- - patched iceberg (coldfront.iceberg_async_parquet ON AND the build --- marker coldfront.iceberg_bakery_patch ON): stage the parquet FIRST --- (writers overlap freely on S3), then claim the bakery only to wrap --- pg_duckdb's deferred commit POST — the patch re-stamps parent_snapshot_id --- at that POST, so the overlap is safe. --- - stock iceberg (the default — OR async REQUESTED without the --- iceberg_bakery_patch marker, which fails safe to here): claim the bakery --- FIRST, then stage+commit inside the held ticket — stock stamps --- parent_snapshot_id at stage time, so the upload must be serialized or a --- peer captures a stale parent and 409s. _iceberg_async_active() gates the --- async path on BOTH GUCs, so an unpatched deployment that flips only the --- async flag can never silently 409 — it lands here and warns once. --- The release is enqueued for the C XactCallback (fires on COMMIT and ABORT), --- so an in-ticket staging failure can't orphan the claim. +-- The serializer is a per-Iceberg-table mutex that _take_iceberg_claim picks by +-- deployment: the Ricart-Agrawala bakery on a mesh node, a transaction-scoped +-- local advisory lock on a vanilla node. Which one it is does not change the +-- ordering here; the loaded duckdb-iceberg and the statement do: -- --- * Vanilla single-node / no mesh (NOT v_armed): a transaction-scoped LOCAL --- advisory lock is the same mutex without any Spock/snowflake/loopback --- dependency. Taken BEFORE staging so a second backend on this node blocks --- before it captures parent_snapshot_id (no stale-parent 409); auto-released --- at commit. This is the path for plain PostgreSQL tiered deployments, which --- have no snowflake.node / loopback_dsn configured. +-- * An append (p_sql is an INSERT: the hook's cold INSERT, the archiver's +-- export, the cold sink's batches) on patched iceberg +-- (coldfront.iceberg_async_parquet ON AND the build marker +-- coldfront.iceberg_bakery_patch ON) is staged FIRST, so writers on one node +-- and across nodes overlap freely on S3, and the claim is queued for the C +-- XactCallback to take at PRE_COMMIT, ahead of pg_duckdb's commit POST, so +-- the serializer wraps that POST alone and an open transaction holds no +-- claim. The patch re-stamps parent_snapshot_id at the POST, under the +-- claim, and a new data file conflicts with nothing a peer or a compaction +-- commits in between (docs/formal Bakery_async.cfg, +-- Bakery_async_samenode.cfg and Bakery_async_single.cfg). +-- * A write that references rows already in the table (DELETE, UPDATE, MERGE, +-- the cross-tier move's bundle) takes the claim FIRST, whatever the build: +-- its position deletes name data files by path, and a compaction or another +-- such write committing between its scan and its commit would rewrite or +-- delete those files, after which the deletes apply to nothing and the rows +-- come back. Under the claim the scan and the commit see one head. +-- * Stock iceberg (the default, OR async REQUESTED without the +-- iceberg_bakery_patch marker, which fails safe to here) takes the claim +-- FIRST for every write: stock stamps parent_snapshot_id at stage time, so +-- the upload must be serialized or a second writer captures a stale parent +-- and 409s. _iceberg_async_active() gates the async path on BOTH GUCs, so +-- an unpatched deployment that flips only the async flag can never silently +-- 409: it lands here and logs once. -- --- _bakery_armed() probes the two GUCs the R-A bakery requires. current_setting(...,true) --- returns NULL for an unrecognised GUC, so the probe is safe with no snowflake --- extension loaded. +-- On a mesh the release is enqueued for the C XactCallback (fires on COMMIT and +-- ABORT), so an in-ticket staging failure can't orphan the claim; the advisory +-- lock ends with the transaction. CREATE FUNCTION coldfront._exec_iceberg_with_claim( p_iceberg_table text, p_sql text ) RETURNS void LANGUAGE plpgsql AS $$ DECLARE - -- v_armed drives BOTH the acquire (via _take_iceberg_claim) and the async - -- vs stock upload ordering below, so it stays a local here. - v_armed boolean := coldfront._bakery_armed(); - -- Async-parquet upload ordering: stage the parquet OUTSIDE the claim (writers - -- overlap on S3), then take the claim only to wrap pg_duckdb's deferred commit - -- POST. Correct ONLY on the bakery-aware duckdb-iceberg build (it re-stamps - -- parent_snapshot_id at the POST, under the claim). _iceberg_async_active() is - -- TRUE only when BOTH coldfront.iceberg_async_parquet AND the build marker - -- coldfront.iceberg_bakery_patch are on; otherwise the stock ordering below is - -- used (always safe). Never a 409 either way. v_async boolean := coldfront._iceberg_async_active(); + -- Every statement here is generated by coldfront, and an append starts + -- with its verb; a DELETE-led bundle or an UPDATE does not. + v_append boolean := upper(ltrim(p_sql)) LIKE 'INSERT %'; BEGIN -- A cluster assignment reads the centroids over pglocal, so a statement that -- names it needs pglocal attached in this backend. Tested on the statement @@ -4190,8 +4218,8 @@ BEGIN PERFORM coldfront.ensure_pg_attached(); END IF; -- This wrapper is the single-statement cold path; the cold sink - -- (_cold_sink_step) and _cross_tier_move issue their own cold writes and - -- carry the same guard. A hot write hits a PG heap, which PG rejects natively. + -- (_cold_sink_step) issues its own cold writes and carries the same guard. + -- A hot write hits a PG heap, which PG rejects natively. PERFORM coldfront._reject_on_standby('execute a cold (Iceberg) write'); -- Fail-safe, not fail-silent: if async was REQUESTED but the bakery-aware -- patch is not asserted, we use the stock ordering (always safe) and note it @@ -4207,15 +4235,13 @@ BEGIN RAISE LOG 'coldfront: iceberg_async_parquet is on but iceberg_bakery_patch is not set — the loaded duckdb-iceberg is not the bakery-aware build; using the SAFE stock upload ordering instead of async. Set coldfront.iceberg_bakery_patch=on ONLY where duckdb-iceberg carries the bakery-aware-commit-refresh patch (the coldfront patched images set both GUCs).'; PERFORM set_config('coldfront._async_downgrade_warned', 'on', false); END IF; - IF v_armed AND v_async THEN - -- Patched iceberg: upload parquet in the background, then take the claim - -- only to wrap pg_duckdb's deferred commit POST. + IF v_async AND v_append THEN + -- Patched iceberg: stage the parquet now, claim at PRE_COMMIT. + PERFORM coldfront._refuse_spock_without_bakery(); PERFORM duckdb.raw_query(p_sql); - PERFORM coldfront._take_iceberg_claim(p_iceberg_table); + PERFORM coldfront._enqueue_claim(p_iceberg_table); ELSE - -- Stock (armed) or vanilla: acquire FIRST (mesh R-A claim or local - -- advisory, chosen inside the helper), then upload+commit inside it so no - -- peer can capture a stale parent_snapshot_id. + -- Acquire FIRST, then stage+commit inside the claim. PERFORM coldfront._take_iceberg_claim(p_iceberg_table); PERFORM duckdb.raw_query(p_sql); END IF; diff --git a/extension/coldfront/src/coldfront.c b/extension/coldfront/src/coldfront.c index 7ab94d9..0d8ee59 100644 --- a/extension/coldfront/src/coldfront.c +++ b/extension/coldfront/src/coldfront.c @@ -77,6 +77,7 @@ #include "parser/parse_type.h" #include "parser/parsetree.h" #include "storage/fd.h" +#include "storage/proc.h" #include "storage/procarray.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" @@ -86,9 +87,11 @@ #include "utils/guc.h" #include "utils/lsyscache.h" #include "utils/memutils.h" +#include "utils/snapmgr.h" #include "utils/regproc.h" #include "utils/ruleutils.h" #include "utils/syscache.h" +#include "utils/timeout.h" #include "utils/timestamp.h" #include "fmgr.h" #include "libpq-fe.h" @@ -4216,8 +4219,19 @@ coldfront_post_parse_analyze(ParseState *pstate, Query *query, * * Allocated in TopMemoryContext so it survives across PG xacts within one * backend session. + * + * The async ordering's appends queue their CLAIM the same way + * (coldfront._enqueue_claim): the statement stages its parquet and the + * callback takes the queued claims at XACT_EVENT_PRE_COMMIT, through + * coldfront._take_iceberg_claim over SPI, while the transaction is still open + * and before pg_duckdb's own PRE_COMMIT callback commits the Iceberg + * transaction (the same registration order as above). So the serializer is + * held for the commit POST alone, and an open transaction with staged cold + * writes blocks no other writer. ABORT drops the queue: nothing was committed + * and nothing was claimed. */ static List *coldfront_pending_releases = NIL; +static List *coldfront_pending_claims = NIL; /* of char *: Iceberg refs */ /* The node's loopback: one libpq connection per backend, opened lazily from the * GUC coldfront.loopback_dsn and kept for the backend's lifetime. Every bakery @@ -4383,14 +4397,82 @@ cf_release_one_ticket(int64 ticket) PQclear(res); } +/* Whether the statement that ends this transaction had statement_timeout + * running, by the rule enable_statement_timeout applies and the timer's own + * start time: PostgreSQL enables the timer in start_xact_command, so a timer + * started inside this transaction belongs to its latest statement, and a + * backend that commits without one (a background worker) never started it. A + * timer still running belongs to the CALL a procedure's COMMIT runs under, + * which keeps it. */ +static bool +cf_statement_timeout_applies(void) +{ + TimestampTz started = get_timeout_start_time(STATEMENT_TIMEOUT); + + return StatementTimeout > 0 +#if PG_VERSION_NUM >= 170000 + && (TransactionTimeout == 0 || StatementTimeout < TransactionTimeout) +#endif + && !get_timeout_active(STATEMENT_TIMEOUT) + && started > 0 && started >= GetCurrentTransactionStartTimestamp(); +} + +/* cf_take_pending_claims takes the claims the async ordering's appends queued, + * at PRE_COMMIT: the transaction is still open, so SPI runs here under a + * snapshot pushed for it (no statement is active at this point), and the ERROR + * a timeout or a refused claim raises aborts the transaction, whose ABORT + * event then drops the queue. PostgreSQL switches statement_timeout off before + * commit processing (finish_xact_command), so the wait for the claim, whose + * protocol has no timeout of its own, would be the one part of the statement + * the setting does not bound: the timer runs again, at the statement's own + * deadline, for the claims alone. */ +static void +cf_take_pending_claims(void) +{ + ListCell *lc; + bool bounded; + + if (coldfront_pending_claims == NIL) + return; + bounded = cf_statement_timeout_applies(); + if (bounded) + enable_timeout_at(STATEMENT_TIMEOUT, get_timeout_finish_time(STATEMENT_TIMEOUT)); + PushActiveSnapshot(GetTransactionSnapshot()); + if (SPI_connect() != SPI_OK_CONNECT) + elog(ERROR, "coldfront: SPI_connect failed while taking the cold writes' claims"); + foreach(lc, coldfront_pending_claims) + { + Oid argtypes[1] = { TEXTOID }; + Datum values[1] = { CStringGetTextDatum((char *) lfirst(lc)) }; + + if (SPI_execute_with_args("SELECT coldfront._take_iceberg_claim($1)", + 1, argtypes, values, NULL, false, 0) != SPI_OK_SELECT) + elog(ERROR, "coldfront: taking the claim on %s failed", (char *) lfirst(lc)); + } + SPI_finish(); + PopActiveSnapshot(); + if (bounded) + disable_timeout(STATEMENT_TIMEOUT, false); + list_free_deep(coldfront_pending_claims); + coldfront_pending_claims = NIL; +} + static void coldfront_xact_callback(XactEvent event, void *arg) { ListCell *lc; + if (event == XACT_EVENT_PRE_COMMIT) + { + cf_take_pending_claims(); + return; + } if (event != XACT_EVENT_COMMIT && event != XACT_EVENT_ABORT) return; + list_free_deep(coldfront_pending_claims); + coldfront_pending_claims = NIL; + /* The registry snapshot's context is a child of TopTransactionContext, * which this transaction's end frees. Drop the pointers with it, so the * next transaction reloads rather than matching a repeated command id. */ @@ -4464,6 +4546,26 @@ coldfront_enqueue_release(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +/* Queue an Iceberg table's claim for the transaction callback to take at + * PRE_COMMIT; a table queued twice is claimed once. */ +PG_FUNCTION_INFO_V1(coldfront_enqueue_claim); +Datum +coldfront_enqueue_claim(PG_FUNCTION_ARGS) +{ + char *table = text_to_cstring(PG_GETARG_TEXT_PP(0)); + MemoryContext old; + ListCell *lc; + + foreach(lc, coldfront_pending_claims) + if (strcmp((char *) lfirst(lc), table) == 0) + PG_RETURN_VOID(); + old = MemoryContextSwitchTo(TopMemoryContext); + coldfront_pending_claims = lappend(coldfront_pending_claims, pstrdup(table)); + MemoryContextSwitchTo(old); + + PG_RETURN_VOID(); +} + /* ---------- DDL synchronization (ProcessUtility_hook) ----------------- */ /* diff --git a/extension/coldfront/test/expected/async_write_before_claim.out b/extension/coldfront/test/expected/async_write_before_claim.out new file mode 100644 index 0000000..dc8f132 --- /dev/null +++ b/extension/coldfront/test/expected/async_write_before_claim.out @@ -0,0 +1,152 @@ +-- Where a cold write's claim sits relative to its DuckDB statement. With the +-- async ordering (coldfront.iceberg_async_parquet and coldfront.iceberg_bakery_patch +-- both on) an append is staged inside its statement and its claim is taken at +-- COMMIT; a write that references rows already in the table, and every write on +-- the stock ordering, takes the claim first. Observed end to end: a DuckDB +-- temporary table stands in for the Iceberg table, another backend (the bakery's +-- loopback, pointed at this server) holds the table's advisory lock, and +-- lock_timeout ends the claim. pg_duckdb reports each DuckDB write as a NOTICE, +-- so a write that ran before the claim shows and one the claim preceded does not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SET coldfront.local_pg_dsn = ''; +SET coldfront.iceberg_async_parquet = on; +SET coldfront.iceberg_bakery_patch = on; +SET lock_timeout = '500ms'; +SELECT set_config('coldfront.loopback_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS loopback_set; + loopback_set +-------------- + t +(1 row) + +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_order (id INTEGER, ts TIMESTAMPTZ, status VARCHAR)') IS NOT NULL AS created; +NOTICE: result: Count +BIGINT +[ Rows: 0] + + + created +--------- + t +(1 row) + +SELECT coldfront._loopback(format('SELECT pg_advisory_lock(hashtext(%L))', + 'coldfront_iceberg:temp.main.cf_order')) IS NOT NULL AS lock_held_elsewhere; + lock_held_elsewhere +--------------------- + t +(1 row) + +-- (1) An append on the async ordering: the write runs in the statement, and +-- the claim at COMMIT meets the held lock. +BEGIN; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_order', + 'INSERT INTO temp.main.cf_order VALUES (1, ''2026-01-01'', ''async'')'); +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + _exec_iceberg_with_claim +-------------------------- + +(1 row) + +COMMIT; +ERROR: canceling statement due to lock timeout +CONTEXT: SQL statement "SELECT pg_advisory_xact_lock(hashtext('coldfront_iceberg:' || p_iceberg_ref))" +PL/pgSQL function _take_iceberg_claim(text) line 13 at PERFORM +SQL statement "SELECT coldfront._take_iceberg_claim($1)" +-- (2) A DELETE references rows the table holds, so the claim comes first and +-- nothing is written. +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_order', + 'DELETE FROM temp.main.cf_order WHERE id = 1'); +ERROR: canceling statement due to lock timeout +CONTEXT: SQL statement "SELECT pg_advisory_xact_lock(hashtext('coldfront_iceberg:' || p_iceberg_ref))" +PL/pgSQL function _take_iceberg_claim(text) line 13 at PERFORM +SQL statement "SELECT coldfront._take_iceberg_claim(p_iceberg_table)" +PL/pgSQL function _exec_iceberg_with_claim(text,text) line 40 at PERFORM +-- (3) The stock ordering claims first for an append too. +SET coldfront.iceberg_async_parquet = off; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_order', + 'INSERT INTO temp.main.cf_order VALUES (2, ''2026-01-01'', ''stock'')'); +ERROR: canceling statement due to lock timeout +CONTEXT: SQL statement "SELECT pg_advisory_xact_lock(hashtext('coldfront_iceberg:' || p_iceberg_ref))" +PL/pgSQL function _take_iceberg_claim(text) line 13 at PERFORM +SQL statement "SELECT coldfront._take_iceberg_claim(p_iceberg_table)" +PL/pgSQL function _exec_iceberg_with_claim(text,text) line 40 at PERFORM +SET coldfront.iceberg_async_parquet = on; +-- (4) The cold sink: a tiered INSERT's cold rows, one row per batch, are all +-- staged in the statement; the claim at COMMIT meets the lock. +CREATE TABLE public._cf_order (id int PRIMARY KEY, ts timestamptz, status text); +CREATE VIEW public.cf_order AS SELECT * FROM public._cf_order; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'cf_order', 'public._cf_order', 'temp.main.cf_order', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'cf_order', '2026-03-01'::timestamptz); +SET coldfront.cold_write_batch_size = 1; +BEGIN; +INSERT INTO public.cf_order (id, ts, status) +VALUES (10, '2026-01-01', 'sink'), (11, '2026-01-02', 'sink'), (12, '2026-01-03', 'sink'); +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 0 | 3 +(1 row) + +COMMIT; +ERROR: canceling statement due to lock timeout +CONTEXT: SQL statement "SELECT pg_advisory_xact_lock(hashtext('coldfront_iceberg:' || p_iceberg_ref))" +PL/pgSQL function _take_iceberg_claim(text) line 13 at PERFORM +SQL statement "SELECT coldfront._take_iceberg_claim($1)" +-- (5) The cross-tier move removes rows by key and re-adds what it read, so its +-- claim comes before its reads: no write happens. +INSERT INTO public._cf_order VALUES (20, '2026-05-01', 'hot'); +UPDATE public.cf_order SET ts = '2026-01-05' WHERE id = 20; +ERROR: canceling statement due to lock timeout +CONTEXT: SQL statement "SELECT pg_advisory_xact_lock(hashtext('coldfront_iceberg:' || p_iceberg_ref))" +PL/pgSQL function _take_iceberg_claim(text) line 13 at PERFORM +SQL statement "SELECT coldfront._take_iceberg_claim(v_iceberg)" +PL/pgSQL function _cross_tier_move(text,text,text,text) line 134 at PERFORM +-- Cleanup. +SELECT coldfront._loopback('SELECT pg_advisory_unlock_all()') IS NOT NULL AS lock_released; + lock_released +--------------- + t +(1 row) + +DELETE FROM coldfront.tiered_views; +DELETE FROM coldfront.archive_watermark; +DROP VIEW public.cf_order; +DROP TABLE public._cf_order; +RESET coldfront.cold_write_batch_size; +RESET lock_timeout; +RESET coldfront.iceberg_async_parquet; +RESET coldfront.iceberg_bakery_patch; +RESET coldfront.loopback_dsn; diff --git a/extension/coldfront/test/sql/async_write_before_claim.sql b/extension/coldfront/test/sql/async_write_before_claim.sql new file mode 100644 index 0000000..6dd797e --- /dev/null +++ b/extension/coldfront/test/sql/async_write_before_claim.sql @@ -0,0 +1,77 @@ +-- Where a cold write's claim sits relative to its DuckDB statement. With the +-- async ordering (coldfront.iceberg_async_parquet and coldfront.iceberg_bakery_patch +-- both on) an append is staged inside its statement and its claim is taken at +-- COMMIT; a write that references rows already in the table, and every write on +-- the stock ordering, takes the claim first. Observed end to end: a DuckDB +-- temporary table stands in for the Iceberg table, another backend (the bakery's +-- loopback, pointed at this server) holds the table's advisory lock, and +-- lock_timeout ends the claim. pg_duckdb reports each DuckDB write as a NOTICE, +-- so a write that ran before the claim shows and one the claim preceded does not. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; + +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SET coldfront.local_pg_dsn = ''; +SET coldfront.iceberg_async_parquet = on; +SET coldfront.iceberg_bakery_patch = on; +SET lock_timeout = '500ms'; +SELECT set_config('coldfront.loopback_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS loopback_set; +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_order (id INTEGER, ts TIMESTAMPTZ, status VARCHAR)') IS NOT NULL AS created; +SELECT coldfront._loopback(format('SELECT pg_advisory_lock(hashtext(%L))', + 'coldfront_iceberg:temp.main.cf_order')) IS NOT NULL AS lock_held_elsewhere; + +-- (1) An append on the async ordering: the write runs in the statement, and +-- the claim at COMMIT meets the held lock. +BEGIN; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_order', + 'INSERT INTO temp.main.cf_order VALUES (1, ''2026-01-01'', ''async'')'); +COMMIT; + +-- (2) A DELETE references rows the table holds, so the claim comes first and +-- nothing is written. +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_order', + 'DELETE FROM temp.main.cf_order WHERE id = 1'); + +-- (3) The stock ordering claims first for an append too. +SET coldfront.iceberg_async_parquet = off; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_order', + 'INSERT INTO temp.main.cf_order VALUES (2, ''2026-01-01'', ''stock'')'); +SET coldfront.iceberg_async_parquet = on; + +-- (4) The cold sink: a tiered INSERT's cold rows, one row per batch, are all +-- staged in the statement; the claim at COMMIT meets the lock. +CREATE TABLE public._cf_order (id int PRIMARY KEY, ts timestamptz, status text); +CREATE VIEW public.cf_order AS SELECT * FROM public._cf_order; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'cf_order', 'public._cf_order', 'temp.main.cf_order', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'cf_order', '2026-03-01'::timestamptz); +SET coldfront.cold_write_batch_size = 1; +BEGIN; +INSERT INTO public.cf_order (id, ts, status) +VALUES (10, '2026-01-01', 'sink'), (11, '2026-01-02', 'sink'), (12, '2026-01-03', 'sink'); +COMMIT; + +-- (5) The cross-tier move removes rows by key and re-adds what it read, so its +-- claim comes before its reads: no write happens. +INSERT INTO public._cf_order VALUES (20, '2026-05-01', 'hot'); +UPDATE public.cf_order SET ts = '2026-01-05' WHERE id = 20; + +-- Cleanup. +SELECT coldfront._loopback('SELECT pg_advisory_unlock_all()') IS NOT NULL AS lock_released; +DELETE FROM coldfront.tiered_views; +DELETE FROM coldfront.archive_watermark; +DROP VIEW public.cf_order; +DROP TABLE public._cf_order; +RESET coldfront.cold_write_batch_size; +RESET lock_timeout; +RESET coldfront.iceberg_async_parquet; +RESET coldfront.iceberg_bakery_patch; +RESET coldfront.loopback_dsn; From a428d97f9ba3e776b3ec48f3faacc4e502e41bca Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Thu, 8 Oct 2026 15:30:28 +0100 Subject: [PATCH 2/9] feat: stream a tiered INSERT's cold rows through the other session --- docs/architecture.md | 18 +- docs/architecture_tiered.md | 114 +++- docs/architecture_vectors.md | 6 +- docs/changelog.md | 16 +- docs/usage.md | 27 +- extension/coldfront/Makefile | 2 +- extension/coldfront/coldfront--1.0.sql | 111 +++- extension/coldfront/src/coldfront.c | 472 ++++++++++++--- .../test/expected/array_cold_render.out | 31 +- .../test/expected/array_cold_render_1.out | 31 +- .../expected/async_write_before_claim.out | 44 ++ .../test/expected/param_cold_via_plpgsql.out | 41 +- .../expected/param_cold_via_plpgsql_1.out | 41 +- .../expected/tiered_insert_single_pass.out | 77 ++- .../expected/tiered_insert_single_pass_1.out | 77 ++- .../test/expected/tiered_insert_stream.out | 557 ++++++++++++++++++ .../test/expected/tiered_insert_stream_1.out | 557 ++++++++++++++++++ .../test/sql/async_write_before_claim.sql | 15 + .../test/sql/param_cold_via_plpgsql.sql | 8 +- .../test/sql/tiered_insert_single_pass.sql | 18 +- .../test/sql/tiered_insert_stream.sql | 151 +++++ 21 files changed, 2133 insertions(+), 281 deletions(-) create mode 100644 extension/coldfront/test/expected/tiered_insert_stream.out create mode 100644 extension/coldfront/test/expected/tiered_insert_stream_1.out create mode 100644 extension/coldfront/test/sql/tiered_insert_stream.sql diff --git a/docs/architecture.md b/docs/architecture.md index 3a7460c..4935eea 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -651,14 +651,16 @@ pg_duckdb has a fully-native, in-process Postgres-table reader for analytics on PG heap data, but that machinery is **not reachable** from the write path into an attached Iceberg catalog. -As the workarounds in use today, a tiered `INSERT` renders its cold rows in the -backend (`coldfront._cold_sink`) and writes them to Iceberg in batched -`duckdb.raw_query` `INSERT`s, and a decoupled `INSERT … SELECT` from a -PostgreSQL table reads it through the DuckDB `postgres` extension's -`pglocal..
` ATTACH, pipelining rows over libpq (loopback) → -DuckDB executor → Iceberg writer → S3. Neither is the in-process reader: -the first pays for the per-row rendering, the second for the libpq round-trip -per row batch. +As the workarounds in use today, a tiered `INSERT` streams its cold rows +through `postgres_query('pgstream', …)` and a decoupled `INSERT … SELECT` +from a PostgreSQL table reads it through the `pglocal..
` +ATTACH, both of the DuckDB `postgres` extension, pipelining rows over libpq +(loopback) → DuckDB executor → Iceberg writer → S3; a tiered `INSERT` whose +source cannot be read from a second session renders its cold rows in the +backend (`coldfront._cold_sink`) and writes them in batched `duckdb.raw_query` +`INSERT`s. None is the in-process reader: the streams pay for the libpq +round-trip per row batch and for reading the source once per tier, the sink +for the per-row rendering. The desired end-state is a way to drive the native in-process reader straight into the Iceberg writer - e.g. a `COPY` form: diff --git a/docs/architecture_tiered.md b/docs/architecture_tiered.md index 1d75890..2f38438 100644 --- a/docs/architecture_tiered.md +++ b/docs/architecture_tiered.md @@ -31,8 +31,9 @@ catalog, the object store, and the archiver: │ coldfront extension: post_parse_analyze_hook │ │ ├── INSERT: splits hot/cold by partition_col vs cutoff; │ │ │ hot side is plain set-based PG INSERT into _events,│ -│ │ cold side is the cold sink, batched DuckDB INSERTs │ -│ │ under one claim (the _cold_sink aggregate) │ +│ │ cold side streams into the Iceberg writer, or goes │ +│ │ through the cold sink when the source cannot be │ +│ │ read twice (the _cold_sink aggregate) │ │ ├── UPDATE/DELETE: classifies WHERE against the watermark│ │ │ and rewrites to target one tier or both │ │ └── errors on ambiguous predicates in strict mode │ @@ -109,8 +110,8 @@ CREATE OR REPLACE VIEW events AS The rename is conditional, so it converts the table on the first run and no-ops afterwards; the `CREATE OR REPLACE VIEW` keeps the view's OID across runs. The C hook rewrites an `INSERT` on the view by the watermark cutoff, into a hot -`INSERT` of the at/after-cutoff rows into `_events` and the cold sink for the -older rows, and the utility hook feeds a `COPY FROM` into the same rewrite in +`INSERT` of the at/after-cutoff rows into `_events` and a cold stream, or the +cold sink, for the older rows, and the utility hook feeds a `COPY FROM` into the same rewrite in batches. An `INSERT` nested in a `WITH` entry is rewritten in place, and a `MERGE` runs on the tier its `ON` condition bounds. The view has no `INSTEAD OF` trigger, so a write the hook does not handle fails in PostgreSQL. @@ -283,15 +284,87 @@ The `post_parse_analyze_hook` (see the [Application Interface](architecture.md#application-interface) section of the Architecture overview) intercepts `INSERT` on a registered tiered view and rewrites it into a single statement that splits the input by the -partition-column watermark: +partition-column watermark. The hot half is always a plain set-based +`INSERT INTO _events`, in which IDENTITY and DEFAULT columns fill server-side. +The cold half takes one of two shapes. + +The stream, the default: ```sql INSERT INTO events (ts, status, data) SELECT ts, status, data FROM staging; -- Rewritten by the hook to (schematically): +WITH coldfront_hot AS MATERIALIZED ( + INSERT INTO _events (ts, status, data) + SELECT ts::timestamptz, status::text, data::jsonb + FROM () AS coldfront_src(ts, status, data) + WHERE coldfront_src.ts::timestamptz >= ''::timestamptz + RETURNING 1 +), +coldfront_cold AS MATERIALIZED ( + SELECT coldfront._tiered_cold_stream('public', 'events', + 'SELECT nextval(''public._events_id_seq''::regclass) AS id, ts::timestamptz AS ts, + status::text AS status, (data::jsonb)::text AS data + FROM () AS coldfront_src(ts, status, data) + WHERE coldfront_src.ts::timestamptz < ''''::timestamptz', + ARRAY['public.staging'::regclass], + EXISTS (SELECT 1 FROM () AS coldfront_src(ts, status, data) + WHERE coldfront_src.ts::timestamptz < ''::timestamptz)) AS n +) +SELECT (SELECT count(*) FROM coldfront_hot) AS hot_rows, + (SELECT n FROM coldfront_cold) AS cold_rows; +``` + +`coldfront._tiered_cold_stream` runs one DuckDB statement, +`INSERT INTO ice.public.events SELECT ... FROM postgres_query('pgstream', )`, through `coldfront._exec_iceberg_with_claim`: the cold projection +runs in PostgreSQL, in a second session the DuckDB `postgres` extension holds +open as `pgstream`, and its rows go over libpq straight into the Iceberg +writer in one vectorized pass, with no row rendered in the backend. The +projection names every hot-table column, with `nextval()` on the hot table's +sequence for an omitted IDENTITY column and the DEFAULT expression for an +omitted column that has one, so cold ids share the hot side's sequence. A +vector column is sent as `real[]`, which the extension maps to the Iceberg +`list`, and a `jsonb`, `json` or `interval` column as its text, the +form the Iceberg `VARCHAR` column holds. On a clustered table the cluster +assignment leads the select list and orders the write, so each row group holds +roughly one cluster. The last argument is the cold test: for a `SELECT` source +an `EXISTS` over it, for a `VALUES` source the hot count against the row count, +so an `INSERT` whose rows are all hot stages nothing and takes no claim. +`cold_rows` is `NULL` when the rows streamed, since DuckDB keeps that count, +and the count otherwise: `0` for an all-hot `INSERT`, the sink's count after +the fallback below. + +The query is deparsed with an empty `search_path`, so every relation, function +and operator is schema-qualified whatever the second session's path is, and +with ISO dates and PostgreSQL-style intervals, so its literals read the same +under that session's `DateStyle` and `IntervalStyle`. `pgstream` is attached at +READ COMMITTED, so each statement's read takes a snapshot of its own and sees +what the hot half saw; the `pglocal` attachment the other cold paths read +through stays at REPEATABLE READ, which the extension needs to give a parallel +table scan one snapshot. The hot half still reads the source as the caller, so +PostgreSQL checks the caller's `SELECT` privilege on it; the second session +reads as the `coldfront.local_pg_dsn` user. + +The stream reads the source twice, once per half, so it is used only when the +hook can prove the second read sees what the first did: the transaction is +READ COMMITTED; the `INSERT` has no `WITH` clause; every function in the source +is IMMUTABLE (a volatile one yields other rows, and a stable one such as +`now()`, `ts + interval '1 day'` or a text-to-timestamptz cast may yield other +values under the second session's settings); the source reads only tables, no +temporary table, no view over one or over such a function, no foreign table; +this transaction has written none of those tables (or their partitions); and +`coldfront.local_pg_dsn` is set. A cached plan runs in transactions the hook +never saw, so the stream checks the isolation level and the written tables +again at execution and, when either fails, runs the same query in the calling +session and feeds its rows to the sink below. + +Every other source takes the sink shape, which reads the source once: + +```sql WITH coldfront_source AS MATERIALIZED ( SELECT ts::timestamptz AS ts, status::text AS status, data::jsonb AS data - FROM () AS s(ts, status, data) + FROM () AS coldfront_src(ts, status, data) ), coldfront_hot AS MATERIALIZED ( INSERT INTO _events (ts, status, data) @@ -310,23 +383,18 @@ SELECT (SELECT count(*) FROM coldfront_hot) AS hot_rows, The source runs once, into the `coldfront_source` tuplestore, and both halves read it, so a volatile source lands every row exactly once and a row the -transaction wrote before the `INSERT` reaches the cold tier. Each column of -`coldfront_source` is cast to the hot table's type, which restates the -coercions PostgreSQL applied when it analyzed the statement (an untyped literal -against a `jsonb` column keeps that type), and `OVERRIDING SYSTEM VALUE` stays -on the hot `INSERT`. - -The hot half is plain set-based `INSERT INTO _events`: IDENTITY and DEFAULT -columns fill server-side at full PG speed. The cold half projects each row to -the hot table's full column list, with `nextval()` on the hot table's sequence -for an omitted IDENTITY column and the DEFAULT expression for an omitted column -that has one, both evaluated by PostgreSQL per row, and hands it to -`coldfront._cold_sink`. That aggregate renders each row as a DuckDB `VALUES` -tuple and flushes every `coldfront.cold_write_batch_size` rows (default 10000) -as one `INSERT` under the table's claim, taken once per table per transaction; -its final step flushes the rest. Throughput is bounded by the per-row rendering -in plpgsql, so for very large mostly-cold seeds, prefer iceberg-only mode where -ids come from the source data. +transaction wrote before the `INSERT` reaches the cold tier. The cold half +projects each row as the stream does and hands it to `coldfront._cold_sink`, +which renders each row as a DuckDB `VALUES` tuple and writes every +`coldfront.cold_write_batch_size` rows (default 10000) as one `INSERT` through +`coldfront._exec_iceberg_with_claim`; its final step writes the rest. The sink +renders in plpgsql, so a large load that cannot stream is slower than one +that can. + +In both shapes each column of the source is cast to the hot table's type, +which restates the coercions PostgreSQL applied when it analyzed the statement +(an untyped literal against a `jsonb` column keeps that type), and +`OVERRIDING SYSTEM VALUE` stays on the hot `INSERT`. A `WITH` clause on the `INSERT` keeps its entries at the top of the rewritten statement, ahead of the three above, so an entry that modifies data (`WITH diff --git a/docs/architecture_vectors.md b/docs/architecture_vectors.md index 10cef48..f93ad6f 100644 --- a/docs/architecture_vectors.md +++ b/docs/architecture_vectors.md @@ -203,7 +203,8 @@ that do so, where each one lives, and its shape: | Path | Where | Shape | |---|---|---| | bulk archive | the Iceberg INSERT in `cmd/archiver`, not the staging SELECT | set-based | -| tiered INSERT, cold half | `coldfront._cold_row_literal`, through the `coldfront._cold_sink` aggregate | per row | +| tiered INSERT, cold half | `coldfront._tiered_cold_stream`, leading its one DuckDB INSERT | set-based | +| tiered INSERT, cold half through the sink | `coldfront._cold_row_literal`, through the `coldfront._cold_sink` aggregate | per row | | cross-tier move | `coldfront._move_row_literal` | per row | | replay drain | `coldfront.replay_archive_delta` | set-based | | decoupled INSERT | the C rewrite | per statement | @@ -518,7 +519,8 @@ The following are properties of the code as it stands, not plans: - `coldfront._cold_sink` writes unsorted: it appends rows in the order the statement delivers them and would need buffering to sort. It is the cold - half of every tiered `INSERT`. + half of a tiered `INSERT` whose source cannot stream; the stream orders its + write by the cluster. - The replay drain (`coldfront.replay_archive_delta`) and the cross-tier move also write without ordering by cluster. diff --git a/docs/changelog.md b/docs/changelog.md index 9f01251..46da9e9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -80,6 +80,14 @@ this project adheres to ### Changed +- A tiered `INSERT` streams its cold rows: the cold projection, identity values + and defaults included, runs in PostgreSQL in a second session and its rows go + straight into the Iceberg writer in one pass, with nothing rendered per row. + A source the hook cannot read a second time (a volatile or stable function, a + temporary table, a `WITH` entry, a table the transaction has written, a + transaction that is not READ COMMITTED) is read once for both tiers and its + cold rows go through `coldfront._cold_sink`. `cold_rows` in the statement's + result is `NULL` for a streamed `INSERT`. - Every append to the cold tier (`INSERT`, `COPY`, the archiver's exports, the tiered `INSERT`'s cold sink) uses the async ordering wherever the patched duckdb-iceberg is loaded, on a single node as on a mesh: the statement @@ -204,9 +212,11 @@ this project adheres to rows differed between the two runs landed some rows in both tiers and others in neither, and when the hot table had no identity column, or the statement supplied one, a source table written earlier in the same transaction lost - its cold rows, with no error either way. The source now runs once and both - tiers read that result; an untyped literal in the source keeps the target - column's type, and `OVERRIDING SYSTEM VALUE` works. + its cold rows, with no error either way. Such a source now runs once and + both tiers read that result; a source is read a second time, by the cold + stream, only where the hook proves the second read sees the same rows. An + untyped literal in the source keeps the target column's type, and + `OVERRIDING SYSTEM VALUE` works. - On PostgreSQL 17 and 18, a transaction block in which a statement had failed could not be ended in a database with the extension: `ROLLBACK`, `COMMIT` and `ROLLBACK TO SAVEPOINT` failed with "ResourceOwnerEnlarge called after diff --git a/docs/usage.md b/docs/usage.md index 41459a9..8f725fe 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -112,7 +112,9 @@ installation.md show: - `coldfront.local_pg_dsn` is the connection string DuckDB uses to read PostgreSQL tables from this server, which a decoupled table's `INSERT … SELECT` from a PostgreSQL table needs, as does a cold write to a - table with a clustered vector column. Only a superuser can set or read it. + table with a clustered vector column. A tiered `INSERT` streams its cold + rows through it, and renders them row by row when it is not set. Only a + superuser can set or read it. Set it before calling `set_storage_secret()`, which installs the DuckDB `postgres` extension this path loads only when the setting is present. @@ -1112,13 +1114,22 @@ Keep the following caveats in mind when running either mode: `ice.public.` is the Iceberg table - only addressable via `iceberg_scan(...)` or `duckdb.raw_query('… ice.… …')`, never via PG-native 3-part names. -- A tiered `INSERT` writes its cold rows through `coldfront._cold_sink`, which - renders each row in plpgsql and writes Iceberg in batches of - `coldfront.cold_write_batch_size` rows. An omitted IDENTITY column takes - `nextval()` on the hot table's sequence, so cold ids share it with the hot - side, and an omitted column with a DEFAULT takes it. The hot rows are one - set-based `INSERT`. For very large historical seeds (mostly-cold), prefer - iceberg-only mode where ids come from your source data. +- A tiered `INSERT` streams its cold rows: the cold projection runs in + PostgreSQL, in a second session the DuckDB `postgres` extension opens from + `coldfront.local_pg_dsn`, and its rows go straight into the Iceberg writer + in one pass. An omitted IDENTITY column takes `nextval()` on the hot table's + sequence, so cold ids share it with the hot side, and an omitted column with + a DEFAULT takes it. The hot rows are one set-based `INSERT`. The stream + reads the source once per tier, so a source it cannot read twice (a volatile + or stable function such as `now()`, a temporary table, a `WITH` entry, a + table the transaction has written, or a transaction that is not READ + COMMITTED) goes through `coldfront._cold_sink` instead, which renders each + row in plpgsql and writes Iceberg in batches of + `coldfront.cold_write_batch_size` rows, as does a server without + `coldfront.local_pg_dsn`. The statement's `cold_rows` column is `NULL` when + the cold rows streamed. A `DateStyle` whose output is ambiguous to read + back, `SQL` with `DMY`, is not one the second session may differ in: the + stream renders its dates in ISO form. - `COPY FROM` reads the rows with PostgreSQL's `COPY` reader and writes them through that same `INSERT` path, `coldfront.cold_write_batch_size` rows per `INSERT`. The format options are the reader's, and a supplied value for a diff --git a/extension/coldfront/Makefile b/extension/coldfront/Makefile index 29ad732..6b7c574 100644 --- a/extension/coldfront/Makefile +++ b/extension/coldfront/Makefile @@ -30,7 +30,7 @@ REGRESS = load_order update_unregistered_view update_heap_table \ cold_write_json_agg duckdb_temp_dir \ vector_centroids vector_type_map vector_cold_render array_cold_render vector_param_render vector_ops vector_probe vector_status vector_multicolumn \ duckdb_keyword_column partition_clause failed_txn_block tiered_insert_single_pass tiered_view_no_trigger copy_tiered_view param_in_subquery merge_on_view delete_without_where update_from_without_alias settings_registered ensure_replicated read_inner_product \ - async_write_before_claim + async_write_before_claim tiered_insert_stream REGRESS_OPTS = --inputdir=test --outputdir=test PG_CONFIG ?= pg_config diff --git a/extension/coldfront/coldfront--1.0.sql b/extension/coldfront/coldfront--1.0.sql index 0d0cac4..bfc8eed 100644 --- a/extension/coldfront/coldfront--1.0.sql +++ b/extension/coldfront/coldfront--1.0.sql @@ -422,6 +422,31 @@ END; -- non-superuser. search_path pinned per SECURITY DEFINER hardening. $$ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp; +-- ensure_stream_attached() attaches the local instance a second time, as +-- `pgstream`, for the tiered INSERT's cold stream (coldfront._tiered_cold_stream). +-- pglocal runs REPEATABLE READ, which DuckDB's postgres extension needs to +-- give a parallel table scan one snapshot, and that pins the first read's +-- snapshot for the rest of the DuckDB transaction; the stream wants each +-- statement to read what its hot half read, so its attachment runs READ +-- COMMITTED, and a stream is one connection with no parallel scan to keep +-- consistent. The stream's query draws identity values and defaults in that +-- session, so unlike pglocal's uses it writes: nextval() on the hot table's +-- sequences. application_name tags the session and keeps the connection +-- string distinct from pglocal's. +CREATE OR REPLACE FUNCTION coldfront.ensure_stream_attached() RETURNS void AS $$ +DECLARE + dsn text := current_setting('coldfront.local_pg_dsn', true); +BEGIN + IF dsn IS NOT NULL AND dsn <> '' THEN + PERFORM duckdb.raw_query('LOAD postgres'); + PERFORM duckdb.raw_query(format( + 'ATTACH IF NOT EXISTS %L AS pgstream (TYPE postgres, isolation_level ''READ COMMITTED'')', + dsn || ' application_name=coldfront_pgstream' + )); + END IF; +END; +$$ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp; + -- coldfront._hot_only: the per-row guard a hot MERGE's INSERT action gets on its -- partition-column value. The hot MERGE runs in PostgreSQL against the hot table -- alone, so a row below the cutoff, which belongs in Iceberg, is refused rather @@ -498,8 +523,10 @@ BEGIN SELECT p.oid::regprocedure AS sig FROM pg_proc p WHERE p.pronamespace = 'coldfront'::regnamespace - AND p.proname IN ('ensure_attached', 'ensure_pg_attached', + AND p.proname IN ('ensure_attached', 'ensure_pg_attached', 'ensure_stream_attached', '_exec_iceberg_with_claim', + -- the tiered INSERT's cold stream and its execution-time check + '_tiered_cold_stream', '_written_in_xact', -- the tiered INSERT's cold sink and its row renderer '_cold_sink', '_cold_sink_step', '_cold_sink_flush', '_cold_sink_final', '_cold_row_literal', @@ -897,7 +924,10 @@ END; $$; -- ============================================================================ --- The cold sink: the cold half of a tiered-view INSERT. +-- The cold sink: the cold half of a tiered-view INSERT whose source the hook +-- cannot read a second time from another session (emit_tiered_insert in +-- coldfront.c lists the conditions), and the in-session fallback of the cold +-- stream below. -- -- The C hook rewrites INSERT INTO ... into one statement that -- holds the source in a MATERIALIZED CTE, inserts the hot rows into the hot @@ -1054,6 +1084,83 @@ CREATE AGGREGATE coldfront._cold_sink(text, text, jsonb) ( FINALFUNC = coldfront._cold_sink_final ); +-- coldfront._written_in_xact: whether this transaction has written any of +-- these tables, or a partition of one, read from the backend's statistics +-- for the live transaction alone (coldfront.c). The cold stream reads its +-- source from another session, which cannot see such rows; the hook asks +-- before it emits the stream, and _tiered_cold_stream asks again at +-- execution, since a cached plan runs in transactions the hook never saw. +-- Without track_counts the answer is unknown, which counts as written. +CREATE FUNCTION coldfront._written_in_xact(p_rels regclass[]) RETURNS boolean +LANGUAGE c AS 'coldfront', 'coldfront_written_in_xact'; + +-- coldfront._tiered_cold_stream: the cold half of a tiered-view INSERT whose +-- source the hook proved safe to read from another session (emit_tiered_insert +-- in coldfront.c). p_pg_sql is that read: the cold projection, every hot-table +-- column with the identity values and defaults an INSERT would give, over the +-- source below the cutoff, fully qualified. It runs in the pgstream session +-- and its rows go straight into the Iceberg writer: INSERT INTO SELECT +-- ... FROM postgres_query('pgstream', ), one DuckDB statement +-- through _exec_iceberg_with_claim, led by the cluster assignment on a +-- clustered table and ordered by it so this write's row groups each hold +-- roughly one cluster. p_any_cold is false when the source has no cold row, +-- and nothing runs: an all-hot INSERT stages nothing and takes no claim. +-- p_sources are the tables the source reads. A cached plan runs in +-- transactions the hook never saw, so two of its conditions are checked again +-- here: when the transaction is not READ COMMITTED, or has written a source +-- table, the other session's snapshot would not match the hot half's, so the +-- same read runs in this session and its rows go through the cold sink. +-- Returns the cold row count where this session counted it, 0 with no cold +-- row and the sink's count after the fallback, and NULL when the rows +-- streamed, since DuckDB keeps that count. +CREATE FUNCTION coldfront._tiered_cold_stream( + p_schema text, p_view text, p_pg_sql text, p_sources regclass[], p_any_cold boolean +) RETURNS bigint +LANGUAGE plpgsql AS $$ +DECLARE + v_hot text; + v_iceberg text; + v_cols text; + v_prefix text; + v_count bigint; +BEGIN + IF NOT p_any_cold THEN + RETURN 0; + END IF; + IF current_setting('transaction_isolation') <> 'read committed' + OR coldfront._written_in_xact(p_sources) THEN + EXECUTE format('SELECT coldfront._cold_sink(%L, %L, to_jsonb(r)) FROM (%s) AS r', + p_schema, p_view, p_pg_sql) INTO v_count; + RETURN v_count; + END IF; + SELECT tv.hot_table, tv.iceberg_table INTO v_hot, v_iceberg + FROM coldfront.tiered_views tv + WHERE tv.schema_name = p_schema AND tv.relname = p_view + AND NOT tv.is_iceberg_only; + IF v_iceberg IS NULL THEN + RAISE EXCEPTION 'coldfront._tiered_cold_stream: view %.% is not a registered tiered view', + p_schema, p_view; + END IF; + SELECT string_agg('s.' || quote_ident(a.attname), ', ' ORDER BY a.attnum) INTO v_cols + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = (parse_ident(v_hot))[1] + AND c.relname = (parse_ident(v_hot))[2] + AND a.attnum > 0 AND NOT a.attisdropped + AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated); + v_prefix := COALESCE(coldfront._vec_list_prefix_for_ref(v_iceberg, 's.'), ''); + SET LOCAL duckdb.unsafe_allow_mixed_transactions = on; + PERFORM coldfront.ensure_attached(); + PERFORM coldfront.ensure_stream_attached(); + PERFORM coldfront._exec_iceberg_with_claim(v_iceberg, format( + 'INSERT INTO %s SELECT %s%s FROM postgres_query(%L, %L) AS s%s', + v_iceberg, v_prefix, v_cols, 'pgstream', p_pg_sql, + CASE WHEN v_prefix <> '' THEN ' ORDER BY 1' ELSE '' END)); + RETURN NULL; +END; +$$; + -- coldfront._cross_tier_move: execute a partition-column UPDATE that crosses the -- hot/cold cutoff, relocating the matched rows between tiers. The C post-parse- -- analyze hook detects the move (partition column in diff --git a/extension/coldfront/src/coldfront.c b/extension/coldfront/src/coldfront.c index 0d8ee59..5cd89d8 100644 --- a/extension/coldfront/src/coldfront.c +++ b/extension/coldfront/src/coldfront.c @@ -55,6 +55,7 @@ #include "catalog/pg_authid.h" #include "catalog/pg_class.h" #include "catalog/pg_collation.h" +#include "catalog/pg_inherits.h" #include "catalog/pg_namespace.h" #include "catalog/pg_proc.h" #include "catalog/pg_type_d.h" @@ -70,18 +71,22 @@ #include "nodes/params.h" #include "nodes/pg_list.h" #include "miscadmin.h" +#include "pgstat.h" +#include "utils/pgstat_internal.h" #include "optimizer/optimizer.h" #include "optimizer/planner.h" #include "parser/analyze.h" #include "parser/parse_func.h" #include "parser/parse_type.h" #include "parser/parsetree.h" +#include "rewrite/rewriteHandler.h" #include "storage/fd.h" #include "storage/proc.h" #include "storage/procarray.h" #include "tcop/tcopprot.h" #include "tcop/utility.h" #include "utils/acl.h" +#include "utils/array.h" #include "utils/builtins.h" #include "utils/datum.h" #include "utils/guc.h" @@ -306,6 +311,7 @@ typedef struct { static char *insert_targetlist_collist(Query *query); static const char *skip_leading_collist(const char *rest); +static const char *skip_override_clause(const char *rest, OverridingKind override); static char *rewrite_merge_inserts(const char *sql, Query *query, TieredViewInfo *info, bool cold); static char *build_iceberg_only_insert_with_cluster(Query *query, @@ -1509,10 +1515,12 @@ build_cold_dml(DeparseResult *dr, TieredViewInfo *info, Query *query) * real[] (a vector column's view type) -> CAST(%P$L AS FLOAT[]) / * translate($K::text,'{}','[]'); any other array -> %P$s / * coldfront._render_cold_param($K); json/jsonb/interval -> %P$L / $K::text; - * else %P$L / $K. + * else %P$L / $K. With for_pg the string is the tiered stream's read, which + * PostgreSQL runs: every value is %P$L / $K, a literal its own input reads + * back, coerced by the projection's casts. */ static char * -cold_sql_arg(const char *cold_dml, ColdParamSet *ps) +cold_sql_arg(const char *cold_dml, ColdParamSet *ps, bool for_pg) { StringInfoData tmpl, args, out; const char *p = cold_dml; @@ -1560,7 +1568,9 @@ cold_sql_arg(const char *cold_dml, ColdParamSet *ps) if (pos_of_id[id - 1] == 0) /* first sight: assign pos + arg */ { pos_of_id[id - 1] = next_pos++; - if (t == BYTEAOID) + if (for_pg) + appendStringInfo(&args, ", $%d", id); + else if (t == BYTEAOID) /* hex string, independent of the caller's bytea_output; * from_hex rebuilds the exact bytes. */ appendStringInfo(&args, ", encode($%d,'hex')", id); @@ -1580,7 +1590,9 @@ cold_sql_arg(const char *cold_dml, ColdParamSet *ps) else appendStringInfo(&args, ", $%d", id); } - if (t == BYTEAOID) + if (for_pg) + appendStringInfo(&tmpl, "%%%d$L", pos_of_id[id - 1]); + else if (t == BYTEAOID) appendStringInfo(&tmpl, "from_hex(%%%d$L)", pos_of_id[id - 1]); else if (t == FLOAT4ARRAYOID) appendStringInfo(&tmpl, "CAST(%%%d$L AS FLOAT[])", pos_of_id[id - 1]); @@ -1635,13 +1647,13 @@ cold_exec_call(const char *iceberg_table, const char *arg_sql) * full rationale on the table in coldfront--1.0.sql. */ static char * -cold_anchor_update(const char *cold_call_expr) +cold_anchor_update(const char *cold_call_expr, const char *test) { StringInfoData buf; initStringInfo(&buf); appendStringInfo(&buf, - "UPDATE coldfront._dummy_dml_target SET anchor = anchor WHERE %s IS NULL", - cold_call_expr); + "UPDATE coldfront._dummy_dml_target SET anchor = anchor WHERE %s %s", + cold_call_expr, test); return buf.data; } @@ -1886,12 +1898,12 @@ emit_cold(Query *query, TieredViewInfo *info, ColdParamSet *ps, bool in_plpgsql) * them; it self-selects the R-A bakery (multi-node mesh) or a local * advisory lock (vanilla single-node), so this is correct in every * deployment. (Tiered INSERT uses a separate path, emit_tiered_insert.) */ - call = cold_exec_call(info->iceberg_table, cold_sql_arg(cold_dml, ps)); + call = cold_exec_call(info->iceberg_table, cold_sql_arg(cold_dml, ps, false)); /* Cause 2: inside plpgsql the statement must be a DML (cold_anchor_update); * at top level keep the byte-identical SELECT shape. */ if (in_plpgsql) - return cold_anchor_update(call); + return cold_anchor_update(call, "IS NULL"); { StringInfoData buf; initStringInfo(&buf); @@ -1926,7 +1938,7 @@ emit_dual(Query *query, TieredViewInfo *info, ColdParamSet *ps, bool in_plpgsql) deparse_and_find_prefix(query, &dr); hot_dml = build_hot_dml(&dr, info); cold_dml = build_cold_dml(&dr, info, query); - call = cold_exec_call(info->iceberg_table, cold_sql_arg(cold_dml, ps)); + call = cold_exec_call(info->iceberg_table, cold_sql_arg(cold_dml, ps, false)); initStringInfo(&buf); if (in_plpgsql) @@ -1940,7 +1952,7 @@ emit_dual(Query *query, TieredViewInfo *info, ColdParamSet *ps, bool in_plpgsql) * no rows. */ append_with_opener(&buf, &dr); appendStringInfo(&buf, "coldfront_cold AS (%s) %s%s %s%s", - cold_anchor_update(call), dr.verb, info->hot_table, + cold_anchor_update(call, "IS NULL"), dr.verb, info->hot_table, dr.alias, dr.rest); } else @@ -2320,16 +2332,35 @@ build_src_cte(Query *query, const char *source, const char *col_list, } /* - * The cold rows' projection: every hot-table column in attnum order, named, - * so to_jsonb() of the row is what the sink renders. A target column is the - * source's value; an omitted identity column (or any identity column under - * OVERRIDING USER VALUE) takes the next value of its sequence, which the hot - * INSERT shares; an omitted column with a DEFAULT takes that expression, - * evaluated by PostgreSQL per row as a hot INSERT would; any other omitted - * column is NULL. + * The PostgreSQL type a streamed column is sent as, one DuckDB's postgres + * extension maps to the Iceberg column's type. A vector is unknown to it and + * would arrive as text, so it goes as real[]; jsonb, json and interval are + * stored as VARCHAR and go as their text, the form the sink stores too. + */ +static const char * +cf_stream_cast(const char *type) +{ + if (strncmp(type, "vector", 6) == 0 || strncmp(type, "halfvec", 7) == 0) + return "::real[]"; + if (strcmp(type, "jsonb") == 0 || strcmp(type, "json") == 0 || + strcmp(type, "interval") == 0) + return "::text"; + return ""; +} + +/* + * The cold rows' projection: every hot-table column in attnum order, named. + * A target column is the source's value; an omitted identity column (or any + * identity column under OVERRIDING USER VALUE) takes the next value of its + * sequence, which the hot INSERT shares; an omitted column with a DEFAULT + * takes that expression, evaluated by PostgreSQL per row as a hot INSERT + * would; any other omitted column is NULL. For the sink the rows come from + * the materialized source, already cast, and to_jsonb() of the row is what it + * renders; for the stream they come from the source itself, so each is cast + * to its hot type and then to the type it is sent as (cf_stream_cast). */ static char * -build_cold_projection(Query *query, const HotColumns *hc) +build_cold_projection(Query *query, const HotColumns *hc, bool stream) { StringInfoData sel; List *targeted = NIL; @@ -2346,12 +2377,16 @@ build_cold_projection(Query *query, const HotColumns *hc) initStringInfo(&sel); for (i = 0; i < hc->n; i++) { - bool in_target = name_in_list(targeted, hc->name[i]); - char *expr; + bool in_target = name_in_list(targeted, hc->name[i]); + const char *cast = stream ? cf_stream_cast(hc->type[i]) : ""; + char *expr; if (hc->seq[i] != NULL && (!in_target || query->override == OVERRIDING_USER_VALUE)) expr = psprintf("nextval(%s::regclass)", quote_literal_cstr(hc->seq[i])); + else if (in_target && stream) + expr = psprintf("coldfront_src.%s::%s", quote_identifier(hc->name[i]), + hc->type[i]); else if (in_target) expr = psprintf("coldfront_source.%s", quote_identifier(hc->name[i])); else if (hc->dflt[i] != NULL) @@ -2364,6 +2399,8 @@ build_cold_projection(Query *query, const HotColumns *hc) if (pg_str_endswith(hc->type[i], "[]")) expr = psprintf("coldfront._cold_list(%s, %s)", expr, quote_literal_cstr(hc->name[i])); + if (cast[0] != '\0') + expr = psprintf("(%s)%s", expr, cast); appendStringInfo(&sel, "%s%s AS %s", i > 0 ? ", " : "", expr, quote_identifier(hc->name[i])); } @@ -2371,19 +2408,272 @@ build_cold_projection(Query *query, const HotColumns *hc) } /* The hot half: a set-based INSERT into the hot table of the source rows at or - * above the cutoff. */ + * above the cutoff, read from the sink shape's materialized source or, in the + * stream shape, from the source itself (select_list, from_clause). */ static char * build_tiered_hot_dml(const char *hot_table, const char *col_list, - const char *override, const char *partition_col, + const char *override, const char *select_list, + const char *from_clause, const char *pc_expr, const char *cutoff_lit) { - return psprintf("INSERT INTO %s (%s) %sSELECT %s FROM coldfront_source WHERE %s >= %s", - hot_table, col_list, override, col_list, - quote_identifier(partition_col), cutoff_lit); + return psprintf("INSERT INTO %s (%s) %sSELECT %s %s WHERE %s >= %s", + hot_table, col_list, override, select_list, from_clause, + pc_expr, cutoff_lit); +} + +/* + * Whether a tiered INSERT's cold half can stream, and the tables its source + * reads. The stream reads the source again from another session over libpq + * (coldfront._tiered_cold_stream), so it is sound only when that session sees + * what the hot half saw: coldfront.local_pg_dsn names it; the transaction is + * READ COMMITTED, so the hot half's snapshot is the statement's and not an + * older one; the source has no WITH entry; every function in it is IMMUTABLE, + * since a volatile one yields other rows and a stable one (now(), the + * timezone-dependent timestamptz arithmetic, a text-to-timestamptz cast) may + * yield other values in a session with other settings; it reads no temporary + * table, no view that reads one or calls such a function, nothing but tables; + * and this transaction has written none of those tables. Anything else goes + * to the sink shape, which reads the source once for both halves. + */ +typedef struct { + Oid result_relid; /* the view being written, or the view being expanded */ + List *rels; /* every table the source reads, views expanded */ +} StreamCheck; + +static bool cf_stream_blocker_walker(Node *node, void *ctx); + +/* A relation the source reads blocks the stream when the other session cannot + * see it as this one does: a temporary table; a view, through what it reads; + * anything that is not a table. A table is collected for the written-in- + * transaction check. A view's stored query names the view itself in its OLD + * and NEW entries, which are skipped as the target is. */ +static bool +cf_rel_blocks_stream(Oid relid, StreamCheck *sc) +{ + char relkind = get_rel_relkind(relid); + + if (get_rel_persistence(relid) == RELPERSISTENCE_TEMP) + return true; + if (relkind == RELKIND_VIEW) + { + Relation rel = relation_open(relid, AccessShareLock); + Query *vq = get_view_query(rel); + StreamCheck inner = { relid, sc->rels }; + bool blocked; + + blocked = contain_mutable_functions((Node *) vq) || + query_tree_walker(vq, cf_stream_blocker_walker, &inner, + QTW_EXAMINE_RTES_BEFORE); + relation_close(rel, NoLock); + sc->rels = inner.rels; + return blocked; + } + if (relkind != RELKIND_RELATION && relkind != RELKIND_PARTITIONED_TABLE && + relkind != RELKIND_MATVIEW) + return true; + sc->rels = lappend_oid(sc->rels, relid); + return false; +} + +static bool +cf_stream_blocker_walker(Node *node, void *ctx) +{ + StreamCheck *sc = (StreamCheck *) ctx; + + if (node == NULL) + return false; + if (IsA(node, RangeTblEntry)) + { + RangeTblEntry *rte = (RangeTblEntry *) node; + + switch (rte->rtekind) + { + case RTE_RELATION: + return rte->relid != sc->result_relid && + cf_rel_blocks_stream(rte->relid, sc); + case RTE_SUBQUERY: + case RTE_JOIN: + case RTE_FUNCTION: + case RTE_VALUES: + case RTE_RESULT: +#if PG_VERSION_NUM >= 180000 + case RTE_GROUP: +#endif + return false; + default: /* a WITH entry, XMLTABLE, a trigger's transition table */ + return true; + } + } + if (IsA(node, Query)) + return query_tree_walker((Query *) node, cf_stream_blocker_walker, ctx, + QTW_EXAMINE_RTES_BEFORE); + return expression_tree_walker(node, cf_stream_blocker_walker, ctx); +} + +/* The tables as a regclass[] literal: the stream's p_sources. */ +static char * +cf_regclass_array(List *rels) +{ + StringInfoData buf; + ListCell *lc; + + if (rels == NIL) + return "'{}'::regclass[]"; + initStringInfo(&buf); + appendStringInfoString(&buf, "ARRAY["); + foreach(lc, rels) + appendStringInfo(&buf, "%s%u::oid::regclass", + lc == list_head(rels) ? "" : ", ", lfirst_oid(lc)); + appendStringInfoChar(&buf, ']'); + return buf.data; +} + +/* + * Whether this transaction has written any of these tables, or a partition of + * one. The backend's pending statistics entry for a table keeps the live + * transaction's counts on its subtransaction chain and folds them into the + * pending totals at commit, where they stay until the next flush, so only the + * chain says what this transaction did: the totals would also count a + * commit made a moment ago by this backend, and the copy find_tabstat_entry + * returns has the chain detached, so the entry is read in place. Without + * track_counts there is no answer, which counts as written. The hook calls + * this before it emits the stream, and coldfront._written_in_xact is the same + * function for _tiered_cold_stream's check at execution. + */ +static bool +cf_written_in_xact(List *rels) +{ + ListCell *lc; + + if (rels == NIL) + return false; + if (!pgstat_track_counts) + return true; + foreach(lc, rels) + { + List *all = find_all_inheritors(lfirst_oid(lc), NoLock, NULL); + ListCell *lp; + + foreach(lp, all) + { + PgStat_EntryRef *ref = pgstat_fetch_pending_entry(PGSTAT_KIND_RELATION, + MyDatabaseId, lfirst_oid(lp)); + PgStat_TableStatus *ts = ref ? (PgStat_TableStatus *) ref->pending : NULL; + PgStat_TableXactStatus *trans; + + for (trans = ts ? ts->trans : NULL; trans != NULL; trans = trans->upper) + if (trans->tuples_inserted + trans->tuples_updated + trans->tuples_deleted > 0) + return true; + } + } + return false; +} + +PG_FUNCTION_INFO_V1(coldfront_written_in_xact); +Datum +coldfront_written_in_xact(PG_FUNCTION_ARGS) +{ + ArrayType *arr = PG_GETARG_ARRAYTYPE_P(0); + Datum *elems; + bool *nulls; + int n, i; + List *rels = NIL; + + deconstruct_array(arr, REGCLASSOID, sizeof(Oid), true, TYPALIGN_INT, + &elems, &nulls, &n); + for (i = 0; i < n; i++) + if (!nulls[i]) + rels = lappend_oid(rels, DatumGetObjectId(elems[i])); + PG_RETURN_BOOL(cf_written_in_xact(rels)); +} + +static bool +cf_tiered_insert_streams(Query *query, Oid view_relid, List **sources) +{ + StreamCheck sc = { view_relid, NIL }; + + *sources = NIL; + if (coldfront_local_pg_dsn == NULL || coldfront_local_pg_dsn[0] == '\0') + return false; + if (IsolationUsesXactSnapshot() || query->cteList != NIL) + return false; + if (contain_mutable_functions((Node *) query)) + return false; + if (query_tree_walker(query, cf_stream_blocker_walker, &sc, QTW_EXAMINE_RTES_BEFORE)) + return false; + if (cf_written_in_xact(sc.rels)) + return false; + *sources = sc.rels; + return true; +} + +/* The row count of a VALUES source: the list's length, 1 when the one row's + * expressions sit in the target list, 0 for a SELECT source. */ +static int +cf_values_rows(Query *query) +{ + ListCell *lc; + int idx = 0; + + foreach(lc, query->rtable) + { + RangeTblEntry *rte = (RangeTblEntry *) lfirst(lc); + + idx++; + if (idx == query->resultRelation) + continue; + return rte->rtekind == RTE_VALUES ? list_length(rte->values_lists) : 0; + } + return 1; +} + +/* The partition column of the source's derived table, cast to its hot type: + * an untyped literal is text there, which no comparison with the cutoff + * accepts. */ +static char * +cf_pc_expr(TieredViewInfo *info, const HotColumns *hc) +{ + const char *type = hot_column_type(hc, info->partition_col); + + return psprintf("coldfront_src.%s%s%s", quote_identifier(info->partition_col), + type ? "::" : "", type ? type : ""); +} + +/* + * The stream's read, as the SQL-text argument coldfront._tiered_cold_stream + * receives (cold_sql_arg): the cold projection over the source below the + * cutoff. Deparsed with an empty search_path, so every relation, function and + * operator is schema-qualified whatever the other session's path is, and with + * ISO dates and PostgreSQL intervals, so its literals read the same under + * that session's DateStyle and IntervalStyle. + */ +static char * +cf_stream_pg_select(Query *query, TieredViewInfo *info, const HotColumns *hc, + const char *col_list, ColdParamSet *ps) +{ + DeparseResult dr; + const char *source; + char *sql; + int nest = NewGUCNestLevel(); + + (void) set_config_option("search_path", "", PGC_USERSET, PGC_S_SESSION, + GUC_ACTION_SAVE, true, 0, false); + (void) set_config_option("DateStyle", "ISO, YMD", PGC_USERSET, PGC_S_SESSION, + GUC_ACTION_SAVE, true, 0, false); + (void) set_config_option("IntervalStyle", "postgres", PGC_USERSET, PGC_S_SESSION, + GUC_ACTION_SAVE, true, 0, false); + deparse_and_find_prefix(query, &dr); + source = skip_override_clause(skip_leading_collist(dr.rest), query->override); + sql = psprintf("SELECT %s FROM (%s) AS coldfront_src(%s) WHERE %s < %s", + build_cold_projection(query, hc, true), source, col_list, + cf_pc_expr(info, hc), format_timestamptz_literal(info->cutoff)); + AtEOXact_GUC(true, nest); + return cold_sql_arg(sql, ps, true); } -/* The cold half: coldfront._cold_sink over the source rows below the cutoff, - * each projected to every hot-table column (build_cold_projection). */ +/* The cold half of the sink shape: coldfront._cold_sink over the source rows + * below the cutoff, each projected to every hot-table column + * (build_cold_projection). */ static char * build_cold_sink_select(const char *vschema, const char *vname, const char *projection, const char *partition_col, @@ -2414,29 +2704,35 @@ skip_override_clause(const char *rest, OverridingKind override) } /* - * Wrap the source CTE, the hot INSERT and the cold sink into one statement, - * behind the statement's own WITH entries (dr->head_len bytes), which stay at - * the top level: PostgreSQL allows an entry that modifies data nowhere else. + * Wrap the hot INSERT and the cold half into one statement, behind the + * statement's own WITH entries (dr->head_len bytes), which stay at the top + * level: PostgreSQL allows an entry that modifies data nowhere else. The sink + * shape adds the materialized source (src_cte) that both halves read. * - * At top level the hot INSERT and the sink are CTEs and the statement reports - * (hot_rows, cold_rows). In plpgsql, and for an INSERT nested in WITH, the hot - * INSERT is the statement itself (the DML tag plpgsql accepts, and the body - * the WITH entry keeps once cf_splice_nested_dml lifts the rest out), and - * the sink rides in a data-modifying CTE (cold_anchor_update) that always runs - * to completion; the sink's count is never NULL, so the anchor updates no row. + * At top level the hot INSERT and the cold half are CTEs and the statement + * reports (hot_rows, cold_rows): the sink's count, or the stream's, which is + * NULL when the rows streamed (DuckDB keeps that count) and the count + * otherwise. In plpgsql, and for an INSERT nested in WITH, the hot INSERT is + * the statement itself (the DML tag plpgsql accepts, and the body the WITH + * entry keeps once cf_splice_nested_dml lifts the rest out), and the cold half + * rides in a data-modifying CTE (cold_anchor_update) that always runs to + * completion and updates no row: the sink's count is never NULL, and the + * stream's count is never negative, with NULL < 0 being no match. */ static char * wrap_tiered_result(bool in_plpgsql, const DeparseResult *dr, const char *src_cte, - const char *hot_dml, const char *cold_select) + const char *hot_dml, const char *cold_select, const char *anchor_test) { StringInfoData buf; initStringInfo(&buf); append_with_opener(&buf, dr); - appendStringInfo(&buf, "coldfront_source AS MATERIALIZED (%s), ", src_cte); + if (src_cte != NULL) + appendStringInfo(&buf, "coldfront_source AS MATERIALIZED (%s), ", src_cte); if (in_plpgsql) appendStringInfo(&buf, "coldfront_cold AS (%s) %s", - cold_anchor_update(psprintf("(%s)", cold_select)), hot_dml); + cold_anchor_update(psprintf("(%s)", cold_select), anchor_test), + hot_dml); else appendStringInfo(&buf, "coldfront_hot AS MATERIALIZED (%s RETURNING 1), " "coldfront_cold AS MATERIALIZED (%s) " @@ -2448,8 +2744,32 @@ wrap_tiered_result(bool in_plpgsql, const DeparseResult *dr, const char *src_cte /* * emit_tiered_insert rewrites a tiered-view INSERT into one statement that - * reads the source once and splits that result by the partition column - * against the watermark: + * splits the source by the partition column against the watermark, in one of + * two shapes. + * + * The stream shape, when cf_tiered_insert_streams proves the source safe to + * read twice: + * + * WITH coldfront_hot AS MATERIALIZED (INSERT INTO (cols) SELECT FROM + * () AS coldfront_src(cols) + * WHERE >= RETURNING 1), + * coldfront_cold AS MATERIALIZED (SELECT coldfront._tiered_cold_stream( + * schema, view, , + * , + * ) AS n) + * SELECT hot_rows, cold_rows + * + * The hot half is plain PG: set-based, IDENTITY and DEFAULT fill server-side. + * The cold half is one DuckDB statement: INSERT INTO SELECT ... FROM + * postgres_query('pgstream', ), so the rows go from PostgreSQL into + * the Iceberg writer in one vectorized pass, with the identity values and + * defaults the projection draws in PostgreSQL. A VALUES source's cold test is + * the hot count against its row count; a SELECT source's is an EXISTS over + * the source, so an all-hot INSERT stages nothing and takes no claim. + * + * The sink shape, for every other source: * * WITH , * coldfront_source AS MATERIALIZED ( < >) AS r) * SELECT hot_rows, cold_rows * - * The hot half is plain PG: set-based, IDENTITY and DEFAULT fill server-side. - * The cold half projects each row to the hot table's full column list - * (build_cold_projection) and hands it to the sink, which renders it as a - * DuckDB VALUES tuple and flushes every coldfront.cold_write_batch_size rows - * under the table's claim (coldfront--1.0.sql). Both halves read the one - * tuplestore, so a volatile source, or a row the transaction wrote before the - * INSERT, lands exactly once. Bound parameters ($N) stay in the source, which - * PostgreSQL runs, so they bind natively. RETURNING is not preserved; the - * top-level statement reports (hot_rows, cold_rows). + * Both halves read the one tuplestore, so a volatile source, or a row the + * transaction wrote before the INSERT, lands exactly once; the sink renders + * each row as a DuckDB VALUES tuple and writes a batch every + * coldfront.cold_write_batch_size rows (coldfront--1.0.sql). + * + * Bound parameters ($N) stay in the source, which PostgreSQL runs, so they + * bind natively; the stream's text carries them as format() arguments. + * RETURNING is not preserved. */ static char * -emit_tiered_insert(Query *query, TieredViewInfo *info, bool in_plpgsql) +emit_tiered_insert(Query *query, TieredViewInfo *info, ColdParamSet *ps, + bool in_plpgsql, Oid view_relid) { DeparseResult dr; HotColumns hc; - char *col_list, *cutoff_lit, *src_cte, *hot_dml, *cold_select; + List *sources; + char *col_list, *cutoff_lit, *hot_dml, *cold_select; const char *source, *vname, *vschema, *override; deparse_and_find_prefix(query, &dr); @@ -2487,22 +2808,39 @@ emit_tiered_insert(Query *query, TieredViewInfo *info, bool in_plpgsql) cutoff_lit = format_timestamptz_literal(info->cutoff); override = query->override == OVERRIDING_SYSTEM_VALUE ? "OVERRIDING SYSTEM VALUE " : query->override == OVERRIDING_USER_VALUE ? "OVERRIDING USER VALUE " : ""; + vname = get_rel_name(view_relid); + vschema = get_namespace_name(get_rel_namespace(view_relid)); + load_hot_columns(info->hot_table, &hc); + if (cf_tiered_insert_streams(query, view_relid, &sources)) { - RangeTblEntry *rte = (RangeTblEntry *) list_nth(query->rtable, - query->resultRelation - 1); - vname = get_rel_name(rte->relid); - vschema = get_namespace_name(get_rel_namespace(rte->relid)); + const char *src_select = build_src_cte(query, source, col_list, &hc); + const char *pc_expr = cf_pc_expr(info, &hc); + int nvalues = cf_values_rows(query); + char *any_cold; + + hot_dml = build_tiered_hot_dml(info->hot_table, col_list, override, + src_select + strlen("SELECT "), "", + pc_expr, cutoff_lit); + any_cold = !in_plpgsql && nvalues > 0 + ? psprintf("(SELECT count(*) FROM coldfront_hot) < %d", nvalues) + : psprintf("EXISTS (SELECT 1 FROM (%s) AS coldfront_src(%s) WHERE %s < %s)", + source, col_list, pc_expr, cutoff_lit); + cold_select = psprintf("SELECT coldfront._tiered_cold_stream(%s, %s, %s, %s, %s) AS n", + quote_literal_cstr(vschema), quote_literal_cstr(vname), + cf_stream_pg_select(query, info, &hc, col_list, ps), + cf_regclass_array(sources), any_cold); + return wrap_tiered_result(in_plpgsql, &dr, NULL, hot_dml, cold_select, "< 0"); } - load_hot_columns(info->hot_table, &hc); - src_cte = build_src_cte(query, source, col_list, &hc); - hot_dml = build_tiered_hot_dml(info->hot_table, col_list, override, - info->partition_col, cutoff_lit); + hot_dml = build_tiered_hot_dml(info->hot_table, col_list, override, col_list, + "FROM coldfront_source", + quote_identifier(info->partition_col), cutoff_lit); cold_select = build_cold_sink_select(vschema, vname, - build_cold_projection(query, &hc), + build_cold_projection(query, &hc, false), info->partition_col, cutoff_lit); - return wrap_tiered_result(in_plpgsql, &dr, src_cte, hot_dml, cold_select); + return wrap_tiered_result(in_plpgsql, &dr, build_src_cte(query, source, col_list, &hc), + hot_dml, cold_select, "IS NULL"); } /* @@ -2648,8 +2986,8 @@ reject_cold_outer_cte(Query *query, const char *vname) * plain hot INSERT. With a watermark, some rows go cold (cannot RETURN them). */ static char * -cf_emit_tiered_insert_path(Query *query, TieredViewInfo *info, ColdParamSet *ps, - bool in_plpgsql, const char *vname) +cf_emit_tiered_insert_path(Query *query, Oid view_relid, TieredViewInfo *info, + ColdParamSet *ps, bool in_plpgsql, const char *vname) { if (!info->has_cutoff) return emit_hot(query, info); @@ -2657,7 +2995,7 @@ cf_emit_tiered_insert_path(Query *query, TieredViewInfo *info, ColdParamSet *ps, /* A watermark-split INSERT sends some rows to the cold tier, * which cannot return them — refuse RETURNING rather than drop it. */ reject_cold_returning(query, vname); - return emit_tiered_insert(query, info, in_plpgsql); + return emit_tiered_insert(query, info, ps, in_plpgsql, view_relid); } /* Build the rewritten SQL for a cold-tier UPDATE/DELETE/INSERT/MERGE (TIER_COLD). */ @@ -3931,7 +4269,7 @@ cf_dispatch_emit(Query *query, RangeTblEntry *rte, TieredViewInfo *info, * Iceberg-only INSERT falls through to the unconditional cold path * (classify_tier short-circuits to TIER_COLD for iceberg-only). */ if (query->commandType == CMD_INSERT && !info->is_iceberg_only) - return cf_emit_tiered_insert_path(query, info, ps, in_plpgsql, vname); + return cf_emit_tiered_insert_path(query, rte->relid, info, ps, in_plpgsql, vname); tier = classify_tier(query, info); diff --git a/extension/coldfront/test/expected/array_cold_render.out b/extension/coldfront/test/expected/array_cold_render.out index dc6aba8..6119cd6 100644 --- a/extension/coldfront/test/expected/array_cold_render.out +++ b/extension/coldfront/test/expected/array_cold_render.out @@ -108,34 +108,33 @@ VALUES ('public', 'tags', '2026-03-01'::timestamptz); -- The INSERT's cold half checks each array's shape in its projection. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.tags VALUES (1, '2026-01-05', '{a,b}', '{1,2}'); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 - CTE coldfront_source - -> Result - Output: 1, '2026-01-05 00:00:00+00'::timestamp with time zone, '{a,b}'::text[], '{1,2}'::integer[] CTE coldfront_hot -> Insert on public._tags Output: 1 - -> CTE Scan on coldfront_source - Output: coldfront_source.id, coldfront_source.ts, coldfront_source.tags, coldfront_source.nums - Filter: (coldfront_source.ts >= '2026-03-01 00:00:00+00'::timestamp with time zone) + -> Result + Output: 1, '2026-01-05 00:00:00+00'::timestamp with time zone, '{a,b}'::text[], '{1,2}'::integer[] + One-Time Filter: false CTE coldfront_cold - -> Aggregate - Output: _cold_sink('public'::text, 'tags'::text, to_jsonb(ROW(coldfront_source_1.id, coldfront_source_1.ts, _cold_list(coldfront_source_1.tags, 'tags'::text), _cold_list(coldfront_source_1.nums, 'nums'::text)))) - -> CTE Scan on coldfront_source coldfront_source_1 - Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.tags, coldfront_source_1.nums - Filter: (coldfront_source_1.ts < '2026-03-01 00:00:00+00'::timestamp with time zone) + -> Result + Output: _tiered_cold_stream('public'::text, 'tags'::text, 'SELECT coldfront_src.id::integer AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront._cold_list(coldfront_src.tags::text[], ''tags'') AS tags, coldfront._cold_list(coldfront_src.nums::integer[], ''nums'') AS nums FROM (VALUES (1, ''2026-01-05 00:00:00+00''::timestamp with time zone, ''{a,b}''::text[], ''{1,2}''::integer[])) AS coldfront_src(id, ts, tags, nums) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ((InitPlan 2).col1 < 1)) + InitPlan 2 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" InitPlan 4 -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot - Output: coldfront_hot."?column?" + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(25 rows) +(24 rows) -- A cold write's bound array parameter is rendered by _render_cold_param at -- execution. force_generic_plan keeps $1 from folding to a constant. diff --git a/extension/coldfront/test/expected/array_cold_render_1.out b/extension/coldfront/test/expected/array_cold_render_1.out index b61a148..760b824 100644 --- a/extension/coldfront/test/expected/array_cold_render_1.out +++ b/extension/coldfront/test/expected/array_cold_render_1.out @@ -108,34 +108,33 @@ VALUES ('public', 'tags', '2026-03-01'::timestamptz); -- The INSERT's cold half checks each array's shape in its projection. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.tags VALUES (1, '2026-01-05', '{a,b}', '{1,2}'); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 - CTE coldfront_source - -> Result - Output: 1, '2026-01-05 00:00:00+00'::timestamp with time zone, '{a,b}'::text[], '{1,2}'::integer[] CTE coldfront_hot -> Insert on public._tags Output: 1 - -> CTE Scan on coldfront_source - Output: coldfront_source.id, coldfront_source.ts, coldfront_source.tags, coldfront_source.nums - Filter: (coldfront_source.ts >= '2026-03-01 00:00:00+00'::timestamp with time zone) + -> Result + Output: 1, '2026-01-05 00:00:00+00'::timestamp with time zone, '{a,b}'::text[], '{1,2}'::integer[] + One-Time Filter: false CTE coldfront_cold - -> Aggregate - Output: _cold_sink('public'::text, 'tags'::text, to_jsonb(ROW(coldfront_source_1.id, coldfront_source_1.ts, _cold_list(coldfront_source_1.tags, 'tags'::text), _cold_list(coldfront_source_1.nums, 'nums'::text)))) - -> CTE Scan on coldfront_source coldfront_source_1 - Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.tags, coldfront_source_1.nums - Filter: (coldfront_source_1.ts < '2026-03-01 00:00:00+00'::timestamp with time zone) + -> Result + Output: _tiered_cold_stream('public'::text, 'tags'::text, 'SELECT coldfront_src.id::integer AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront._cold_list(coldfront_src.tags::text[], ''tags'') AS tags, coldfront._cold_list(coldfront_src.nums::integer[], ''nums'') AS nums FROM (VALUES (1, ''2026-01-05 00:00:00+00''::timestamp with time zone, ''{a,b}''::text[], ''{1,2}''::integer[])) AS coldfront_src(id, ts, tags, nums) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ($2 < 1)) + InitPlan 2 (returns $2) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" InitPlan 4 (returns $4) -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot - Output: coldfront_hot."?column?" + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(25 rows) +(24 rows) -- A cold write's bound array parameter is rendered by _render_cold_param at -- execution. force_generic_plan keeps $1 from folding to a constant. diff --git a/extension/coldfront/test/expected/async_write_before_claim.out b/extension/coldfront/test/expected/async_write_before_claim.out index dc8f132..3870dec 100644 --- a/extension/coldfront/test/expected/async_write_before_claim.out +++ b/extension/coldfront/test/expected/async_write_before_claim.out @@ -134,6 +134,49 @@ CONTEXT: SQL statement "SELECT pg_advisory_xact_lock(hashtext('coldfront_iceber PL/pgSQL function _take_iceberg_claim(text) line 13 at PERFORM SQL statement "SELECT coldfront._take_iceberg_claim(v_iceberg)" PL/pgSQL function _cross_tier_move(text,text,text,text) line 134 at PERFORM +-- (6) The cold stream: a tiered INSERT whose source streams is one DuckDB +-- write in the statement, and the claim at COMMIT meets the lock. The +-- stream's second session connects to the regress database. +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; + dsn_set +--------- + t +(1 row) + +CREATE TABLE public.cf_order_src (id int, ts timestamptz, status text); +INSERT INTO public.cf_order_src VALUES (30, '2026-01-06', 'stream'), (31, '2026-01-07', 'stream'); +BEGIN; +INSERT INTO public.cf_order (id, ts, status) SELECT id, ts, status FROM public.cf_order_src; +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +2 + + + hot_rows | cold_rows +----------+----------- + 0 | +(1 row) + +COMMIT; +ERROR: canceling statement due to lock timeout +CONTEXT: SQL statement "SELECT pg_advisory_xact_lock(hashtext('coldfront_iceberg:' || p_iceberg_ref))" +PL/pgSQL function _take_iceberg_claim(text) line 13 at PERFORM +SQL statement "SELECT coldfront._take_iceberg_claim($1)" +DROP TABLE public.cf_order_src; -- Cleanup. SELECT coldfront._loopback('SELECT pg_advisory_unlock_all()') IS NOT NULL AS lock_released; lock_released @@ -150,3 +193,4 @@ RESET lock_timeout; RESET coldfront.iceberg_async_parquet; RESET coldfront.iceberg_bakery_patch; RESET coldfront.loopback_dsn; +RESET coldfront.local_pg_dsn; diff --git a/extension/coldfront/test/expected/param_cold_via_plpgsql.out b/extension/coldfront/test/expected/param_cold_via_plpgsql.out index 43daedb..bb4654c 100644 --- a/extension/coldfront/test/expected/param_cold_via_plpgsql.out +++ b/extension/coldfront/test/expected/param_cold_via_plpgsql.out @@ -118,10 +118,10 @@ EXPLAIN (COSTS OFF, VERBOSE) Output: _exec_iceberg_with_claim('ice.default.events'::text, 'UPDATE ice.default.events SET status = ''x''::text WHERE (ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz)'::text) (2 rows) --- (8) tiered INSERT with an omitted IDENTITY column and a bytea param: the params --- stay native in the source CTE, which PostgreSQL runs; the hot INSERT and the --- cold projection each take nextval() for the identity, and the cold projection --- spells the bytea as hex text, independent of the session's bytea_output. +-- (8) tiered INSERT with an omitted IDENTITY column and a bytea param: the hot +-- INSERT keeps the params native, and the cold stream's text carries them as +-- format() arguments, each a literal the other session reads back and casts; +-- both halves take nextval() for the identity. CREATE TABLE public._eid (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz, data bytea); CREATE VIEW public.eid AS SELECT * FROM public._eid; INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) @@ -130,36 +130,33 @@ INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) VALUES ('public', 'eid', '2026-03-01'::timestamptz); PREPARE eid_ins(timestamptz, bytea) AS INSERT INTO public.eid (ts, data) VALUES ($1, $2); EXPLAIN (COSTS OFF, VERBOSE) EXECUTE eid_ins('2026-01-10 00:00+00', '\xcafe'::bytea); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 - CTE coldfront_source - -> Result - Output: $1, $2 CTE coldfront_hot -> Insert on public._eid Output: 1 - -> CTE Scan on coldfront_source - Output: nextval('_eid_id_seq'), coldfront_source.ts, coldfront_source.data - Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Result + Output: nextval('_eid_id_seq'), $1, $2 + One-Time Filter: ($1 >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold - -> Aggregate - Output: _cold_sink('public'::text, 'eid'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_eid_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.data - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Result + Output: _tiered_cold_stream('public'::text, 'eid'::text, format('SELECT nextval(''public._eid_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.data::bytea AS data FROM (VALUES (%1$L, %2$L)) AS coldfront_src(ts, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, $1, $2), '{}'::regclass[], ((InitPlan 2).col1 < 1)) + InitPlan 2 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" InitPlan 4 -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot - Output: coldfront_hot."?column?" + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(24 rows) DEALLOCATE cold_upd; DEALLOCATE cold_del; diff --git a/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out b/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out index c310337..fcff944 100644 --- a/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out +++ b/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out @@ -118,10 +118,10 @@ EXPLAIN (COSTS OFF, VERBOSE) Output: _exec_iceberg_with_claim('ice.default.events'::text, 'UPDATE ice.default.events SET status = ''x''::text WHERE (ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz)'::text) (2 rows) --- (8) tiered INSERT with an omitted IDENTITY column and a bytea param: the params --- stay native in the source CTE, which PostgreSQL runs; the hot INSERT and the --- cold projection each take nextval() for the identity, and the cold projection --- spells the bytea as hex text, independent of the session's bytea_output. +-- (8) tiered INSERT with an omitted IDENTITY column and a bytea param: the hot +-- INSERT keeps the params native, and the cold stream's text carries them as +-- format() arguments, each a literal the other session reads back and casts; +-- both halves take nextval() for the identity. CREATE TABLE public._eid (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz, data bytea); CREATE VIEW public.eid AS SELECT * FROM public._eid; INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) @@ -130,36 +130,33 @@ INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) VALUES ('public', 'eid', '2026-03-01'::timestamptz); PREPARE eid_ins(timestamptz, bytea) AS INSERT INTO public.eid (ts, data) VALUES ($1, $2); EXPLAIN (COSTS OFF, VERBOSE) EXECUTE eid_ins('2026-01-10 00:00+00', '\xcafe'::bytea); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 - CTE coldfront_source - -> Result - Output: $1, $2 CTE coldfront_hot -> Insert on public._eid Output: 1 - -> CTE Scan on coldfront_source - Output: nextval('_eid_id_seq'), coldfront_source.ts, coldfront_source.data - Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Result + Output: nextval('_eid_id_seq'), $1, $2 + One-Time Filter: ($1 >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold - -> Aggregate - Output: _cold_sink('public'::text, 'eid'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_eid_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.data - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Result + Output: _tiered_cold_stream('public'::text, 'eid'::text, format('SELECT nextval(''public._eid_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.data::bytea AS data FROM (VALUES (%1$L, %2$L)) AS coldfront_src(ts, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, $1, $2), '{}'::regclass[], ($2 < 1)) + InitPlan 2 (returns $2) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" InitPlan 4 (returns $4) -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot - Output: coldfront_hot."?column?" + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(24 rows) DEALLOCATE cold_upd; DEALLOCATE cold_del; diff --git a/extension/coldfront/test/expected/tiered_insert_single_pass.out b/extension/coldfront/test/expected/tiered_insert_single_pass.out index 908d08c..254dbe2 100644 --- a/extension/coldfront/test/expected/tiered_insert_single_pass.out +++ b/extension/coldfront/test/expected/tiered_insert_single_pass.out @@ -1,10 +1,14 @@ --- A tiered INSERT ... SELECT reads its source once: the rewrite holds the source --- in a MATERIALIZED CTE that the hot INSERT and the cold sink both read, casts --- every column to the hot table's type (an untyped literal keeps the target's --- type), fills an omitted identity column with nextval() and an omitted column --- with its DEFAULT on the cold side, and keeps OVERRIDING SYSTEM VALUE on the --- hot INSERT. White-box: EXPLAIN VERBOSE shows the rewritten shape; no Iceberg --- I/O (warehouse and endpoint left ''). +-- The two shapes of a tiered INSERT. A source the hook cannot read a second +-- time from another session (here a stable function, timestamptz plus an +-- interval) is read once: the rewrite holds it in a MATERIALIZED CTE that the +-- hot INSERT and the cold sink both read. A source it can read twice streams: +-- the hot INSERT reads it, and the cold half is one call carrying the cold +-- projection as text for the other session. Both cast every column to the hot +-- table's type (an untyped literal keeps the target's type), fill an omitted +-- identity column with nextval() and an omitted column with its DEFAULT on +-- the cold side, and keep OVERRIDING SYSTEM VALUE on the hot INSERT. +-- White-box: EXPLAIN VERBOSE shows the rewritten shape; no Iceberg I/O +-- (warehouse and endpoint left ''). SET client_min_messages = warning; CREATE EXTENSION IF NOT EXISTS pg_duckdb; CREATE EXTENSION IF NOT EXISTS coldfront; @@ -59,26 +63,22 @@ EXPLAIN (COSTS OFF, VERBOSE) EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (id, ts, status, data) OVERRIDING SYSTEM VALUE SELECT i, '2026-01-01'::timestamptz, 'old', '{}'::jsonb FROM generate_series(1, 3) i; - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 - CTE coldfront_source - -> Function Scan on pg_catalog.generate_series i - Output: (i.i)::bigint, 'Thu Jan 01 00:00:00 2026 UTC'::timestamp with time zone, 'old'::text, '{}'::jsonb - Function Call: generate_series(1, 3) CTE coldfront_hot -> Insert on public._events Output: 1 - -> CTE Scan on coldfront_source - Output: coldfront_source.id, coldfront_source.ts, coldfront_source.status, coldfront_source.data - Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Result + Output: (i)::bigint, 'Thu Jan 01 00:00:00 2026 UTC'::timestamp with time zone, 'old'::text, '{}'::jsonb + One-Time Filter: false CTE coldfront_cold - -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, to_jsonb(ROW(coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status, coldfront_source_1.data))) - -> CTE Scan on coldfront_source coldfront_source_1 - Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status, coldfront_source_1.data - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Result + Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT coldfront_src.id::bigint AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (coldfront_src.data::jsonb)::text AS data FROM (SELECT i.i, ''2026-01-01 00:00:00+00''::timestamp with time zone AS timestamptz, ''old'', ''{}''::jsonb AS jsonb FROM generate_series(1, 3) i(i)) AS coldfront_src(id, ts, status, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], (InitPlan 2).col1) + InitPlan 2 + -> Function Scan on pg_catalog.generate_series i + Function Call: generate_series(1, 3) InitPlan 4 -> Aggregate Output: count(*) @@ -87,41 +87,38 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(26 rows) +(22 rows) -- VALUES takes the same shape. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (ts, status) VALUES ('2026-01-01', 'a'), ('2026-05-01', 'b'); - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 - CTE coldfront_source - -> Values Scan on "*VALUES*" - Output: "*VALUES*".column1, "*VALUES*".column2 CTE coldfront_hot -> Insert on public._events Output: 1 - -> CTE Scan on coldfront_source - Output: nextval('_events_id_seq'), coldfront_source.ts, coldfront_source.status, NULL::jsonb - Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Values Scan on "*VALUES*" + Output: nextval('_events_id_seq'), "*VALUES*".column1, "*VALUES*".column2, NULL::jsonb + Filter: ("*VALUES*".column1 >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold - -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_events_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, NULL::jsonb - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Result + Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT nextval(''public._events_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (NULL::jsonb)::text AS data FROM (VALUES (''2026-01-01 00:00:00+00''::timestamp with time zone,''a''::text), (''2026-05-01 00:00:00+00''::timestamp with time zone,''b''::text)) AS coldfront_src(ts, status) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ((InitPlan 2).col1 < 2)) + InitPlan 2 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" InitPlan 4 -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot - Output: coldfront_hot."?column?" + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(24 rows) -- Cleanup. DELETE FROM coldfront.tiered_views; diff --git a/extension/coldfront/test/expected/tiered_insert_single_pass_1.out b/extension/coldfront/test/expected/tiered_insert_single_pass_1.out index 5ed364a..16dc6ec 100644 --- a/extension/coldfront/test/expected/tiered_insert_single_pass_1.out +++ b/extension/coldfront/test/expected/tiered_insert_single_pass_1.out @@ -1,10 +1,14 @@ --- A tiered INSERT ... SELECT reads its source once: the rewrite holds the source --- in a MATERIALIZED CTE that the hot INSERT and the cold sink both read, casts --- every column to the hot table's type (an untyped literal keeps the target's --- type), fills an omitted identity column with nextval() and an omitted column --- with its DEFAULT on the cold side, and keeps OVERRIDING SYSTEM VALUE on the --- hot INSERT. White-box: EXPLAIN VERBOSE shows the rewritten shape; no Iceberg --- I/O (warehouse and endpoint left ''). +-- The two shapes of a tiered INSERT. A source the hook cannot read a second +-- time from another session (here a stable function, timestamptz plus an +-- interval) is read once: the rewrite holds it in a MATERIALIZED CTE that the +-- hot INSERT and the cold sink both read. A source it can read twice streams: +-- the hot INSERT reads it, and the cold half is one call carrying the cold +-- projection as text for the other session. Both cast every column to the hot +-- table's type (an untyped literal keeps the target's type), fill an omitted +-- identity column with nextval() and an omitted column with its DEFAULT on +-- the cold side, and keep OVERRIDING SYSTEM VALUE on the hot INSERT. +-- White-box: EXPLAIN VERBOSE shows the rewritten shape; no Iceberg I/O +-- (warehouse and endpoint left ''). SET client_min_messages = warning; CREATE EXTENSION IF NOT EXISTS pg_duckdb; CREATE EXTENSION IF NOT EXISTS coldfront; @@ -59,26 +63,22 @@ EXPLAIN (COSTS OFF, VERBOSE) EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (id, ts, status, data) OVERRIDING SYSTEM VALUE SELECT i, '2026-01-01'::timestamptz, 'old', '{}'::jsonb FROM generate_series(1, 3) i; - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 - CTE coldfront_source - -> Function Scan on pg_catalog.generate_series i - Output: (i.i)::bigint, 'Thu Jan 01 00:00:00 2026 UTC'::timestamp with time zone, 'old'::text, '{}'::jsonb - Function Call: generate_series(1, 3) CTE coldfront_hot -> Insert on public._events Output: 1 - -> CTE Scan on coldfront_source - Output: coldfront_source.id, coldfront_source.ts, coldfront_source.status, coldfront_source.data - Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Result + Output: (i)::bigint, 'Thu Jan 01 00:00:00 2026 UTC'::timestamp with time zone, 'old'::text, '{}'::jsonb + One-Time Filter: false CTE coldfront_cold - -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, to_jsonb(ROW(coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status, coldfront_source_1.data))) - -> CTE Scan on coldfront_source coldfront_source_1 - Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status, coldfront_source_1.data - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Result + Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT coldfront_src.id::bigint AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (coldfront_src.data::jsonb)::text AS data FROM (SELECT i.i, ''2026-01-01 00:00:00+00''::timestamp with time zone AS timestamptz, ''old'', ''{}''::jsonb AS jsonb FROM generate_series(1, 3) i(i)) AS coldfront_src(id, ts, status, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], $2) + InitPlan 2 (returns $2) + -> Function Scan on pg_catalog.generate_series i + Function Call: generate_series(1, 3) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -87,41 +87,38 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(26 rows) +(22 rows) -- VALUES takes the same shape. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (ts, status) VALUES ('2026-01-01', 'a'), ('2026-05-01', 'b'); - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 - CTE coldfront_source - -> Values Scan on "*VALUES*" - Output: "*VALUES*".column1, "*VALUES*".column2 CTE coldfront_hot -> Insert on public._events Output: 1 - -> CTE Scan on coldfront_source - Output: nextval('_events_id_seq'), coldfront_source.ts, coldfront_source.status, NULL::jsonb - Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Values Scan on "*VALUES*" + Output: nextval('_events_id_seq'), "*VALUES*".column1, "*VALUES*".column2, NULL::jsonb + Filter: ("*VALUES*".column1 >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold - -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_events_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, NULL::jsonb - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + -> Result + Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT nextval(''public._events_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (NULL::jsonb)::text AS data FROM (VALUES (''2026-01-01 00:00:00+00''::timestamp with time zone,''a''::text), (''2026-05-01 00:00:00+00''::timestamp with time zone,''b''::text)) AS coldfront_src(ts, status) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ($2 < 2)) + InitPlan 2 (returns $2) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" InitPlan 4 (returns $4) -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot - Output: coldfront_hot."?column?" + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(24 rows) -- Cleanup. DELETE FROM coldfront.tiered_views; diff --git a/extension/coldfront/test/expected/tiered_insert_stream.out b/extension/coldfront/test/expected/tiered_insert_stream.out new file mode 100644 index 0000000..f9a34f6 --- /dev/null +++ b/extension/coldfront/test/expected/tiered_insert_stream.out @@ -0,0 +1,557 @@ +-- A tiered INSERT whose source the hook can read a second time from another +-- session streams its cold rows: the hot half is a set-based INSERT and the +-- cold half is one DuckDB INSERT ... SELECT FROM postgres_query('pgstream', ...) +-- through coldfront._exec_iceberg_with_claim, with the identity values and the +-- defaults drawn in that session. Observed end to end: a DuckDB temporary table +-- stands in for the Iceberg table and is read back with duckdb.query. A source +-- the hook cannot read twice (a volatile or stable function, a temporary table, +-- a WITH entry, a view over one, a table this transaction wrote, a transaction +-- that is not READ COMMITTED, no local_pg_dsn) goes through the cold sink in one +-- pass, and a cached plan that meets such a transaction falls back inside the +-- stream. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +-- The regress database is not the one the image's local_pg_dsn names. +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; + dsn_set +--------- + t +(1 row) + +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_stream (id BIGINT, ts TIMESTAMPTZ, status VARCHAR, n INTEGER, data VARCHAR, b BLOB, tags INTEGER[])') IS NOT NULL AS created; +NOTICE: result: Count +BIGINT +[ Rows: 0] + + + created +--------- + t +(1 row) + +CREATE TABLE public._cf_stream (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz NOT NULL, + status text NOT NULL DEFAULT 'new', n int DEFAULT 7, + data jsonb, b bytea, tags int[]); +CREATE VIEW public.cf_stream AS SELECT * FROM public._cf_stream; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'cf_stream', 'public._cf_stream', 'temp.main.cf_stream', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'cf_stream', '2026-03-01'::timestamptz); +CREATE TABLE public.cf_staging (ts timestamptz, status text, data jsonb, b bytea, tags int[]); +INSERT INTO public.cf_staging VALUES + ('2026-01-01', 'cold1', '{"a": 1}', '\xcafe', '{1,2}'), + ('2026-02-01', 'cold2', NULL, NULL, NULL), + ('2026-05-01', 'hot1', '{}', '\xbeef', '{3}'); +-- The rows of each tier, without the ids, which the two halves draw from one +-- sequence in no fixed order. The cold read runs in DuckDB, so the two reads +-- are separate views. +CREATE VIEW public.cf_stream_hot AS + SELECT ts, status, n, data::text AS data, b, tags FROM public._cf_stream; +CREATE VIEW public.cf_stream_cold AS + SELECT r['ts']::timestamptz AS ts, r['status']::text AS status, r['n']::int AS n, + r['data']::text AS data, r['b']::bytea AS b, r['tags']::int[] AS tags + FROM duckdb.query('SELECT * FROM temp.main.cf_stream') AS t(r); +CREATE VIEW public.cf_stream_ids AS + SELECT count(*) AS rows, count(DISTINCT id) AS ids, min(id) AS min_id, max(id) AS max_id + FROM (SELECT id FROM public._cf_stream + UNION ALL + SELECT r['id']::bigint FROM duckdb.query('SELECT id FROM temp.main.cf_stream') AS t(r)) AS u; +-- (1) A SELECT source over a table: the hot row goes to the heap, the two cold +-- rows stream, every column type lands as the sink would store it, and +-- the omitted identity and default columns are filled on both tiers. +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status, data, b, tags) + SELECT ts, status, data, b, tags FROM public.cf_staging; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Result + Output: (InitPlan 4).col1, (InitPlan 5).col1 + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> Seq Scan on public.cf_staging + Output: nextval('_cf_stream_id_seq'), cf_staging.ts, cf_staging.status, 7, cf_staging.data, cf_staging.b, cf_staging.tags + Filter: (cf_staging.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Result + Output: _tiered_cold_stream('public'::text, 'cf_stream'::text, 'SELECT nextval(''public._cf_stream_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (7) AS n, (coldfront_src.data::jsonb)::text AS data, coldfront_src.b::bytea AS b, coldfront._cold_list(coldfront_src.tags::integer[], ''tags'') AS tags FROM (SELECT cf_staging.ts, cf_staging.status, cf_staging.data, cf_staging.b, cf_staging.tags FROM public.cf_staging) AS coldfront_src(ts, status, data, b, tags) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{cf_staging}'::regclass[], (InitPlan 2).col1) + InitPlan 2 + -> Seq Scan on public.cf_staging cf_staging_1 + Filter: (cf_staging_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(22 rows) + +INSERT INTO public.cf_stream (ts, status, data, b, tags) +SELECT ts, status, data, b, tags FROM public.cf_staging; +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +2 + + + hot_rows | cold_rows +----------+----------- + 1 | +(1 row) + +SELECT * FROM public.cf_stream_hot ORDER BY ts; + ts | status | n | data | b | tags +------------------------------+--------+---+------+--------+------ + Fri May 01 00:00:00 2026 UTC | hot1 | 7 | {} | \xbeef | {3} +(1 row) + +SELECT * FROM public.cf_stream_cold ORDER BY ts; + ts | status | n | data | b | tags +------------------------------+--------+---+----------+--------+------- + Thu Jan 01 00:00:00 2026 UTC | cold1 | 7 | {"a": 1} | \xcafe | {1,2} + Sun Feb 01 00:00:00 2026 UTC | cold2 | 7 | | | +(2 rows) + +SELECT * FROM public.cf_stream_ids; + rows | ids | min_id | max_id +------+-----+--------+-------- + 3 | 3 | 1 | 3 +(1 row) + +-- (2) A VALUES source: the cold test is the hot count against the row count. +-- With a cold row DuckDB writes (and reports it); with none it is never +-- called, so an all-hot INSERT stages nothing. +SET client_min_messages = notice; +INSERT INTO public.cf_stream (ts, status) VALUES ('2026-01-10', 'cold3'), ('2026-06-10', 'hot2'); +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 1 | +(1 row) + +INSERT INTO public.cf_stream (ts, status) VALUES ('2026-06-11', 'hot3'), ('2026-06-12', 'hot4'); + hot_rows | cold_rows +----------+----------- + 2 | 0 +(1 row) + +SET client_min_messages = warning; +SELECT status FROM public.cf_stream_hot WHERE status IN ('hot2', 'hot3', 'hot4') ORDER BY status; + status +-------- + hot2 + hot3 + hot4 +(3 rows) + +SELECT status FROM public.cf_stream_cold WHERE status = 'cold3'; + status +-------- + cold3 +(1 row) + +-- (3) OVERRIDING SYSTEM VALUE supplies the identity on both tiers. +INSERT INTO public.cf_stream (id, ts, status) OVERRIDING SYSTEM VALUE +VALUES (901, '2026-01-11', 'ovr_cold'), (902, '2026-06-13', 'ovr_hot'); + hot_rows | cold_rows +----------+----------- + 1 | +(1 row) + +SELECT * FROM public.cf_stream_ids; + rows | ids | min_id | max_id +------+-----+--------+-------- + 9 | 9 | 1 | 902 +(1 row) + +-- (4) A prepared statement with a bound parameter streams from its cached plan +-- and keeps the parameter live. Run again in a transaction that wrote the +-- source table, the stream's check falls back to reading in this session, +-- so the row written in the transaction is tiered too. +PREPARE cf_prep(text) AS + INSERT INTO public.cf_stream (ts, status) SELECT ts, $1 FROM public.cf_staging WHERE ts < '2026-03-01'; +EXECUTE cf_prep('prep1'); + hot_rows | cold_rows +----------+----------- + 0 | +(1 row) + +BEGIN; +INSERT INTO public.cf_staging VALUES ('2026-01-20', 'cold4', NULL, NULL, NULL); +EXECUTE cf_prep('prep2'); + hot_rows | cold_rows +----------+----------- + 0 | 3 +(1 row) + +COMMIT; +BEGIN ISOLATION LEVEL REPEATABLE READ; +EXECUTE cf_prep('prep3'); + hot_rows | cold_rows +----------+----------- + 0 | 3 +(1 row) + +COMMIT; +SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'prep%' GROUP BY status ORDER BY status; + status | count +--------+------- + prep1 | 2 + prep2 | 3 + prep3 | 3 +(3 rows) + +-- (5) COPY into the view goes through the same rewrite, batch by batch. +SET coldfront.cold_write_batch_size = 2; +COPY public.cf_stream (ts, status) FROM stdin WITH (FORMAT csv); +RESET coldfront.cold_write_batch_size; +SELECT status, count(*) FROM public.cf_stream_hot WHERE status LIKE 'copy%' GROUP BY status; + status | count +----------+------- + copy_hot | 1 +(1 row) + +SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'copy%' GROUP BY status; + status | count +-----------+------- + copy_cold | 3 +(1 row) + +-- (6) Sources the hook sends to the sink in one pass: a volatile function, a +-- stable one, a temporary table, a WITH entry, a view that calls a stable +-- function, a table this transaction wrote, a transaction that is not READ +-- COMMITTED, and a session without local_pg_dsn. +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, random()::text FROM public.cf_staging; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: (InitPlan 4).col1, (InitPlan 5).col1 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: cf_staging.ts, (random())::text + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) VALUES (now() - interval '1 year', 'stable'); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: (InitPlan 4).col1, (InitPlan 5).col1 + CTE coldfront_source + -> Result + Output: (now() - '@ 1 year'::interval), 'stable'::text + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +CREATE TEMP TABLE cf_tmp (ts timestamptz, status text); +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM cf_tmp; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: (InitPlan 4).col1, (InitPlan 5).col1 + CTE coldfront_source + -> Seq Scan on pg_temp.cf_tmp + Output: cf_tmp.ts, cf_tmp.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +EXPLAIN (COSTS OFF, VERBOSE) + WITH s AS (SELECT ts, status FROM public.cf_staging) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM s; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: (InitPlan 4).col1, (InitPlan 5).col1 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: cf_staging.ts, cf_staging.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +CREATE VIEW public.cf_now_view AS SELECT now() AS ts, status FROM public.cf_staging; +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_now_view; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: (InitPlan 4).col1, (InitPlan 5).col1 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: now(), cf_staging.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +BEGIN; +INSERT INTO public.cf_staging VALUES ('2026-01-30', 'cold5', NULL, NULL, NULL); +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: (InitPlan 4).col1, (InitPlan 5).col1 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: cf_staging.ts, cf_staging.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +INSERT INTO public.cf_stream (ts, status) SELECT ts, 'txn' FROM public.cf_staging WHERE status = 'cold5'; + hot_rows | cold_rows +----------+----------- + 0 | 1 +(1 row) + +COMMIT; +BEGIN ISOLATION LEVEL REPEATABLE READ; +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: (InitPlan 4).col1, (InitPlan 5).col1 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: cf_staging.ts, cf_staging.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +COMMIT; +SET coldfront.local_pg_dsn = ''; +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: (InitPlan 4).col1, (InitPlan 5).col1 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: cf_staging.ts, cf_staging.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +SELECT status FROM public.cf_stream_cold WHERE status = 'txn'; + status +-------- + txn +(1 row) + +SELECT * FROM public.cf_stream_ids; + rows | ids | min_id | max_id +------+-----+--------+-------- + 22 | 22 | 1 | 902 +(1 row) + +-- Cleanup. +DEALLOCATE cf_prep; +DROP VIEW public.cf_now_view, public.cf_stream_hot, public.cf_stream_cold, public.cf_stream_ids; +DELETE FROM coldfront.tiered_views; +DELETE FROM coldfront.archive_watermark; +DROP VIEW public.cf_stream; +DROP TABLE public._cf_stream, public.cf_staging, cf_tmp; +RESET coldfront.local_pg_dsn; diff --git a/extension/coldfront/test/expected/tiered_insert_stream_1.out b/extension/coldfront/test/expected/tiered_insert_stream_1.out new file mode 100644 index 0000000..ae099b0 --- /dev/null +++ b/extension/coldfront/test/expected/tiered_insert_stream_1.out @@ -0,0 +1,557 @@ +-- A tiered INSERT whose source the hook can read a second time from another +-- session streams its cold rows: the hot half is a set-based INSERT and the +-- cold half is one DuckDB INSERT ... SELECT FROM postgres_query('pgstream', ...) +-- through coldfront._exec_iceberg_with_claim, with the identity values and the +-- defaults drawn in that session. Observed end to end: a DuckDB temporary table +-- stands in for the Iceberg table and is read back with duckdb.query. A source +-- the hook cannot read twice (a volatile or stable function, a temporary table, +-- a WITH entry, a view over one, a table this transaction wrote, a transaction +-- that is not READ COMMITTED, no local_pg_dsn) goes through the cold sink in one +-- pass, and a cached plan that meets such a transaction falls back inside the +-- stream. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +-- The regress database is not the one the image's local_pg_dsn names. +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; + dsn_set +--------- + t +(1 row) + +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_stream (id BIGINT, ts TIMESTAMPTZ, status VARCHAR, n INTEGER, data VARCHAR, b BLOB, tags INTEGER[])') IS NOT NULL AS created; +NOTICE: result: Count +BIGINT +[ Rows: 0] + + + created +--------- + t +(1 row) + +CREATE TABLE public._cf_stream (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz NOT NULL, + status text NOT NULL DEFAULT 'new', n int DEFAULT 7, + data jsonb, b bytea, tags int[]); +CREATE VIEW public.cf_stream AS SELECT * FROM public._cf_stream; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'cf_stream', 'public._cf_stream', 'temp.main.cf_stream', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'cf_stream', '2026-03-01'::timestamptz); +CREATE TABLE public.cf_staging (ts timestamptz, status text, data jsonb, b bytea, tags int[]); +INSERT INTO public.cf_staging VALUES + ('2026-01-01', 'cold1', '{"a": 1}', '\xcafe', '{1,2}'), + ('2026-02-01', 'cold2', NULL, NULL, NULL), + ('2026-05-01', 'hot1', '{}', '\xbeef', '{3}'); +-- The rows of each tier, without the ids, which the two halves draw from one +-- sequence in no fixed order. The cold read runs in DuckDB, so the two reads +-- are separate views. +CREATE VIEW public.cf_stream_hot AS + SELECT ts, status, n, data::text AS data, b, tags FROM public._cf_stream; +CREATE VIEW public.cf_stream_cold AS + SELECT r['ts']::timestamptz AS ts, r['status']::text AS status, r['n']::int AS n, + r['data']::text AS data, r['b']::bytea AS b, r['tags']::int[] AS tags + FROM duckdb.query('SELECT * FROM temp.main.cf_stream') AS t(r); +CREATE VIEW public.cf_stream_ids AS + SELECT count(*) AS rows, count(DISTINCT id) AS ids, min(id) AS min_id, max(id) AS max_id + FROM (SELECT id FROM public._cf_stream + UNION ALL + SELECT r['id']::bigint FROM duckdb.query('SELECT id FROM temp.main.cf_stream') AS t(r)) AS u; +-- (1) A SELECT source over a table: the hot row goes to the heap, the two cold +-- rows stream, every column type lands as the sink would store it, and +-- the omitted identity and default columns are filled on both tiers. +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status, data, b, tags) + SELECT ts, status, data, b, tags FROM public.cf_staging; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: $4, $5 + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> Seq Scan on public.cf_staging + Output: nextval('_cf_stream_id_seq'), cf_staging.ts, cf_staging.status, 7, cf_staging.data, cf_staging.b, cf_staging.tags + Filter: (cf_staging.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Result + Output: _tiered_cold_stream('public'::text, 'cf_stream'::text, 'SELECT nextval(''public._cf_stream_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (7) AS n, (coldfront_src.data::jsonb)::text AS data, coldfront_src.b::bytea AS b, coldfront._cold_list(coldfront_src.tags::integer[], ''tags'') AS tags FROM (SELECT cf_staging.ts, cf_staging.status, cf_staging.data, cf_staging.b, cf_staging.tags FROM public.cf_staging) AS coldfront_src(ts, status, data, b, tags) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{cf_staging}'::regclass[], $2) + InitPlan 2 (returns $2) + -> Seq Scan on public.cf_staging cf_staging_1 + Filter: (cf_staging_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 (returns $4) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 (returns $5) + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(22 rows) + +INSERT INTO public.cf_stream (ts, status, data, b, tags) +SELECT ts, status, data, b, tags FROM public.cf_staging; +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +2 + + + hot_rows | cold_rows +----------+----------- + 1 | +(1 row) + +SELECT * FROM public.cf_stream_hot ORDER BY ts; + ts | status | n | data | b | tags +------------------------------+--------+---+------+--------+------ + Fri May 01 00:00:00 2026 UTC | hot1 | 7 | {} | \xbeef | {3} +(1 row) + +SELECT * FROM public.cf_stream_cold ORDER BY ts; + ts | status | n | data | b | tags +------------------------------+--------+---+----------+--------+------- + Thu Jan 01 00:00:00 2026 UTC | cold1 | 7 | {"a": 1} | \xcafe | {1,2} + Sun Feb 01 00:00:00 2026 UTC | cold2 | 7 | | | +(2 rows) + +SELECT * FROM public.cf_stream_ids; + rows | ids | min_id | max_id +------+-----+--------+-------- + 3 | 3 | 1 | 3 +(1 row) + +-- (2) A VALUES source: the cold test is the hot count against the row count. +-- With a cold row DuckDB writes (and reports it); with none it is never +-- called, so an all-hot INSERT stages nothing. +SET client_min_messages = notice; +INSERT INTO public.cf_stream (ts, status) VALUES ('2026-01-10', 'cold3'), ('2026-06-10', 'hot2'); +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 1 | +(1 row) + +INSERT INTO public.cf_stream (ts, status) VALUES ('2026-06-11', 'hot3'), ('2026-06-12', 'hot4'); + hot_rows | cold_rows +----------+----------- + 2 | 0 +(1 row) + +SET client_min_messages = warning; +SELECT status FROM public.cf_stream_hot WHERE status IN ('hot2', 'hot3', 'hot4') ORDER BY status; + status +-------- + hot2 + hot3 + hot4 +(3 rows) + +SELECT status FROM public.cf_stream_cold WHERE status = 'cold3'; + status +-------- + cold3 +(1 row) + +-- (3) OVERRIDING SYSTEM VALUE supplies the identity on both tiers. +INSERT INTO public.cf_stream (id, ts, status) OVERRIDING SYSTEM VALUE +VALUES (901, '2026-01-11', 'ovr_cold'), (902, '2026-06-13', 'ovr_hot'); + hot_rows | cold_rows +----------+----------- + 1 | +(1 row) + +SELECT * FROM public.cf_stream_ids; + rows | ids | min_id | max_id +------+-----+--------+-------- + 9 | 9 | 1 | 902 +(1 row) + +-- (4) A prepared statement with a bound parameter streams from its cached plan +-- and keeps the parameter live. Run again in a transaction that wrote the +-- source table, the stream's check falls back to reading in this session, +-- so the row written in the transaction is tiered too. +PREPARE cf_prep(text) AS + INSERT INTO public.cf_stream (ts, status) SELECT ts, $1 FROM public.cf_staging WHERE ts < '2026-03-01'; +EXECUTE cf_prep('prep1'); + hot_rows | cold_rows +----------+----------- + 0 | +(1 row) + +BEGIN; +INSERT INTO public.cf_staging VALUES ('2026-01-20', 'cold4', NULL, NULL, NULL); +EXECUTE cf_prep('prep2'); + hot_rows | cold_rows +----------+----------- + 0 | 3 +(1 row) + +COMMIT; +BEGIN ISOLATION LEVEL REPEATABLE READ; +EXECUTE cf_prep('prep3'); + hot_rows | cold_rows +----------+----------- + 0 | 3 +(1 row) + +COMMIT; +SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'prep%' GROUP BY status ORDER BY status; + status | count +--------+------- + prep1 | 2 + prep2 | 3 + prep3 | 3 +(3 rows) + +-- (5) COPY into the view goes through the same rewrite, batch by batch. +SET coldfront.cold_write_batch_size = 2; +COPY public.cf_stream (ts, status) FROM stdin WITH (FORMAT csv); +RESET coldfront.cold_write_batch_size; +SELECT status, count(*) FROM public.cf_stream_hot WHERE status LIKE 'copy%' GROUP BY status; + status | count +----------+------- + copy_hot | 1 +(1 row) + +SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'copy%' GROUP BY status; + status | count +-----------+------- + copy_cold | 3 +(1 row) + +-- (6) Sources the hook sends to the sink in one pass: a volatile function, a +-- stable one, a temporary table, a WITH entry, a view that calls a stable +-- function, a table this transaction wrote, a transaction that is not READ +-- COMMITTED, and a session without local_pg_dsn. +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, random()::text FROM public.cf_staging; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: $4, $5 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: cf_staging.ts, (random())::text + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 (returns $4) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 (returns $5) + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) VALUES (now() - interval '1 year', 'stable'); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: $4, $5 + CTE coldfront_source + -> Result + Output: (now() - '@ 1 year'::interval), 'stable'::text + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 (returns $4) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 (returns $5) + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +CREATE TEMP TABLE cf_tmp (ts timestamptz, status text); +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM cf_tmp; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: $4, $5 + CTE coldfront_source + -> Seq Scan on pg_temp.cf_tmp + Output: cf_tmp.ts, cf_tmp.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 (returns $4) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 (returns $5) + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +EXPLAIN (COSTS OFF, VERBOSE) + WITH s AS (SELECT ts, status FROM public.cf_staging) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM s; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: $4, $5 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: cf_staging.ts, cf_staging.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 (returns $4) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 (returns $5) + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +CREATE VIEW public.cf_now_view AS SELECT now() AS ts, status FROM public.cf_staging; +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_now_view; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: $4, $5 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: now(), cf_staging.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 (returns $4) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 (returns $5) + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +BEGIN; +INSERT INTO public.cf_staging VALUES ('2026-01-30', 'cold5', NULL, NULL, NULL); +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: $4, $5 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: cf_staging.ts, cf_staging.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 (returns $4) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 (returns $5) + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +INSERT INTO public.cf_stream (ts, status) SELECT ts, 'txn' FROM public.cf_staging WHERE status = 'cold5'; + hot_rows | cold_rows +----------+----------- + 0 | 1 +(1 row) + +COMMIT; +BEGIN ISOLATION LEVEL REPEATABLE READ; +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: $4, $5 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: cf_staging.ts, cf_staging.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 (returns $4) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 (returns $5) + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +COMMIT; +SET coldfront.local_pg_dsn = ''; +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Result + Output: $4, $5 + CTE coldfront_source + -> Seq Scan on public.cf_staging + Output: cf_staging.ts, cf_staging.status + CTE coldfront_hot + -> Insert on public._cf_stream + Output: 1 + -> CTE Scan on coldfront_source + Output: nextval('_cf_stream_id_seq'), coldfront_source.ts, coldfront_source.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + CTE coldfront_cold + -> Aggregate + Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) + -> Subquery Scan on r + Output: r.* + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 4 (returns $4) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot + Output: coldfront_hot."?column?" + InitPlan 5 (returns $5) + -> CTE Scan on coldfront_cold + Output: coldfront_cold.n +(27 rows) + +SELECT status FROM public.cf_stream_cold WHERE status = 'txn'; + status +-------- + txn +(1 row) + +SELECT * FROM public.cf_stream_ids; + rows | ids | min_id | max_id +------+-----+--------+-------- + 22 | 22 | 1 | 902 +(1 row) + +-- Cleanup. +DEALLOCATE cf_prep; +DROP VIEW public.cf_now_view, public.cf_stream_hot, public.cf_stream_cold, public.cf_stream_ids; +DELETE FROM coldfront.tiered_views; +DELETE FROM coldfront.archive_watermark; +DROP VIEW public.cf_stream; +DROP TABLE public._cf_stream, public.cf_staging, cf_tmp; +RESET coldfront.local_pg_dsn; diff --git a/extension/coldfront/test/sql/async_write_before_claim.sql b/extension/coldfront/test/sql/async_write_before_claim.sql index 6dd797e..e21065e 100644 --- a/extension/coldfront/test/sql/async_write_before_claim.sql +++ b/extension/coldfront/test/sql/async_write_before_claim.sql @@ -64,6 +64,20 @@ COMMIT; INSERT INTO public._cf_order VALUES (20, '2026-05-01', 'hot'); UPDATE public.cf_order SET ts = '2026-01-05' WHERE id = 20; +-- (6) The cold stream: a tiered INSERT whose source streams is one DuckDB +-- write in the statement, and the claim at COMMIT meets the lock. The +-- stream's second session connects to the regress database. +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; +CREATE TABLE public.cf_order_src (id int, ts timestamptz, status text); +INSERT INTO public.cf_order_src VALUES (30, '2026-01-06', 'stream'), (31, '2026-01-07', 'stream'); +BEGIN; +INSERT INTO public.cf_order (id, ts, status) SELECT id, ts, status FROM public.cf_order_src; +COMMIT; +DROP TABLE public.cf_order_src; + -- Cleanup. SELECT coldfront._loopback('SELECT pg_advisory_unlock_all()') IS NOT NULL AS lock_released; DELETE FROM coldfront.tiered_views; @@ -75,3 +89,4 @@ RESET lock_timeout; RESET coldfront.iceberg_async_parquet; RESET coldfront.iceberg_bakery_patch; RESET coldfront.loopback_dsn; +RESET coldfront.local_pg_dsn; diff --git a/extension/coldfront/test/sql/param_cold_via_plpgsql.sql b/extension/coldfront/test/sql/param_cold_via_plpgsql.sql index e35c866..bffe358 100644 --- a/extension/coldfront/test/sql/param_cold_via_plpgsql.sql +++ b/extension/coldfront/test/sql/param_cold_via_plpgsql.sql @@ -75,10 +75,10 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE dual_upd('x'); EXPLAIN (COSTS OFF, VERBOSE) UPDATE public.events SET status = 'x' WHERE ts < '2026-03-01'; --- (8) tiered INSERT with an omitted IDENTITY column and a bytea param: the params --- stay native in the source CTE, which PostgreSQL runs; the hot INSERT and the --- cold projection each take nextval() for the identity, and the cold projection --- spells the bytea as hex text, independent of the session's bytea_output. +-- (8) tiered INSERT with an omitted IDENTITY column and a bytea param: the hot +-- INSERT keeps the params native, and the cold stream's text carries them as +-- format() arguments, each a literal the other session reads back and casts; +-- both halves take nextval() for the identity. CREATE TABLE public._eid (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz, data bytea); CREATE VIEW public.eid AS SELECT * FROM public._eid; INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) diff --git a/extension/coldfront/test/sql/tiered_insert_single_pass.sql b/extension/coldfront/test/sql/tiered_insert_single_pass.sql index aa43a0a..7f9fe9a 100644 --- a/extension/coldfront/test/sql/tiered_insert_single_pass.sql +++ b/extension/coldfront/test/sql/tiered_insert_single_pass.sql @@ -1,10 +1,14 @@ --- A tiered INSERT ... SELECT reads its source once: the rewrite holds the source --- in a MATERIALIZED CTE that the hot INSERT and the cold sink both read, casts --- every column to the hot table's type (an untyped literal keeps the target's --- type), fills an omitted identity column with nextval() and an omitted column --- with its DEFAULT on the cold side, and keeps OVERRIDING SYSTEM VALUE on the --- hot INSERT. White-box: EXPLAIN VERBOSE shows the rewritten shape; no Iceberg --- I/O (warehouse and endpoint left ''). +-- The two shapes of a tiered INSERT. A source the hook cannot read a second +-- time from another session (here a stable function, timestamptz plus an +-- interval) is read once: the rewrite holds it in a MATERIALIZED CTE that the +-- hot INSERT and the cold sink both read. A source it can read twice streams: +-- the hot INSERT reads it, and the cold half is one call carrying the cold +-- projection as text for the other session. Both cast every column to the hot +-- table's type (an untyped literal keeps the target's type), fill an omitted +-- identity column with nextval() and an omitted column with its DEFAULT on +-- the cold side, and keep OVERRIDING SYSTEM VALUE on the hot INSERT. +-- White-box: EXPLAIN VERBOSE shows the rewritten shape; no Iceberg I/O +-- (warehouse and endpoint left ''). SET client_min_messages = warning; CREATE EXTENSION IF NOT EXISTS pg_duckdb; CREATE EXTENSION IF NOT EXISTS coldfront; diff --git a/extension/coldfront/test/sql/tiered_insert_stream.sql b/extension/coldfront/test/sql/tiered_insert_stream.sql new file mode 100644 index 0000000..0d68ebb --- /dev/null +++ b/extension/coldfront/test/sql/tiered_insert_stream.sql @@ -0,0 +1,151 @@ +-- A tiered INSERT whose source the hook can read a second time from another +-- session streams its cold rows: the hot half is a set-based INSERT and the +-- cold half is one DuckDB INSERT ... SELECT FROM postgres_query('pgstream', ...) +-- through coldfront._exec_iceberg_with_claim, with the identity values and the +-- defaults drawn in that session. Observed end to end: a DuckDB temporary table +-- stands in for the Iceberg table and is read back with duckdb.query. A source +-- the hook cannot read twice (a volatile or stable function, a temporary table, +-- a WITH entry, a view over one, a table this transaction wrote, a transaction +-- that is not READ COMMITTED, no local_pg_dsn) goes through the cold sink in one +-- pass, and a cached plan that meets such a transaction falls back inside the +-- stream. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; + +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +-- The regress database is not the one the image's local_pg_dsn names. +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_stream (id BIGINT, ts TIMESTAMPTZ, status VARCHAR, n INTEGER, data VARCHAR, b BLOB, tags INTEGER[])') IS NOT NULL AS created; + +CREATE TABLE public._cf_stream (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz NOT NULL, + status text NOT NULL DEFAULT 'new', n int DEFAULT 7, + data jsonb, b bytea, tags int[]); +CREATE VIEW public.cf_stream AS SELECT * FROM public._cf_stream; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'cf_stream', 'public._cf_stream', 'temp.main.cf_stream', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'cf_stream', '2026-03-01'::timestamptz); +CREATE TABLE public.cf_staging (ts timestamptz, status text, data jsonb, b bytea, tags int[]); +INSERT INTO public.cf_staging VALUES + ('2026-01-01', 'cold1', '{"a": 1}', '\xcafe', '{1,2}'), + ('2026-02-01', 'cold2', NULL, NULL, NULL), + ('2026-05-01', 'hot1', '{}', '\xbeef', '{3}'); + +-- The rows of each tier, without the ids, which the two halves draw from one +-- sequence in no fixed order. The cold read runs in DuckDB, so the two reads +-- are separate views. +CREATE VIEW public.cf_stream_hot AS + SELECT ts, status, n, data::text AS data, b, tags FROM public._cf_stream; +CREATE VIEW public.cf_stream_cold AS + SELECT r['ts']::timestamptz AS ts, r['status']::text AS status, r['n']::int AS n, + r['data']::text AS data, r['b']::bytea AS b, r['tags']::int[] AS tags + FROM duckdb.query('SELECT * FROM temp.main.cf_stream') AS t(r); +CREATE VIEW public.cf_stream_ids AS + SELECT count(*) AS rows, count(DISTINCT id) AS ids, min(id) AS min_id, max(id) AS max_id + FROM (SELECT id FROM public._cf_stream + UNION ALL + SELECT r['id']::bigint FROM duckdb.query('SELECT id FROM temp.main.cf_stream') AS t(r)) AS u; + +-- (1) A SELECT source over a table: the hot row goes to the heap, the two cold +-- rows stream, every column type lands as the sink would store it, and +-- the omitted identity and default columns are filled on both tiers. +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status, data, b, tags) + SELECT ts, status, data, b, tags FROM public.cf_staging; +INSERT INTO public.cf_stream (ts, status, data, b, tags) +SELECT ts, status, data, b, tags FROM public.cf_staging; +SELECT * FROM public.cf_stream_hot ORDER BY ts; +SELECT * FROM public.cf_stream_cold ORDER BY ts; +SELECT * FROM public.cf_stream_ids; + +-- (2) A VALUES source: the cold test is the hot count against the row count. +-- With a cold row DuckDB writes (and reports it); with none it is never +-- called, so an all-hot INSERT stages nothing. +SET client_min_messages = notice; +INSERT INTO public.cf_stream (ts, status) VALUES ('2026-01-10', 'cold3'), ('2026-06-10', 'hot2'); +INSERT INTO public.cf_stream (ts, status) VALUES ('2026-06-11', 'hot3'), ('2026-06-12', 'hot4'); +SET client_min_messages = warning; +SELECT status FROM public.cf_stream_hot WHERE status IN ('hot2', 'hot3', 'hot4') ORDER BY status; +SELECT status FROM public.cf_stream_cold WHERE status = 'cold3'; + +-- (3) OVERRIDING SYSTEM VALUE supplies the identity on both tiers. +INSERT INTO public.cf_stream (id, ts, status) OVERRIDING SYSTEM VALUE +VALUES (901, '2026-01-11', 'ovr_cold'), (902, '2026-06-13', 'ovr_hot'); +SELECT * FROM public.cf_stream_ids; + +-- (4) A prepared statement with a bound parameter streams from its cached plan +-- and keeps the parameter live. Run again in a transaction that wrote the +-- source table, the stream's check falls back to reading in this session, +-- so the row written in the transaction is tiered too. +PREPARE cf_prep(text) AS + INSERT INTO public.cf_stream (ts, status) SELECT ts, $1 FROM public.cf_staging WHERE ts < '2026-03-01'; +EXECUTE cf_prep('prep1'); +BEGIN; +INSERT INTO public.cf_staging VALUES ('2026-01-20', 'cold4', NULL, NULL, NULL); +EXECUTE cf_prep('prep2'); +COMMIT; +BEGIN ISOLATION LEVEL REPEATABLE READ; +EXECUTE cf_prep('prep3'); +COMMIT; +SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'prep%' GROUP BY status ORDER BY status; + +-- (5) COPY into the view goes through the same rewrite, batch by batch. +SET coldfront.cold_write_batch_size = 2; +COPY public.cf_stream (ts, status) FROM stdin WITH (FORMAT csv); +2026-01-21 00:00:00+00,copy_cold +2026-01-22 00:00:00+00,copy_cold +2026-01-23 00:00:00+00,copy_cold +2026-06-21 00:00:00+00,copy_hot +\. +RESET coldfront.cold_write_batch_size; +SELECT status, count(*) FROM public.cf_stream_hot WHERE status LIKE 'copy%' GROUP BY status; +SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'copy%' GROUP BY status; + +-- (6) Sources the hook sends to the sink in one pass: a volatile function, a +-- stable one, a temporary table, a WITH entry, a view that calls a stable +-- function, a table this transaction wrote, a transaction that is not READ +-- COMMITTED, and a session without local_pg_dsn. +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, random()::text FROM public.cf_staging; +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) VALUES (now() - interval '1 year', 'stable'); +CREATE TEMP TABLE cf_tmp (ts timestamptz, status text); +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM cf_tmp; +EXPLAIN (COSTS OFF, VERBOSE) + WITH s AS (SELECT ts, status FROM public.cf_staging) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM s; +CREATE VIEW public.cf_now_view AS SELECT now() AS ts, status FROM public.cf_staging; +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_now_view; +BEGIN; +INSERT INTO public.cf_staging VALUES ('2026-01-30', 'cold5', NULL, NULL, NULL); +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; +INSERT INTO public.cf_stream (ts, status) SELECT ts, 'txn' FROM public.cf_staging WHERE status = 'cold5'; +COMMIT; +BEGIN ISOLATION LEVEL REPEATABLE READ; +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; +COMMIT; +SET coldfront.local_pg_dsn = ''; +EXPLAIN (COSTS OFF, VERBOSE) + INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; +SELECT status FROM public.cf_stream_cold WHERE status = 'txn'; +SELECT * FROM public.cf_stream_ids; + +-- Cleanup. +DEALLOCATE cf_prep; +DROP VIEW public.cf_now_view, public.cf_stream_hot, public.cf_stream_cold, public.cf_stream_ids; +DELETE FROM coldfront.tiered_views; +DELETE FROM coldfront.archive_watermark; +DROP VIEW public.cf_stream; +DROP TABLE public._cf_stream, public.cf_staging, cf_tmp; +RESET coldfront.local_pg_dsn; From 98d039f1e749820e36a26bf4194b3749f25fc8dc Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Thu, 8 Oct 2026 15:41:51 +0100 Subject: [PATCH 3/9] perf: gather the cold sink's batches with string_agg --- ci/journey.sh | 2 +- docs/architecture_tiered.md | 27 ++- docs/changelog.md | 5 +- extension/coldfront/coldfront--1.0.sql | 147 ++++++-------- extension/coldfront/src/coldfront.c | 50 ++++- .../expected/cold_write_batch_size_guc.out | 8 +- .../test/expected/cold_write_json_agg.out | 22 +- .../test/expected/cold_write_json_agg_1.out | 21 +- .../coldfront/test/expected/cte_on_insert.out | 22 +- .../test/expected/cte_on_insert_1.out | 21 +- .../expected/tiered_insert_single_pass.out | 24 ++- .../expected/tiered_insert_single_pass_1.out | 23 ++- .../test/expected/tiered_insert_stream.out | 192 +++++++++++------- .../test/expected/tiered_insert_stream_1.out | 184 ++++++++++------- .../test/sql/cold_write_batch_size_guc.sql | 8 +- 15 files changed, 455 insertions(+), 301 deletions(-) diff --git a/ci/journey.sh b/ci/journey.sh index 7f1e090..bce8e6e 100755 --- a/ci/journey.sh +++ b/ci/journey.sh @@ -4207,7 +4207,7 @@ story_standby_reads() { fi # The cold sink's step function is reached through the INSERT rewrite, so call # it directly: the guard is its first statement, ahead of every lookup. - local c; c=$(q_may "$STANDBY" "SELECT coldfront._cold_sink_step(NULL, '${vn%%.*}', '${vn##*.}', '{}'::jsonb);") + local c; c=$(q_may "$STANDBY" "SELECT coldfront._cold_sink_step(NULL, '${vn%%.*}', '${vn##*.}', '', 0);") assert_err "cold sink on standby → coldfront refuses before the claim" \ "cannot execute a cold (Iceberg) write on a read-only standby" "$c" diff --git a/docs/architecture_tiered.md b/docs/architecture_tiered.md index 2f38438..3f0dd9f 100644 --- a/docs/architecture_tiered.md +++ b/docs/architecture_tiered.md @@ -373,9 +373,16 @@ coldfront_hot AS MATERIALIZED ( RETURNING 1 ), coldfront_cold AS MATERIALIZED ( - SELECT coldfront._cold_sink('public', 'events', to_jsonb(r)) AS n - FROM (SELECT nextval('_events_id_seq') AS id, ts, status, data - FROM coldfront_source WHERE ts < ''::timestamptz) AS r + SELECT coldfront._cold_sink('public', 'events', b.tuples, b.n) AS n + FROM (SELECT string_agg('(' || coldfront._cold_row_literal(to_jsonb(r), '{id,ts,status,data}', + '{bigint,"timestamp with time zone",text,jsonb}', 'public', 'events') || ')', + ', ') AS tuples, + count(*) AS n + FROM (SELECT r0.*, (row_number() OVER () - 1) + / current_setting('coldfront.cold_write_batch_size')::int AS coldfront_batch + FROM (SELECT nextval('_events_id_seq') AS id, ts, status, data + FROM coldfront_source WHERE ts < ''::timestamptz) AS r0) AS r + GROUP BY r.coldfront_batch) AS b ) SELECT (SELECT count(*) FROM coldfront_hot) AS hot_rows, (SELECT n FROM coldfront_cold) AS cold_rows; @@ -384,12 +391,14 @@ SELECT (SELECT count(*) FROM coldfront_hot) AS hot_rows, The source runs once, into the `coldfront_source` tuplestore, and both halves read it, so a volatile source lands every row exactly once and a row the transaction wrote before the `INSERT` reaches the cold tier. The cold half -projects each row as the stream does and hands it to `coldfront._cold_sink`, -which renders each row as a DuckDB `VALUES` tuple and writes every -`coldfront.cold_write_batch_size` rows (default 10000) as one `INSERT` through -`coldfront._exec_iceberg_with_claim`; its final step writes the rest. The sink -renders in plpgsql, so a large load that cannot stream is slower than one -that can. +projects each row as the stream does, renders it as a DuckDB `VALUES` tuple +(`coldfront._cold_row_literal`), gathers `coldfront.cold_write_batch_size` tuples +(default 10000) at a time with `string_agg`, and hands each batch to the +`coldfront._cold_sink` aggregate, which writes it as one `INSERT` through +`coldfront._exec_iceberg_with_claim`. The statement is built by +`coldfront._cold_sink_sql`, which the stream's fallback runs too. The work per +row is the rendering in plpgsql, so a load that cannot stream costs in +proportion to its rows and more per row than one that can. In both shapes each column of the source is cast to the hot table's type, which restates the coercions PostgreSQL applied when it analyzed the statement diff --git a/docs/changelog.md b/docs/changelog.md index 46da9e9..e4932de 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -86,8 +86,9 @@ this project adheres to A source the hook cannot read a second time (a volatile or stable function, a temporary table, a `WITH` entry, a table the transaction has written, a transaction that is not READ COMMITTED) is read once for both tiers and its - cold rows go through `coldfront._cold_sink`. `cold_rows` in the statement's - result is `NULL` for a streamed `INSERT`. + cold rows go through `coldfront._cold_sink`, whose cost is in proportion to + the rows it renders. `cold_rows` in the + statement's result is `NULL` for a streamed `INSERT`. - Every append to the cold tier (`INSERT`, `COPY`, the archiver's exports, the tiered `INSERT`'s cold sink) uses the async ordering wherever the patched duckdb-iceberg is loaded, on a single node as on a mesh: the statement diff --git a/extension/coldfront/coldfront--1.0.sql b/extension/coldfront/coldfront--1.0.sql index bfc8eed..9445db6 100644 --- a/extension/coldfront/coldfront--1.0.sql +++ b/extension/coldfront/coldfront--1.0.sql @@ -527,9 +527,9 @@ BEGIN '_exec_iceberg_with_claim', -- the tiered INSERT's cold stream and its execution-time check '_tiered_cold_stream', '_written_in_xact', - -- the tiered INSERT's cold sink and its row renderer - '_cold_sink', '_cold_sink_step', '_cold_sink_flush', - '_cold_sink_final', '_cold_row_literal', + -- the tiered INSERT's cold sink, its statement and its row renderer + '_cold_sink', '_cold_sink_step', '_cold_sink_final', + '_cold_sink_sql', '_cold_row_literal', -- an array column's shape check in that INSERT, and an -- array parameter's renderer in a cold write '_cold_list', '_render_cold_param', @@ -931,27 +931,25 @@ $$; -- -- The C hook rewrites INSERT INTO ... into one statement that -- holds the source in a MATERIALIZED CTE, inserts the hot rows into the hot --- table from it, and aggregates the cold rows from it with _cold_sink. Each --- row reaches the sink as jsonb with one key per hot-table column, projected --- in the statement: the user's value, nextval() for an omitted identity --- column, the DEFAULT expression for an omitted column that has one, NULL --- otherwise. The sink renders and batches and does nothing else: the step --- function renders each row as a DuckDB VALUES tuple and flushes every --- coldfront.cold_write_batch_size rows as one INSERT through --- _exec_iceberg_with_claim, and the final function flushes the rest and --- returns the row count. Each batch is an append, so in the async ordering --- every batch is staged inside the statement and the table's one claim is --- taken at the transaction's commit; in the stock ordering the claim comes --- before the first batch. The state is NULL until the first row, so an --- aggregate over no cold rows writes nothing and returns 0. +-- table from it, and runs the statement _cold_sink_sql builds over the cold +-- rows from it. Each row is projected to one key per hot-table column: the +-- user's value, nextval() for an omitted identity column, the DEFAULT +-- expression for an omitted column that has one, NULL otherwise; the +-- statement renders each row as a DuckDB VALUES tuple (_cold_row_literal), +-- gathers coldfront.cold_write_batch_size of them at a time with string_agg, +-- and hands each batch to the _cold_sink aggregate, whose step writes it as +-- one INSERT through _exec_iceberg_with_claim and whose final function returns +-- the row count. The work per row is the rendering alone: the batches are +-- built by string_agg, so a large load costs in proportion to its rows. Each +-- batch is an append, so in the async ordering every batch is staged inside +-- the statement and the table's one claim is taken at the transaction's +-- commit; in the stock ordering the claim comes before the first batch. The +-- state is NULL until the first batch, so an aggregate over no cold rows +-- writes nothing and returns 0. -- ============================================================================ CREATE TYPE coldfront._cold_sink_state AS ( iceberg text, -- the DuckDB ref the batches are written to - cols text[], -- hot-table columns in attnum order, vector companions excluded - types text[], -- their PG types, for _cold_row_literal - batch int, -- rows per flush - buf text[], -- rendered tuples waiting for a flush - total bigint -- rows rendered so far + total bigint -- rows written so far ); -- coldfront._cold_row_literal: render one row (jsonb keyed by column name) as a @@ -999,86 +997,66 @@ BEGIN END; $$; --- One batch to Iceberg, in the ordering every cold write uses. -CREATE FUNCTION coldfront._cold_sink_flush(s coldfront._cold_sink_state) -RETURNS coldfront._cold_sink_state -LANGUAGE plpgsql AS $$ -BEGIN - IF cardinality(s.buf) > 0 THEN - PERFORM coldfront._exec_iceberg_with_claim(s.iceberg, - format('INSERT INTO %s VALUES %s', s.iceberg, array_to_string(s.buf, ', '))); - s.buf := '{}'; - END IF; - RETURN s; -END; +-- coldfront._cold_sink_sql: the statement that feeds the sink, over p_rows_sql, +-- a query yielding the cold rows projected to every hot-table column (p_cols, +-- in attnum order, with p_types their PG types). Each row is rendered as a +-- VALUES tuple, the tuples are gathered coldfront.cold_write_batch_size at a +-- time, read at execution so a cached plan follows the session's setting, and +-- each batch reaches the aggregate with its row count. The C hook embeds the +-- statement in a tiered INSERT's rewrite and _tiered_cold_stream runs it when +-- it falls back to reading in this session, so the shape has one home. +CREATE FUNCTION coldfront._cold_sink_sql( + p_schema text, p_view text, p_cols text[], p_types text[], p_rows_sql text +) RETURNS text +LANGUAGE sql IMMUTABLE AS $$ + SELECT format( + 'SELECT coldfront._cold_sink(%1$L, %2$L, b.tuples, b.n) AS n ' + 'FROM (SELECT string_agg(''('' || coldfront._cold_row_literal(to_jsonb(r), %3$L::text[], %4$L::text[], %1$L, %2$L) || '')'', '', '') AS tuples, ' + 'count(*) AS n ' + 'FROM (SELECT r0.*, (row_number() OVER () - 1) / current_setting(''coldfront.cold_write_batch_size'')::int AS coldfront_batch ' + 'FROM (%5$s) AS r0) AS r ' + 'GROUP BY r.coldfront_batch) AS b', + p_schema, p_view, p_cols, p_types, p_rows_sql); $$; --- The step function. The first row resolves the registry row and the hot --- table's columns, with the standby guard ahead of every lookup. +-- The step function takes one rendered batch. The first call resolves the +-- registry row, with the standby guard ahead of the lookup. CREATE FUNCTION coldfront._cold_sink_step( - s coldfront._cold_sink_state, p_schema text, p_view text, p_row jsonb + s coldfront._cold_sink_state, p_schema text, p_view text, p_tuples text, p_n bigint ) RETURNS coldfront._cold_sink_state LANGUAGE plpgsql AS $$ -DECLARE - v_hot_table text; - v_iceberg text; BEGIN IF s.iceberg IS NULL THEN PERFORM coldfront._reject_on_standby('execute a cold (Iceberg) write'); -- The statement may run from a cached plan (PREPARE, a plpgsql -- function's later call) in a transaction the hook never saw, so the -- catalog attach and pg_duckdb's mixed-write guard belong here, at - -- execution, ahead of the first flush. + -- execution, ahead of the first batch. PERFORM coldfront.ensure_attached(); SET LOCAL duckdb.unsafe_allow_mixed_transactions = on; - SELECT tv.hot_table, tv.iceberg_table INTO v_hot_table, v_iceberg + SELECT tv.iceberg_table, 0::bigint INTO s FROM coldfront.tiered_views tv WHERE tv.schema_name = p_schema AND tv.relname = p_view AND NOT tv.is_iceberg_only; - IF v_iceberg IS NULL THEN + IF s.iceberg IS NULL THEN RAISE EXCEPTION 'coldfront._cold_sink: view %.% is not a registered tiered view', p_schema, p_view; END IF; - SELECT v_iceberg, - array_agg(a.attname::text ORDER BY a.attnum), - array_agg(format_type(a.atttypid, a.atttypmod) ORDER BY a.attnum), - current_setting('coldfront.cold_write_batch_size')::int, - '{}'::text[], 0::bigint - INTO s - FROM pg_attribute a - JOIN pg_class c ON c.oid = a.attrelid - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = (parse_ident(v_hot_table))[1] - AND c.relname = (parse_ident(v_hot_table))[2] - AND a.attnum > 0 AND NOT a.attisdropped - AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated); - -- A cluster assignment reads the centroids over pglocal. - IF coldfront._types_have_vector(s.types) THEN - PERFORM coldfront.ensure_pg_attached(); - END IF; - END IF; - s.buf := s.buf || ('(' || coldfront._cold_row_literal(p_row, s.cols, s.types, p_schema, p_view) || ')'); - s.total := s.total + 1; - IF cardinality(s.buf) >= s.batch THEN - s := coldfront._cold_sink_flush(s); END IF; + PERFORM coldfront._exec_iceberg_with_claim(s.iceberg, + format('INSERT INTO %s VALUES %s', s.iceberg, p_tuples)); + s.total := s.total + p_n; RETURN s; END; $$; CREATE FUNCTION coldfront._cold_sink_final(s coldfront._cold_sink_state) RETURNS bigint -LANGUAGE plpgsql AS $$ -BEGIN - IF s.iceberg IS NULL THEN - RETURN 0; - END IF; - s := coldfront._cold_sink_flush(s); - RETURN s.total; -END; +LANGUAGE sql AS $$ + SELECT COALESCE(s.total, 0); $$; -CREATE AGGREGATE coldfront._cold_sink(text, text, jsonb) ( +CREATE AGGREGATE coldfront._cold_sink(text, text, text, bigint) ( SFUNC = coldfront._cold_sink_step, STYPE = coldfront._cold_sink_state, FINALFUNC = coldfront._cold_sink_final @@ -1120,19 +1098,14 @@ LANGUAGE plpgsql AS $$ DECLARE v_hot text; v_iceberg text; - v_cols text; + v_names text[]; + v_types text[]; v_prefix text; v_count bigint; BEGIN IF NOT p_any_cold THEN RETURN 0; END IF; - IF current_setting('transaction_isolation') <> 'read committed' - OR coldfront._written_in_xact(p_sources) THEN - EXECUTE format('SELECT coldfront._cold_sink(%L, %L, to_jsonb(r)) FROM (%s) AS r', - p_schema, p_view, p_pg_sql) INTO v_count; - RETURN v_count; - END IF; SELECT tv.hot_table, tv.iceberg_table INTO v_hot, v_iceberg FROM coldfront.tiered_views tv WHERE tv.schema_name = p_schema AND tv.relname = p_view @@ -1141,7 +1114,9 @@ BEGIN RAISE EXCEPTION 'coldfront._tiered_cold_stream: view %.% is not a registered tiered view', p_schema, p_view; END IF; - SELECT string_agg('s.' || quote_ident(a.attname), ', ' ORDER BY a.attnum) INTO v_cols + SELECT array_agg(a.attname::text ORDER BY a.attnum), + array_agg(format_type(a.atttypid, a.atttypmod) ORDER BY a.attnum) + INTO v_names, v_types FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid JOIN pg_namespace n ON n.oid = c.relnamespace @@ -1149,13 +1124,21 @@ BEGIN AND c.relname = (parse_ident(v_hot))[2] AND a.attnum > 0 AND NOT a.attisdropped AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated); + IF current_setting('transaction_isolation') <> 'read committed' + OR coldfront._written_in_xact(p_sources) THEN + EXECUTE coldfront._cold_sink_sql(p_schema, p_view, v_names, v_types, p_pg_sql) + INTO v_count; + RETURN v_count; + END IF; v_prefix := COALESCE(coldfront._vec_list_prefix_for_ref(v_iceberg, 's.'), ''); SET LOCAL duckdb.unsafe_allow_mixed_transactions = on; PERFORM coldfront.ensure_attached(); PERFORM coldfront.ensure_stream_attached(); PERFORM coldfront._exec_iceberg_with_claim(v_iceberg, format( 'INSERT INTO %s SELECT %s%s FROM postgres_query(%L, %L) AS s%s', - v_iceberg, v_prefix, v_cols, 'pgstream', p_pg_sql, + v_iceberg, v_prefix, + (SELECT string_agg('s.' || quote_ident(c), ', ' ORDER BY o) FROM unnest(v_names) WITH ORDINALITY AS u(c, o)), + 'pgstream', p_pg_sql, CASE WHEN v_prefix <> '' THEN ' ORDER BY 1' ELSE '' END)); RETURN NULL; END; diff --git a/extension/coldfront/src/coldfront.c b/extension/coldfront/src/coldfront.c index 5cd89d8..16c8efc 100644 --- a/extension/coldfront/src/coldfront.c +++ b/extension/coldfront/src/coldfront.c @@ -2671,18 +2671,50 @@ cf_stream_pg_select(Query *query, TieredViewInfo *info, const HotColumns *hc, return cold_sql_arg(sql, ps, true); } -/* The cold half of the sink shape: coldfront._cold_sink over the source rows - * below the cutoff, each projected to every hot-table column - * (build_cold_projection). */ +/* The hot table's column names, or types, as a text[] literal. */ static char * -build_cold_sink_select(const char *vschema, const char *vname, +cf_text_array(char **items, int n) +{ + StringInfoData buf; + int i; + + initStringInfo(&buf); + appendStringInfoString(&buf, "ARRAY["); + for (i = 0; i < n; i++) + appendStringInfo(&buf, "%s%s", i ? ", " : "", quote_literal_cstr(items[i])); + appendStringInfoString(&buf, "]::text[]"); + return buf.data; +} + +/* The cold half of the sink shape: the statement coldfront._cold_sink_sql + * builds over the source rows below the cutoff, each projected to every + * hot-table column (build_cold_projection). */ +static char * +build_cold_sink_select(const char *vschema, const char *vname, const HotColumns *hc, const char *projection, const char *partition_col, const char *cutoff_lit) { - return psprintf("SELECT coldfront._cold_sink(%s, %s, to_jsonb(r)) AS n " - "FROM (SELECT %s FROM coldfront_source WHERE coldfront_source.%s < %s) AS r", - quote_literal_cstr(vschema), quote_literal_cstr(vname), - projection, quote_identifier(partition_col), cutoff_lit); + char *sql = NULL; + + if (SPI_connect() != SPI_OK_CONNECT) + elog(ERROR, "coldfront: SPI_connect failed while building the cold sink's statement"); + if (SPI_execute(psprintf("SELECT coldfront._cold_sink_sql(%s, %s, %s, %s, %s)", + quote_literal_cstr(vschema), quote_literal_cstr(vname), + cf_text_array(hc->name, hc->n), cf_text_array(hc->type, hc->n), + quote_literal_cstr(psprintf( + "SELECT %s FROM coldfront_source WHERE coldfront_source.%s < %s", + projection, quote_identifier(partition_col), cutoff_lit))), + true, 1) == SPI_OK_SELECT && SPI_processed == 1) + { + MemoryContext oldcxt = MemoryContextSwitchTo(CurTransactionContext); + + sql = SPI_getvalue(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1); + MemoryContextSwitchTo(oldcxt); + } + SPI_finish(); + if (sql == NULL) + elog(ERROR, "coldfront: coldfront._cold_sink_sql returned nothing"); + return sql; } /* The deparsed INSERT spells OVERRIDING SYSTEM VALUE or OVERRIDING USER VALUE @@ -2836,7 +2868,7 @@ emit_tiered_insert(Query *query, TieredViewInfo *info, ColdParamSet *ps, hot_dml = build_tiered_hot_dml(info->hot_table, col_list, override, col_list, "FROM coldfront_source", quote_identifier(info->partition_col), cutoff_lit); - cold_select = build_cold_sink_select(vschema, vname, + cold_select = build_cold_sink_select(vschema, vname, &hc, build_cold_projection(query, &hc, false), info->partition_col, cutoff_lit); return wrap_tiered_result(in_plpgsql, &dr, build_src_cte(query, source, col_list, &hc), diff --git a/extension/coldfront/test/expected/cold_write_batch_size_guc.out b/extension/coldfront/test/expected/cold_write_batch_size_guc.out index ae484b6..df054eb 100644 --- a/extension/coldfront/test/expected/cold_write_batch_size_guc.out +++ b/extension/coldfront/test/expected/cold_write_batch_size_guc.out @@ -1,8 +1,8 @@ -- coldfront.cold_write_batch_size is a GUC (DefineCustomIntVariable; default --- 10000, min 1) that the cold sink (coldfront._cold_sink_step) reads to size each --- cold-tier Iceberg flush (one duckdb.raw_query per batch). White-box: this --- checks the GUC's definition, default, settability, and lower bound — the flush --- behavior itself is exercised live in ci/journey.sh. +-- 10000, min 1) that the cold sink's statement (coldfront._cold_sink_sql) reads +-- at execution to size each cold-tier Iceberg batch (one INSERT per batch). +-- White-box: this checks the GUC's definition, default, settability, and lower +-- bound; the batching itself is exercised live in ci/journey.sh. CREATE EXTENSION IF NOT EXISTS pg_duckdb; NOTICE: extension "pg_duckdb" already exists, skipping CREATE EXTENSION IF NOT EXISTS coldfront; diff --git a/extension/coldfront/test/expected/cold_write_json_agg.out b/extension/coldfront/test/expected/cold_write_json_agg.out index 007a626..1521ce7 100644 --- a/extension/coldfront/test/expected/cold_write_json_agg.out +++ b/extension/coldfront/test/expected/cold_write_json_agg.out @@ -61,8 +61,8 @@ WHERE ts < '2019-01-01'::timestamptz; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events SELECT 9, '2019-01-01'::timestamptz, jsonb_agg(k ORDER BY k) FROM public.src; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -81,10 +81,18 @@ SELECT 9, '2019-01-01'::timestamptz, jsonb_agg(k ORDER BY k) FROM public.src; Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, to_jsonb(ROW(coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.data))) - -> CTE Scan on coldfront_source coldfront_source_1 - Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.data - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.data, ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.data + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -93,7 +101,7 @@ SELECT 9, '2019-01-01'::timestamptz, jsonb_agg(k ORDER BY k) FROM public.src; InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(30 rows) +(38 rows) -- (A5) An aggregate nested inside an object builder: the added paren must land -- at the aggregate's own close, leaving the builder's intact. diff --git a/extension/coldfront/test/expected/cold_write_json_agg_1.out b/extension/coldfront/test/expected/cold_write_json_agg_1.out index de0c118..063d903 100644 --- a/extension/coldfront/test/expected/cold_write_json_agg_1.out +++ b/extension/coldfront/test/expected/cold_write_json_agg_1.out @@ -61,8 +61,8 @@ WHERE ts < '2019-01-01'::timestamptz; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events SELECT 9, '2019-01-01'::timestamptz, jsonb_agg(k ORDER BY k) FROM public.src; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -81,10 +81,17 @@ SELECT 9, '2019-01-01'::timestamptz, jsonb_agg(k ORDER BY k) FROM public.src; Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, to_jsonb(ROW(coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.data))) - -> CTE Scan on coldfront_source coldfront_source_1 - Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.data - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.data, ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.data + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -93,7 +100,7 @@ SELECT 9, '2019-01-01'::timestamptz, jsonb_agg(k ORDER BY k) FROM public.src; InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(30 rows) +(37 rows) -- (A5) An aggregate nested inside an object builder: the added paren must land -- at the aggregate's own close, leaving the builder's intact. diff --git a/extension/coldfront/test/expected/cte_on_insert.out b/extension/coldfront/test/expected/cte_on_insert.out index e2619b9..67ea03d 100644 --- a/extension/coldfront/test/expected/cte_on_insert.out +++ b/extension/coldfront/test/expected/cte_on_insert.out @@ -28,8 +28,8 @@ CREATE TABLE public.staging (id int, ts timestamptz, status text); EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT 7 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, 'new' AS status) INSERT INTO public.events SELECT id, ts, status FROM s; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -43,10 +43,18 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, to_jsonb(ROW(coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status))) - -> CTE Scan on coldfront_source coldfront_source_1 - Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status, ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -55,7 +63,7 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(25 rows) +(33 rows) -- An entry may take a name the rewrite's own entries once had. WITH src AS (SELECT 1 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, 'src' AS status) diff --git a/extension/coldfront/test/expected/cte_on_insert_1.out b/extension/coldfront/test/expected/cte_on_insert_1.out index 4db0a7c..e8f1430 100644 --- a/extension/coldfront/test/expected/cte_on_insert_1.out +++ b/extension/coldfront/test/expected/cte_on_insert_1.out @@ -28,8 +28,8 @@ CREATE TABLE public.staging (id int, ts timestamptz, status text); EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT 7 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, 'new' AS status) INSERT INTO public.events SELECT id, ts, status FROM s; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -43,10 +43,17 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, to_jsonb(ROW(coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status))) - -> CTE Scan on coldfront_source coldfront_source_1 - Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status, ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: coldfront_source_1.id, coldfront_source_1.ts, coldfront_source_1.status + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -55,7 +62,7 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(25 rows) +(32 rows) -- An entry may take a name the rewrite's own entries once had. WITH src AS (SELECT 1 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, 'src' AS status) diff --git a/extension/coldfront/test/expected/tiered_insert_single_pass.out b/extension/coldfront/test/expected/tiered_insert_single_pass.out index 254dbe2..459af75 100644 --- a/extension/coldfront/test/expected/tiered_insert_single_pass.out +++ b/extension/coldfront/test/expected/tiered_insert_single_pass.out @@ -27,8 +27,8 @@ VALUES ('public', 'events', '2026-03-01'::timestamptz); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (ts, data) SELECT '2026-01-01'::timestamptz + i * interval '1 day', '{}' FROM generate_series(1, 3) i; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -43,12 +43,18 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_events_id_seq'::regclass), coldfront_source_1.ts, 'new'::text, coldfront_source_1.data - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_events_id_seq'::regclass)), coldfront_source_1.ts, ('new'::text), coldfront_source_1.data, ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_events_id_seq'::regclass), coldfront_source_1.ts, 'new'::text, coldfront_source_1.data + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -57,7 +63,7 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(28 rows) +(34 rows) -- OVERRIDING SYSTEM VALUE supplies the identity; the cold side takes it as given. EXPLAIN (COSTS OFF, VERBOSE) diff --git a/extension/coldfront/test/expected/tiered_insert_single_pass_1.out b/extension/coldfront/test/expected/tiered_insert_single_pass_1.out index 16dc6ec..6cf930a 100644 --- a/extension/coldfront/test/expected/tiered_insert_single_pass_1.out +++ b/extension/coldfront/test/expected/tiered_insert_single_pass_1.out @@ -27,8 +27,8 @@ VALUES ('public', 'events', '2026-03-01'::timestamptz); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (ts, data) SELECT '2026-01-01'::timestamptz + i * interval '1 day', '{}' FROM generate_series(1, 3) i; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -43,12 +43,17 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_events_id_seq'::regclass), coldfront_source_1.ts, 'new'::text, coldfront_source_1.data - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_events_id_seq'::regclass)), coldfront_source_1.ts, ('new'::text), coldfront_source_1.data, ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_events_id_seq'::regclass), coldfront_source_1.ts, 'new'::text, coldfront_source_1.data + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -57,7 +62,7 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(28 rows) +(33 rows) -- OVERRIDING SYSTEM VALUE supplies the identity; the cold side takes it as given. EXPLAIN (COSTS OFF, VERBOSE) diff --git a/extension/coldfront/test/expected/tiered_insert_stream.out b/extension/coldfront/test/expected/tiered_insert_stream.out index f9a34f6..258cfaf 100644 --- a/extension/coldfront/test/expected/tiered_insert_stream.out +++ b/extension/coldfront/test/expected/tiered_insert_stream.out @@ -258,8 +258,8 @@ SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'copy%' GRO -- COMMITTED, and a session without local_pg_dsn. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, random()::text FROM public.cf_staging; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -273,12 +273,18 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -287,12 +293,12 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(33 rows) EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) VALUES (now() - interval '1 year', 'stable'); - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -306,12 +312,18 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -320,13 +332,13 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(33 rows) CREATE TEMP TABLE cf_tmp (ts timestamptz, status text); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM cf_tmp; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -340,12 +352,18 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -354,13 +372,13 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(33 rows) EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT ts, status FROM public.cf_staging) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM s; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -374,12 +392,18 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -388,13 +412,13 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(33 rows) CREATE VIEW public.cf_now_view AS SELECT now() AS ts, status FROM public.cf_staging; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_now_view; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -408,12 +432,18 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -422,14 +452,14 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(33 rows) BEGIN; INSERT INTO public.cf_staging VALUES ('2026-01-30', 'cold5', NULL, NULL, NULL); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -443,12 +473,18 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -457,7 +493,7 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(33 rows) INSERT INTO public.cf_stream (ts, status) SELECT ts, 'txn' FROM public.cf_staging WHERE status = 'cold5'; hot_rows | cold_rows @@ -469,8 +505,8 @@ COMMIT; BEGIN ISOLATION LEVEL REPEATABLE READ; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -484,12 +520,18 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -498,14 +540,14 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(33 rows) COMMIT; SET coldfront.local_pg_dsn = ''; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -519,12 +561,18 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER w1 - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + Window: w1 AS (ROWS UNBOUNDED PRECEDING) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 -> Aggregate Output: count(*) @@ -533,7 +581,7 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(33 rows) SELECT status FROM public.cf_stream_cold WHERE status = 'txn'; status diff --git a/extension/coldfront/test/expected/tiered_insert_stream_1.out b/extension/coldfront/test/expected/tiered_insert_stream_1.out index ae099b0..d64334e 100644 --- a/extension/coldfront/test/expected/tiered_insert_stream_1.out +++ b/extension/coldfront/test/expected/tiered_insert_stream_1.out @@ -258,8 +258,8 @@ SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'copy%' GRO -- COMMITTED, and a session without local_pg_dsn. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, random()::text FROM public.cf_staging; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 CTE coldfront_source @@ -273,12 +273,17 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -287,12 +292,12 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(32 rows) EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) VALUES (now() - interval '1 year', 'stable'); - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 CTE coldfront_source @@ -306,12 +311,17 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -320,13 +330,13 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(32 rows) CREATE TEMP TABLE cf_tmp (ts timestamptz, status text); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM cf_tmp; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 CTE coldfront_source @@ -340,12 +350,17 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -354,13 +369,13 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(32 rows) EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT ts, status FROM public.cf_staging) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM s; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 CTE coldfront_source @@ -374,12 +389,17 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -388,13 +408,13 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(32 rows) CREATE VIEW public.cf_now_view AS SELECT now() AS ts, status FROM public.cf_staging; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_now_view; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 CTE coldfront_source @@ -408,12 +428,17 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -422,14 +447,14 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(32 rows) BEGIN; INSERT INTO public.cf_staging VALUES ('2026-01-30', 'cold5', NULL, NULL, NULL); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 CTE coldfront_source @@ -443,12 +468,17 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -457,7 +487,7 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(32 rows) INSERT INTO public.cf_stream (ts, status) SELECT ts, 'txn' FROM public.cf_staging WHERE status = 'cold5'; hot_rows | cold_rows @@ -469,8 +499,8 @@ COMMIT; BEGIN ISOLATION LEVEL REPEATABLE READ; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 CTE coldfront_source @@ -484,12 +514,17 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -498,14 +533,14 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(32 rows) COMMIT; SET coldfront.local_pg_dsn = ''; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 CTE coldfront_source @@ -519,12 +554,17 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, to_jsonb(r.*)) - -> Subquery Scan on r - Output: r.* - -> CTE Scan on coldfront_source coldfront_source_1 - Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] - Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + -> HashAggregate + Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Group Key: r.coldfront_batch + -> Subquery Scan on r + Output: r.coldfront_batch, r.* + -> WindowAgg + Output: (nextval('_cf_stream_id_seq'::regclass)), coldfront_source_1.ts, coldfront_source_1.status, (7), (NULL::jsonb), (NULL::bytea), (NULL::integer[]), ((row_number() OVER (?) - 1) / (current_setting('coldfront.cold_write_batch_size'::text))::integer) + -> CTE Scan on coldfront_source coldfront_source_1 + Output: nextval('_cf_stream_id_seq'::regclass), coldfront_source_1.ts, coldfront_source_1.status, 7, NULL::jsonb, NULL::bytea, NULL::integer[] + Filter: (coldfront_source_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) InitPlan 4 (returns $4) -> Aggregate Output: count(*) @@ -533,7 +573,7 @@ EXPLAIN (COSTS OFF, VERBOSE) InitPlan 5 (returns $5) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(27 rows) +(32 rows) SELECT status FROM public.cf_stream_cold WHERE status = 'txn'; status diff --git a/extension/coldfront/test/sql/cold_write_batch_size_guc.sql b/extension/coldfront/test/sql/cold_write_batch_size_guc.sql index 68c53e4..15e5d40 100644 --- a/extension/coldfront/test/sql/cold_write_batch_size_guc.sql +++ b/extension/coldfront/test/sql/cold_write_batch_size_guc.sql @@ -1,8 +1,8 @@ -- coldfront.cold_write_batch_size is a GUC (DefineCustomIntVariable; default --- 10000, min 1) that the cold sink (coldfront._cold_sink_step) reads to size each --- cold-tier Iceberg flush (one duckdb.raw_query per batch). White-box: this --- checks the GUC's definition, default, settability, and lower bound — the flush --- behavior itself is exercised live in ci/journey.sh. +-- 10000, min 1) that the cold sink's statement (coldfront._cold_sink_sql) reads +-- at execution to size each cold-tier Iceberg batch (one INSERT per batch). +-- White-box: this checks the GUC's definition, default, settability, and lower +-- bound; the batching itself is exercised live in ci/journey.sh. CREATE EXTENSION IF NOT EXISTS pg_duckdb; CREATE EXTENSION IF NOT EXISTS coldfront; From 23e935033d710f3270954e65b0dca904489d7915 Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 9 Oct 2026 12:20:25 +0100 Subject: [PATCH 4/9] docs: state the design tenets in CLAUDE.md --- CLAUDE.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 6ddd614..b59dfcc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,3 +1,13 @@ +# HARD RULES: Design Tenets + +Every change keeps all five. A change that cannot is raised before it is made, never worked around. + +- Transparent to the user: a ColdFront table is used exactly as a native PostgreSQL table. For every supported feature the SQL, the result, the errors and the transaction semantics are what PostgreSQL gives; no new syntax, no extra step, no ColdFront function in the user's statement. A feature outside that set is refused with a clear error, never approximated. +- Async where possible: a write does its work inside the statement and takes the serializer (the bakery claim, or the node's advisory lock) at commit, for the catalog commit alone. Claim-first is the exception, for a write whose position deletes name data files a concurrent commit could rewrite. +- Streaming where possible: rows flow from their source to the cold tier in one pass through the DuckDB pipeline, never materialized or rendered row by row. A row-at-a-time path is the fallback for the cases where streaming is proven unsafe, and the fallback must be linear in the row count. +- One isolation level across both tiers: the transaction isolation PostgreSQL gives a session applies to its cold-tier reads and writes as to its hot-tier ones. The two halves of a statement see one snapshot, and nothing on the cold side sees what the hot side of the same transaction cannot. +- Interoperable Iceberg: a ColdFront table is a standard Apache Iceberg table that any other Iceberg tool can read, write and compact, and whose writes by those tools ColdFront reads. Nothing ColdFront writes needs a private layout, property or metadata field, and a commit by another tool is a normal event, never a corruption. + # Development Preferences ## Process From 24f2a78b523e2f71c64feab2164ace9b32c4527d Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 9 Oct 2026 14:13:41 +0100 Subject: [PATCH 5/9] fix: render a cold write's text values in one canonical form --- docs/architecture_tiered.md | 5 +- docs/changelog.md | 8 + docs/usage.md | 9 +- extension/coldfront/Makefile | 2 +- extension/coldfront/coldfront--1.0.sql | 48 ++++- extension/coldfront/src/coldfront.c | 35 ++-- .../test/expected/array_cold_render.out | 8 +- .../test/expected/array_cold_render_1.out | 8 +- .../test/expected/cold_render_canonical.out | 181 ++++++++++++++++++ .../test/expected/cold_write_json_agg.out | 8 +- .../test/expected/cold_write_json_agg_1.out | 8 +- .../coldfront/test/expected/cte_on_insert.out | 8 +- .../test/expected/cte_on_insert_1.out | 8 +- .../test/expected/param_cold_via_plpgsql.out | 36 ++-- .../expected/param_cold_via_plpgsql_1.out | 36 ++-- .../expected/tiered_insert_single_pass.out | 8 +- .../expected/tiered_insert_single_pass_1.out | 8 +- .../test/expected/tiered_insert_stream.out | 64 +++---- .../test/expected/tiered_insert_stream_1.out | 64 +++---- .../test/sql/cold_render_canonical.sql | 80 ++++++++ 20 files changed, 472 insertions(+), 160 deletions(-) create mode 100644 extension/coldfront/test/expected/cold_render_canonical.out create mode 100644 extension/coldfront/test/sql/cold_render_canonical.sql diff --git a/docs/architecture_tiered.md b/docs/architecture_tiered.md index 3f0dd9f..8368618 100644 --- a/docs/architecture_tiered.md +++ b/docs/architecture_tiered.md @@ -337,8 +337,9 @@ the fallback below. The query is deparsed with an empty `search_path`, so every relation, function and operator is schema-qualified whatever the second session's path is, and -with ISO dates and PostgreSQL-style intervals, so its literals read the same -under that session's `DateStyle` and `IntervalStyle`. `pgstream` is attached at +its literals and parameters are rendered by `coldfront._canonical_text`, whose +ISO dates, full-precision floats and fixed interval shape read the same under +that session's `DateStyle`, `IntervalStyle` and `extra_float_digits`. `pgstream` is attached at READ COMMITTED, so each statement's read takes a snapshot of its own and sees what the hot half saw; the `pglocal` attachment the other cold paths read through stays at REPEATABLE READ, which the extension needs to give a parallel diff --git a/docs/changelog.md b/docs/changelog.md index e4932de..7fb2cdc 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -152,6 +152,14 @@ this project adheres to ### Fixed +- A value a cold write renders as text, a statement's parameter, a streamed + column or a stored `interval`, took the caller's `DateStyle`, `IntervalStyle` + and `extra_float_digits`, so a day-first date landed as another date, a + mixed-sign interval as another interval and a double with fewer digits. Every + such value is rendered in one canonical form (ISO dates and timestamps, + full-precision floats, an interval as + ` months days seconds`) that PostgreSQL and DuckDB + read back as the same value whatever the session's settings. - A tiered table with a column whose name DuckDB parses as a keyword, such as `by`, `at` or `show`, registered cleanly and then failed every archive pass, because pg_duckdb passes the name to DuckDB unquoted diff --git a/docs/usage.md b/docs/usage.md index 8f725fe..d5d6a68 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -956,7 +956,8 @@ pg_duckdb has no fixed-length `char` type. Use `text` or `varchar` if blank-padded display matters. `json`, `jsonb` and `interval` are stored as `varchar` in Iceberg (no native -primitive). On read, `interval` is view-cast back to the rich PG type; `json` +primitive), an `interval` as ` months days seconds`, +the text PostgreSQL and DuckDB both read as the same value. On read, `interval` is view-cast back to the rich PG type; `json` and `jsonb` come back as DuckDB's `json` (the equivalent of PG's `jsonb`), not the rich PG `jsonb` type, because Iceberg-backed reads run entirely in DuckDB. Queries like `data->>'key'` and `data->'key'` work, and ColdFront translates @@ -1127,9 +1128,9 @@ Keep the following caveats in mind when running either mode: row in plpgsql and writes Iceberg in batches of `coldfront.cold_write_batch_size` rows, as does a server without `coldfront.local_pg_dsn`. The statement's `cold_rows` column is `NULL` when - the cold rows streamed. A `DateStyle` whose output is ambiguous to read - back, `SQL` with `DMY`, is not one the second session may differ in: the - stream renders its dates in ISO form. + the cold rows streamed. A value that reaches the second session or DuckDB + as text is rendered in a form every reader takes as the same value, whatever + the caller's `DateStyle`, `IntervalStyle` or `extra_float_digits`. - `COPY FROM` reads the rows with PostgreSQL's `COPY` reader and writes them through that same `INSERT` path, `coldfront.cold_write_batch_size` rows per `INSERT`. The format options are the reader's, and a supplied value for a diff --git a/extension/coldfront/Makefile b/extension/coldfront/Makefile index 6b7c574..9fae8ec 100644 --- a/extension/coldfront/Makefile +++ b/extension/coldfront/Makefile @@ -30,7 +30,7 @@ REGRESS = load_order update_unregistered_view update_heap_table \ cold_write_json_agg duckdb_temp_dir \ vector_centroids vector_type_map vector_cold_render array_cold_render vector_param_render vector_ops vector_probe vector_status vector_multicolumn \ duckdb_keyword_column partition_clause failed_txn_block tiered_insert_single_pass tiered_view_no_trigger copy_tiered_view param_in_subquery merge_on_view delete_without_where update_from_without_alias settings_registered ensure_replicated read_inner_product \ - async_write_before_claim tiered_insert_stream + async_write_before_claim tiered_insert_stream cold_render_canonical REGRESS_OPTS = --inputdir=test --outputdir=test PG_CONFIG ?= pg_config diff --git a/extension/coldfront/coldfront--1.0.sql b/extension/coldfront/coldfront--1.0.sql index 9445db6..b21cbeb 100644 --- a/extension/coldfront/coldfront--1.0.sql +++ b/extension/coldfront/coldfront--1.0.sql @@ -530,6 +530,8 @@ BEGIN -- the tiered INSERT's cold sink, its statement and its row renderer '_cold_sink', '_cold_sink_step', '_cold_sink_final', '_cold_sink_sql', '_cold_row_literal', + -- the canonical text of a value another session or DuckDB reads + '_canonical_text', '_canonical_jsonb', -- an array column's shape check in that INSERT, and an -- array parameter's renderer in a cold write '_cold_list', '_render_cold_param', @@ -1011,7 +1013,7 @@ CREATE FUNCTION coldfront._cold_sink_sql( LANGUAGE sql IMMUTABLE AS $$ SELECT format( 'SELECT coldfront._cold_sink(%1$L, %2$L, b.tuples, b.n) AS n ' - 'FROM (SELECT string_agg(''('' || coldfront._cold_row_literal(to_jsonb(r), %3$L::text[], %4$L::text[], %1$L, %2$L) || '')'', '', '') AS tuples, ' + 'FROM (SELECT string_agg(''('' || coldfront._cold_row_literal(coldfront._canonical_jsonb(r), %3$L::text[], %4$L::text[], %1$L, %2$L) || '')'', '', '') AS tuples, ' 'count(*) AS n ' 'FROM (SELECT r0.*, (row_number() OVER () - 1) / current_setting(''coldfront.cold_write_batch_size'')::int AS coldfront_batch ' 'FROM (%5$s) AS r0) AS r ' @@ -1328,7 +1330,7 @@ BEGIN LOOP FETCH cur INTO rec; EXIT WHEN NOT FOUND; - payload := to_jsonb(rec); + payload := coldfront._canonical_jsonb(rec); -- OLD primary key, each part cast to its Iceberg storage type so DuckDB's -- row IN matches the typed columns (a bare string literal would not coerce). SELECT string_agg(quote_literal(payload->>nm) || '::' || typ, ', ' ORDER BY ord) @@ -1355,7 +1357,7 @@ BEGIN 'DELETE FROM %1$I.%2$I h WHERE (%5$s) AND %3$s >= %4$s AND (%6$s) < %4$s RETURNING h.*, (%6$s) AS cf_new_ts', v_hot_schema, v_hot_relname, v_pc, v_cut_lit, p_where, p_newpc) LOOP - payload := to_jsonb(rec); + payload := coldfront._canonical_jsonb(rec); ins_arr := ins_arr || ('(' || coldfront._move_row_literal(payload, full_cols, full_types, v_partcol, p_view_schema, p_view_name) || ')'); END LOOP; @@ -2621,6 +2623,42 @@ BEGIN END; $$; +-- ============================================================================ +-- Canonical text. A value that leaves the caller's session as text (a cold +-- write's rendered parameter, a streamed row's text column, a cold VARCHAR +-- holding an interval) is read back by another session, or by DuckDB, under +-- settings of their own, so it is rendered in a form every reader takes as +-- the same value: ISO dates and timestamps, floats at full precision, and an +-- interval as " months days seconds" with a +-- non-negative field first. That interval shape is the one both parsers +-- accept: DuckDB 1.5.4 rejects an ISO 8601 duration and any '+' sign, and +-- PostgreSQL's sql_standard style spreads a leading '-' over the unsigned +-- fields after it. Display conversion stays with the reading session. +-- ============================================================================ +CREATE FUNCTION coldfront._canonical_text(p anyelement) RETURNS text +LANGUAGE sql STABLE STRICT +SET DateStyle = 'ISO, YMD' SET extra_float_digits = 3 AS $$ + SELECT p::text +$$; + +CREATE FUNCTION coldfront._canonical_text(p interval) RETURNS text +LANGUAGE sql IMMUTABLE STRICT AS $$ + SELECT string_agg(f.v || ' ' || f.u, ' ' ORDER BY f.v < 0, f.o) + FROM (VALUES (1, EXTRACT(YEAR FROM p) * 12 + EXTRACT(MONTH FROM p), 'months'), + (2, EXTRACT(DAY FROM p), 'days'), + (3, trim_scale(EXTRACT(HOUR FROM p) * 3600 + EXTRACT(MINUTE FROM p) * 60 + + EXTRACT(SECOND FROM p)), 'seconds')) AS f(o, v, u) +$$; + +-- A row for the cold sink: to_jsonb under fixed settings, the interval in ISO +-- 8601, which _render_cold_value reads back in the caller's session and +-- renders in the shape above. +CREATE FUNCTION coldfront._canonical_jsonb(p anyelement) RETURNS jsonb +LANGUAGE sql STABLE STRICT +SET DateStyle = 'ISO, YMD' SET IntervalStyle = iso_8601 SET extra_float_digits = 3 AS $$ + SELECT to_jsonb(p) +$$; + -- Cold-value rendering, shared so the write paths cannot disagree. DuckDB's list -- cast accepts [1,2,3] and rejects PG's {1,2,3}, and whitespace between elements -- is fine. A value taken from a jsonb payload is already bracketed: jsonb spells @@ -2647,6 +2685,7 @@ LANGUAGE sql IMMUTABLE STRICT AS $$ -- own input reads either. DuckDB mis-parses PG's \x escape into a BLOB, -- so the bytes are rebuilt from their hex digits. WHEN p_pg_type = 'bytea' THEN format('from_hex(%L)', encode(p_val_text::bytea, 'hex')) + WHEN p_pg_type = 'interval' THEN quote_literal(coldfront._canonical_text(p_val_text::interval)) -- The Iceberg column is FLOAT[]; without the cast the literal stays a -- VARCHAR and the INSERT fails. WHEN coldfront._is_vector_type(p_pg_type) THEN format('CAST(%L AS FLOAT[])', p_val_text) @@ -2683,7 +2722,8 @@ $$; -- makes of an array column. The C hook's cold_sql_arg calls this at execution. CREATE FUNCTION coldfront._render_cold_param(p anyarray) RETURNS text -LANGUAGE sql STABLE AS $$ +LANGUAGE sql STABLE +SET DateStyle = 'ISO, YMD' SET extra_float_digits = 3 AS $$ SELECT coalesce(coldfront._render_cold_value(to_jsonb(coldfront._cold_list(p, NULL))::text, pg_typeof(p)::text), 'NULL'); diff --git a/extension/coldfront/src/coldfront.c b/extension/coldfront/src/coldfront.c index 16c8efc..569272f 100644 --- a/extension/coldfront/src/coldfront.c +++ b/extension/coldfront/src/coldfront.c @@ -1514,10 +1514,12 @@ build_cold_dml(DeparseResult *dr, TieredViewInfo *info, Query *query) * renders as a DuckDB literal: bytea -> from_hex(%P$L) / encode($K,'hex'); * real[] (a vector column's view type) -> CAST(%P$L AS FLOAT[]) / * translate($K::text,'{}','[]'); any other array -> %P$s / - * coldfront._render_cold_param($K); json/jsonb/interval -> %P$L / $K::text; - * else %P$L / $K. With for_pg the string is the tiered stream's read, which - * PostgreSQL runs: every value is %P$L / $K, a literal its own input reads - * back, coerced by the projection's casts. + * coldfront._render_cold_param($K); else %P$L / + * coldfront._canonical_text($K), a text every reader takes as the same value + * whatever the caller's DateStyle, IntervalStyle and extra_float_digits. With + * for_pg the string is the tiered stream's read, which PostgreSQL runs in + * another session: every value is that canonical text, coerced by the + * projection's casts. */ static char * cold_sql_arg(const char *cold_dml, ColdParamSet *ps, bool for_pg) @@ -1568,27 +1570,23 @@ cold_sql_arg(const char *cold_dml, ColdParamSet *ps, bool for_pg) if (pos_of_id[id - 1] == 0) /* first sight: assign pos + arg */ { pos_of_id[id - 1] = next_pos++; - if (for_pg) - appendStringInfo(&args, ", $%d", id); - else if (t == BYTEAOID) + if (!for_pg && t == BYTEAOID) /* hex string, independent of the caller's bytea_output; * from_hex rebuilds the exact bytes. */ appendStringInfo(&args, ", encode($%d,'hex')", id); - else if (t == FLOAT4ARRAYOID) + else if (!for_pg && t == FLOAT4ARRAYOID) /* A vector column reaches this path as real[], the type the * view exposes. PG spells that {1,2,3} and DuckDB's list * cast takes [1,2,3], so translate rewrites the delimiters * and the template supplies the cast. */ appendStringInfo(&args, ", translate($%d::text,'{}','[]')", id); - else if (type_is_array(t)) + else if (!for_pg && type_is_array(t)) /* Any other array: coldfront._render_cold_param checks its * shape and renders the DuckDB list an array column's value * becomes, so the template takes the result as SQL. */ appendStringInfo(&args, ", coldfront._render_cold_param($%d)", id); - else if (t == JSONOID || t == JSONBOID || t == INTERVALOID) - appendStringInfo(&args, ", $%d::text", id); else - appendStringInfo(&args, ", $%d", id); + appendStringInfo(&args, ", coldfront._canonical_text($%d)", id); } if (for_pg) appendStringInfo(&tmpl, "%%%d$L", pos_of_id[id - 1]); @@ -2334,16 +2332,17 @@ build_src_cte(Query *query, const char *source, const char *col_list, /* * The PostgreSQL type a streamed column is sent as, one DuckDB's postgres * extension maps to the Iceberg column's type. A vector is unknown to it and - * would arrive as text, so it goes as real[]; jsonb, json and interval are - * stored as VARCHAR and go as their text, the form the sink stores too. + * would arrive as text, so it goes as real[]; jsonb and json are stored as + * VARCHAR and go as their text. An interval is stored as VARCHAR too and goes + * as coldfront._canonical_text's shape (build_cold_projection), the one text + * both parsers read back as the same value. */ static const char * cf_stream_cast(const char *type) { if (strncmp(type, "vector", 6) == 0 || strncmp(type, "halfvec", 7) == 0) return "::real[]"; - if (strcmp(type, "jsonb") == 0 || strcmp(type, "json") == 0 || - strcmp(type, "interval") == 0) + if (strcmp(type, "jsonb") == 0 || strcmp(type, "json") == 0) return "::text"; return ""; } @@ -2399,7 +2398,9 @@ build_cold_projection(Query *query, const HotColumns *hc, bool stream) if (pg_str_endswith(hc->type[i], "[]")) expr = psprintf("coldfront._cold_list(%s, %s)", expr, quote_literal_cstr(hc->name[i])); - if (cast[0] != '\0') + if (stream && strcmp(hc->type[i], "interval") == 0) + expr = psprintf("coldfront._canonical_text((%s)::interval)", expr); + else if (cast[0] != '\0') expr = psprintf("(%s)%s", expr, cast); appendStringInfo(&sel, "%s%s AS %s", i > 0 ? ", " : "", expr, quote_identifier(hc->name[i])); diff --git a/extension/coldfront/test/expected/array_cold_render.out b/extension/coldfront/test/expected/array_cold_render.out index 6119cd6..d37907e 100644 --- a/extension/coldfront/test/expected/array_cold_render.out +++ b/extension/coldfront/test/expected/array_cold_render.out @@ -79,7 +79,7 @@ SELECT coldfront._render_cold_param(ARRAY['a', NULL]::text[]) AS text_param, SELECT coldfront._render_cold_param('[0:1]={1,2}'::int[]); ERROR: coldfront: a bound parameter holds an array of 1 dimension(s) starting at element 0; the cold tier stores one-dimensional arrays starting at element 1 CONTEXT: PL/pgSQL function _cold_list(anyarray,text) line 4 at RAISE -SQL function "_render_cold_param" during inlining +SQL function "_render_cold_param" statement 1 -- The archiver stages a partition with CREATE TABLE ... USING duckdb AS SELECT, -- and pg_duckdb (c04e6a2) loses a numeric array's precision there and refuses -- the column, so the archiver exports a numeric array as text[] @@ -142,10 +142,10 @@ SET plan_cache_mode = force_generic_plan; PREPARE cold_tags(text[]) AS UPDATE public.tags SET tags = $1 WHERE ts < '2026-03-01'; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_tags('{x,y}'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result - Output: _exec_iceberg_with_claim('ice.default.tags'::text, format('UPDATE ice.default.tags SET tags = %1$s WHERE (ts < ''2026-03-01 00:00:00+00''::timestamptz)'::text, COALESCE(_render_cold_value((to_jsonb(_cold_list($1, NULL::text)))::text, (pg_typeof($1))::text), 'NULL'::text))) + Output: _exec_iceberg_with_claim('ice.default.tags'::text, format('UPDATE ice.default.tags SET tags = %1$s WHERE (ts < ''2026-03-01 00:00:00+00''::timestamptz)'::text, _render_cold_param($1))) (2 rows) DEALLOCATE cold_tags; diff --git a/extension/coldfront/test/expected/array_cold_render_1.out b/extension/coldfront/test/expected/array_cold_render_1.out index 760b824..5a63481 100644 --- a/extension/coldfront/test/expected/array_cold_render_1.out +++ b/extension/coldfront/test/expected/array_cold_render_1.out @@ -79,7 +79,7 @@ SELECT coldfront._render_cold_param(ARRAY['a', NULL]::text[]) AS text_param, SELECT coldfront._render_cold_param('[0:1]={1,2}'::int[]); ERROR: coldfront: a bound parameter holds an array of 1 dimension(s) starting at element 0; the cold tier stores one-dimensional arrays starting at element 1 CONTEXT: PL/pgSQL function _cold_list(anyarray,text) line 4 at RAISE -SQL function "_render_cold_param" during inlining +SQL function "_render_cold_param" statement 1 -- The archiver stages a partition with CREATE TABLE ... USING duckdb AS SELECT, -- and pg_duckdb (c04e6a2) loses a numeric array's precision there and refuses -- the column, so the archiver exports a numeric array as text[] @@ -142,10 +142,10 @@ SET plan_cache_mode = force_generic_plan; PREPARE cold_tags(text[]) AS UPDATE public.tags SET tags = $1 WHERE ts < '2026-03-01'; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_tags('{x,y}'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result - Output: _exec_iceberg_with_claim('ice.default.tags'::text, format('UPDATE ice.default.tags SET tags = %1$s WHERE (ts < ''2026-03-01 00:00:00+00''::timestamptz)'::text, COALESCE(_render_cold_value((to_jsonb(_cold_list($1, NULL::text)))::text, (pg_typeof($1))::text), 'NULL'::text))) + Output: _exec_iceberg_with_claim('ice.default.tags'::text, format('UPDATE ice.default.tags SET tags = %1$s WHERE (ts < ''2026-03-01 00:00:00+00''::timestamptz)'::text, _render_cold_param($1))) (2 rows) DEALLOCATE cold_tags; diff --git a/extension/coldfront/test/expected/cold_render_canonical.out b/extension/coldfront/test/expected/cold_render_canonical.out new file mode 100644 index 0000000..3f75a08 --- /dev/null +++ b/extension/coldfront/test/expected/cold_render_canonical.out @@ -0,0 +1,181 @@ +-- A value that leaves the caller's session as text reads back as the same +-- value whatever that session's DateStyle, IntervalStyle and extra_float_digits +-- are, and a cold interval is stored as ISO 8601 text. A tiered INSERT carries +-- a timestamp, an interval and a double to the cold tier through the stream +-- (its parameters, then its columns), through the sink, and a cold UPDATE +-- carries them as its parameters. Observed end to end: a DuckDB temporary +-- table stands in for the Iceberg table and is read back with duckdb.query. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; + dsn_set +--------- + t +(1 row) + +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_canon (id BIGINT, ts TIMESTAMPTZ, iv VARCHAR, x DOUBLE)') IS NOT NULL AS created; +NOTICE: result: Count +BIGINT +[ Rows: 0] + + + created +--------- + t +(1 row) + +CREATE TABLE public._cf_canon (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz NOT NULL, + iv interval, x double precision); +CREATE VIEW public.cf_canon AS SELECT * FROM public._cf_canon; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'cf_canon', 'public._cf_canon', 'temp.main.cf_canon', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'cf_canon', '2026-03-15'::timestamptz); +CREATE TABLE public.cf_canon_src (tag text, ts timestamptz, iv interval, x double precision); +INSERT INTO public.cf_canon_src VALUES + ('cold', '2026-03-01 08:00:00+00', '-1 days -02:03:04', 0.1::float8 + 0.2::float8), + ('hot', '2026-04-01 08:00:00+00', '1 year 2 mons -3 days 04:05:06.5', 1e-7); +CREATE VIEW public.cf_canon_cold AS + SELECT r['ts']::timestamptz AS ts, r['iv']::text AS iv, r['x']::float8 AS x + FROM duckdb.query('SELECT * FROM temp.main.cf_canon') AS t(r); +-- The caller's settings: day-first dates, SQL-standard intervals, and floats +-- printed with fewer digits than they hold. +SET DateStyle = 'SQL, DMY'; +SET IntervalStyle = sql_standard; +SET extra_float_digits = -3; +-- (1) The stream's parameters. +PREPARE cf_canon_ins(timestamptz, interval, double precision) AS + INSERT INTO public.cf_canon (ts, iv, x) VALUES ($1, $2, $3); +EXECUTE cf_canon_ins('2026-03-01 08:00:00+00', '-1 days -02:03:04', 0.1::float8 + 0.2::float8); +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 0 | +(1 row) + +-- (2) The stream's columns. +INSERT INTO public.cf_canon (ts, iv, x) SELECT ts, iv, x FROM public.cf_canon_src WHERE tag = 'cold'; +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 0 | +(1 row) + +-- (3) The sink, for both. +SET coldfront.local_pg_dsn = ''; +EXECUTE cf_canon_ins('2026-03-01 08:00:00+00', '-1 days -02:03:04', 0.1::float8 + 0.2::float8); +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 0 | +(1 row) + +INSERT INTO public.cf_canon (ts, iv, x) SELECT ts, iv, x FROM public.cf_canon_src; +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 1 | 1 +(1 row) + +-- Every cold row holds the values the caller gave, the interval in the +-- canonical shape. The rows are copied to a heap table first, so the checks +-- run in PostgreSQL: pg_duckdb renders a query's interval literal in the +-- session's IntervalStyle, which DuckDB does not read. +CREATE TEMP TABLE cf_canon_got AS SELECT * FROM public.cf_canon_cold; +SELECT ts = '2026-03-01 08:00:00+00'::timestamptz AS ts_kept, iv, + iv::interval = interval '-1 days -02:03:04' AS iv_kept, + x = 0.1::float8 + 0.2::float8 AS x_kept + FROM cf_canon_got ORDER BY 1, 2, 3, 4; + ts_kept | iv | iv_kept | x_kept +---------+--------------------------------+---------+-------- + t | 0 months -1 days -7384 seconds | t | t + t | 0 months -1 days -7384 seconds | t | t + t | 0 months -1 days -7384 seconds | t | t + t | 0 months -1 days -7384 seconds | t | t +(4 rows) + +SELECT count(*) AS hot_rows FROM public._cf_canon; + hot_rows +---------- + 1 +(1 row) + +-- (4) A cold UPDATE's parameters: the interval it sets and the timestamp it +-- matches on. +PREPARE cf_canon_upd(interval, timestamptz) AS UPDATE public.cf_canon SET iv = $1 WHERE ts = $2; +EXECUTE cf_canon_upd('1 year 2 mons -3 days 04:05:06.5', '2026-03-01 08:00:00+00'); +NOTICE: result: Count +BIGINT +[ Rows: 1] +4 + + + id | ts | iv | x +----+----+----+--- +(0 rows) + +SELECT iv, count(*) FROM public.cf_canon_cold GROUP BY iv ORDER BY iv; + iv | count +-----------------------------------+------- + 14 months 14706.5 seconds -3 days | 4 +(1 row) + +RESET DateStyle; +RESET IntervalStyle; +RESET extra_float_digits; +-- The interval shape is the one DuckDB 1.5.4 reads. When its INTERVAL cast +-- takes an ISO 8601 duration, this stops failing and the shape can become +-- ISO 8601. +SELECT duckdb.raw_query('SELECT ''PT2H''::INTERVAL'); +ERROR: (PGDuckDB/pgduckdb_raw_query_cpp) Conversion Error: Could not convert string 'PT2H' to INTERVAL + +LINE 1: SELECT 'PT2H'::INTERVAL + ^ diff --git a/extension/coldfront/test/expected/cold_write_json_agg.out b/extension/coldfront/test/expected/cold_write_json_agg.out index 1521ce7..3a37755 100644 --- a/extension/coldfront/test/expected/cold_write_json_agg.out +++ b/extension/coldfront/test/expected/cold_write_json_agg.out @@ -61,8 +61,8 @@ WHERE ts < '2019-01-01'::timestamptz; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events SELECT 9, '2019-01-01'::timestamptz, jsonb_agg(k ORDER BY k) FROM public.src; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -81,9 +81,9 @@ SELECT 9, '2019-01-01'::timestamptz, jsonb_agg(k ORDER BY k) FROM public.src; Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* diff --git a/extension/coldfront/test/expected/cold_write_json_agg_1.out b/extension/coldfront/test/expected/cold_write_json_agg_1.out index 063d903..4831322 100644 --- a/extension/coldfront/test/expected/cold_write_json_agg_1.out +++ b/extension/coldfront/test/expected/cold_write_json_agg_1.out @@ -61,8 +61,8 @@ WHERE ts < '2019-01-01'::timestamptz; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events SELECT 9, '2019-01-01'::timestamptz, jsonb_agg(k ORDER BY k) FROM public.src; - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result Output: $4, $5 CTE coldfront_source @@ -81,9 +81,9 @@ SELECT 9, '2019-01-01'::timestamptz, jsonb_agg(k ORDER BY k) FROM public.src; Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,data}'::text[], '{integer,"timestamp with time zone",jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* diff --git a/extension/coldfront/test/expected/cte_on_insert.out b/extension/coldfront/test/expected/cte_on_insert.out index 67ea03d..cce4df7 100644 --- a/extension/coldfront/test/expected/cte_on_insert.out +++ b/extension/coldfront/test/expected/cte_on_insert.out @@ -28,8 +28,8 @@ CREATE TABLE public.staging (id int, ts timestamptz, status text); EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT 7 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, 'new' AS status) INSERT INTO public.events SELECT id, ts, status FROM s; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -43,9 +43,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* diff --git a/extension/coldfront/test/expected/cte_on_insert_1.out b/extension/coldfront/test/expected/cte_on_insert_1.out index e8f1430..1050f62 100644 --- a/extension/coldfront/test/expected/cte_on_insert_1.out +++ b/extension/coldfront/test/expected/cte_on_insert_1.out @@ -28,8 +28,8 @@ CREATE TABLE public.staging (id int, ts timestamptz, status text); EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT 7 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, 'new' AS status) INSERT INTO public.events SELECT id, ts, status FROM s; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -43,9 +43,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status}'::text[], '{integer,"timestamp with time zone",text}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* diff --git a/extension/coldfront/test/expected/param_cold_via_plpgsql.out b/extension/coldfront/test/expected/param_cold_via_plpgsql.out index bb4654c..f0cb1ee 100644 --- a/extension/coldfront/test/expected/param_cold_via_plpgsql.out +++ b/extension/coldfront/test/expected/param_cold_via_plpgsql.out @@ -36,20 +36,20 @@ VALUES ('public', 'events', '2026-03-01'::timestamptz); PREPARE cold_upd(text) AS UPDATE public.events SET status = $1 WHERE ts < '2026-03-01'; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_upd('x'); - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE (ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz)'::text, $1)) + Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE (ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz)'::text, _canonical_text($1))) (2 rows) -- (2) pure-cold DELETE, param in WHERE alongside the literal cold predicate. PREPARE cold_del(text) AS DELETE FROM public.events WHERE ts < '2026-03-01' AND status = $1; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_del('x'); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result - Output: _exec_iceberg_with_claim('ice.default.events'::text, format('DELETE FROM ice.default.events WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %1$L))'::text, $1)) + Output: _exec_iceberg_with_claim('ice.default.events'::text, format('DELETE FROM ice.default.events WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %1$L))'::text, _canonical_text($1))) (2 rows) -- (3) reordered params: $2 appears first (SET), $1 second (WHERE) -> positional @@ -57,10 +57,10 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_del('x'); PREPARE cold_rep(text, text) AS UPDATE public.events SET status = $2 WHERE ts < '2026-03-01' AND status = $1; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_rep('a', 'b'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %2$L))'::text, $2, $1)) + Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %2$L))'::text, _canonical_text($2), _canonical_text($1))) (2 rows) -- (4) bytea param -> from_hex(%N$L) placeholder with an encode($K,'hex') arg. @@ -77,10 +77,10 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_bytea('\xcafe'::bytea); PREPARE cold_like(text) AS UPDATE public.events SET status = 'pct%done' WHERE ts < '2026-03-01' AND status = $1; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_like('x'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = ''pct%%done''::text WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %1$L))'::text, $1)) + Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = ''pct%%done''::text WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %1$L))'::text, _canonical_text($1))) (2 rows) -- (6) dual-tier (ambiguous predicate on a non-partition column) with a param: @@ -88,8 +88,8 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_like('x'); PREPARE dual_upd(text) AS UPDATE public.events SET status = $1 WHERE id = 1; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE dual_upd('x'); - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Nested Loop Output: h.id, h.ts, h.status, h.data CTE coldfront_hot @@ -100,7 +100,7 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE dual_upd('x'); Filter: (_events.id = 1) CTE coldfront_cold -> Result - Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE (id = 1)'::text, $1)) + Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE (id = 1)'::text, _canonical_text($1))) -> CTE Scan on coldfront_cold c Output: c._exec_iceberg_with_claim -> CTE Scan on coldfront_hot h @@ -130,8 +130,8 @@ INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) VALUES ('public', 'eid', '2026-03-01'::timestamptz); PREPARE eid_ins(timestamptz, bytea) AS INSERT INTO public.eid (ts, data) VALUES ($1, $2); EXPLAIN (COSTS OFF, VERBOSE) EXECUTE eid_ins('2026-01-10 00:00+00', '\xcafe'::bytea); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_hot @@ -142,7 +142,7 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE eid_ins('2026-01-10 00:00+00', '\xcafe'::by One-Time Filter: ($1 >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'eid'::text, format('SELECT nextval(''public._eid_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.data::bytea AS data FROM (VALUES (%1$L, %2$L)) AS coldfront_src(ts, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, $1, $2), '{}'::regclass[], ((InitPlan 2).col1 < 1)) + Output: _tiered_cold_stream('public'::text, 'eid'::text, format('SELECT nextval(''public._eid_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.data::bytea AS data FROM (VALUES (%1$L, %2$L)) AS coldfront_src(ts, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, _canonical_text($1), _canonical_text($2)), '{}'::regclass[], ((InitPlan 2).col1 < 1)) InitPlan 2 -> Aggregate Output: count(*) diff --git a/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out b/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out index fcff944..e857b90 100644 --- a/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out +++ b/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out @@ -36,20 +36,20 @@ VALUES ('public', 'events', '2026-03-01'::timestamptz); PREPARE cold_upd(text) AS UPDATE public.events SET status = $1 WHERE ts < '2026-03-01'; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_upd('x'); - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE (ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz)'::text, $1)) + Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE (ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz)'::text, _canonical_text($1))) (2 rows) -- (2) pure-cold DELETE, param in WHERE alongside the literal cold predicate. PREPARE cold_del(text) AS DELETE FROM public.events WHERE ts < '2026-03-01' AND status = $1; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_del('x'); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result - Output: _exec_iceberg_with_claim('ice.default.events'::text, format('DELETE FROM ice.default.events WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %1$L))'::text, $1)) + Output: _exec_iceberg_with_claim('ice.default.events'::text, format('DELETE FROM ice.default.events WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %1$L))'::text, _canonical_text($1))) (2 rows) -- (3) reordered params: $2 appears first (SET), $1 second (WHERE) -> positional @@ -57,10 +57,10 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_del('x'); PREPARE cold_rep(text, text) AS UPDATE public.events SET status = $2 WHERE ts < '2026-03-01' AND status = $1; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_rep('a', 'b'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %2$L))'::text, $2, $1)) + Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %2$L))'::text, _canonical_text($2), _canonical_text($1))) (2 rows) -- (4) bytea param -> from_hex(%N$L) placeholder with an encode($K,'hex') arg. @@ -77,10 +77,10 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_bytea('\xcafe'::bytea); PREPARE cold_like(text) AS UPDATE public.events SET status = 'pct%done' WHERE ts < '2026-03-01' AND status = $1; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_like('x'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = ''pct%%done''::text WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %1$L))'::text, $1)) + Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = ''pct%%done''::text WHERE ((ts < ''Sun Mar 01 00:00:00 2026 UTC''::timestamptz) AND (status = %1$L))'::text, _canonical_text($1))) (2 rows) -- (6) dual-tier (ambiguous predicate on a non-partition column) with a param: @@ -88,8 +88,8 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE cold_like('x'); PREPARE dual_upd(text) AS UPDATE public.events SET status = $1 WHERE id = 1; EXPLAIN (COSTS OFF, VERBOSE) EXECUTE dual_upd('x'); - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Nested Loop Output: h.id, h.ts, h.status, h.data CTE coldfront_hot @@ -100,7 +100,7 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE dual_upd('x'); Filter: (_events.id = 1) CTE coldfront_cold -> Result - Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE (id = 1)'::text, $1)) + Output: _exec_iceberg_with_claim('ice.default.events'::text, format('UPDATE ice.default.events SET status = %1$L WHERE (id = 1)'::text, _canonical_text($1))) -> CTE Scan on coldfront_cold c Output: c._exec_iceberg_with_claim -> CTE Scan on coldfront_hot h @@ -130,8 +130,8 @@ INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) VALUES ('public', 'eid', '2026-03-01'::timestamptz); PREPARE eid_ins(timestamptz, bytea) AS INSERT INTO public.eid (ts, data) VALUES ($1, $2); EXPLAIN (COSTS OFF, VERBOSE) EXECUTE eid_ins('2026-01-10 00:00+00', '\xcafe'::bytea); - QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_hot @@ -142,7 +142,7 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE eid_ins('2026-01-10 00:00+00', '\xcafe'::by One-Time Filter: ($1 >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'eid'::text, format('SELECT nextval(''public._eid_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.data::bytea AS data FROM (VALUES (%1$L, %2$L)) AS coldfront_src(ts, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, $1, $2), '{}'::regclass[], ($2 < 1)) + Output: _tiered_cold_stream('public'::text, 'eid'::text, format('SELECT nextval(''public._eid_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.data::bytea AS data FROM (VALUES (%1$L, %2$L)) AS coldfront_src(ts, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, _canonical_text($1), _canonical_text($2)), '{}'::regclass[], ($2 < 1)) InitPlan 2 (returns $2) -> Aggregate Output: count(*) diff --git a/extension/coldfront/test/expected/tiered_insert_single_pass.out b/extension/coldfront/test/expected/tiered_insert_single_pass.out index 459af75..7383468 100644 --- a/extension/coldfront/test/expected/tiered_insert_single_pass.out +++ b/extension/coldfront/test/expected/tiered_insert_single_pass.out @@ -27,8 +27,8 @@ VALUES ('public', 'events', '2026-03-01'::timestamptz); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (ts, data) SELECT '2026-01-01'::timestamptz + i * interval '1 day', '{}' FROM generate_series(1, 3) i; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -43,9 +43,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* diff --git a/extension/coldfront/test/expected/tiered_insert_single_pass_1.out b/extension/coldfront/test/expected/tiered_insert_single_pass_1.out index 6cf930a..47365ca 100644 --- a/extension/coldfront/test/expected/tiered_insert_single_pass_1.out +++ b/extension/coldfront/test/expected/tiered_insert_single_pass_1.out @@ -27,8 +27,8 @@ VALUES ('public', 'events', '2026-03-01'::timestamptz); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (ts, data) SELECT '2026-01-01'::timestamptz + i * interval '1 day', '{}' FROM generate_series(1, 3) i; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -43,9 +43,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'events'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,data}'::text[], '{bigint,"timestamp with time zone",text,jsonb}'::text[], 'public'::text, 'events'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* diff --git a/extension/coldfront/test/expected/tiered_insert_stream.out b/extension/coldfront/test/expected/tiered_insert_stream.out index 258cfaf..e200301 100644 --- a/extension/coldfront/test/expected/tiered_insert_stream.out +++ b/extension/coldfront/test/expected/tiered_insert_stream.out @@ -258,8 +258,8 @@ SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'copy%' GRO -- COMMITTED, and a session without local_pg_dsn. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, random()::text FROM public.cf_staging; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -273,9 +273,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -297,8 +297,8 @@ EXPLAIN (COSTS OFF, VERBOSE) EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) VALUES (now() - interval '1 year', 'stable'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -312,9 +312,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -337,8 +337,8 @@ EXPLAIN (COSTS OFF, VERBOSE) CREATE TEMP TABLE cf_tmp (ts timestamptz, status text); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM cf_tmp; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -352,9 +352,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -377,8 +377,8 @@ EXPLAIN (COSTS OFF, VERBOSE) EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT ts, status FROM public.cf_staging) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM s; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -392,9 +392,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -417,8 +417,8 @@ EXPLAIN (COSTS OFF, VERBOSE) CREATE VIEW public.cf_now_view AS SELECT now() AS ts, status FROM public.cf_staging; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_now_view; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -432,9 +432,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -458,8 +458,8 @@ BEGIN; INSERT INTO public.cf_staging VALUES ('2026-01-30', 'cold5', NULL, NULL, NULL); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -473,9 +473,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -505,8 +505,8 @@ COMMIT; BEGIN ISOLATION LEVEL REPEATABLE READ; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -520,9 +520,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -546,8 +546,8 @@ COMMIT; SET coldfront.local_pg_dsn = ''; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: (InitPlan 4).col1, (InitPlan 5).col1 CTE coldfront_source @@ -561,9 +561,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* diff --git a/extension/coldfront/test/expected/tiered_insert_stream_1.out b/extension/coldfront/test/expected/tiered_insert_stream_1.out index d64334e..2294e86 100644 --- a/extension/coldfront/test/expected/tiered_insert_stream_1.out +++ b/extension/coldfront/test/expected/tiered_insert_stream_1.out @@ -258,8 +258,8 @@ SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'copy%' GRO -- COMMITTED, and a session without local_pg_dsn. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, random()::text FROM public.cf_staging; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -273,9 +273,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -296,8 +296,8 @@ EXPLAIN (COSTS OFF, VERBOSE) EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) VALUES (now() - interval '1 year', 'stable'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -311,9 +311,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -335,8 +335,8 @@ EXPLAIN (COSTS OFF, VERBOSE) CREATE TEMP TABLE cf_tmp (ts timestamptz, status text); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM cf_tmp; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -350,9 +350,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -374,8 +374,8 @@ EXPLAIN (COSTS OFF, VERBOSE) EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT ts, status FROM public.cf_staging) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM s; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -389,9 +389,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -413,8 +413,8 @@ EXPLAIN (COSTS OFF, VERBOSE) CREATE VIEW public.cf_now_view AS SELECT now() AS ts, status FROM public.cf_staging; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_now_view; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -428,9 +428,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -453,8 +453,8 @@ BEGIN; INSERT INTO public.cf_staging VALUES ('2026-01-30', 'cold5', NULL, NULL, NULL); EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -468,9 +468,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -499,8 +499,8 @@ COMMIT; BEGIN ISOLATION LEVEL REPEATABLE READ; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -514,9 +514,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* @@ -539,8 +539,8 @@ COMMIT; SET coldfront.local_pg_dsn = ''; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result Output: $4, $5 CTE coldfront_source @@ -554,9 +554,9 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (coldfront_source.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Aggregate - Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) + Output: _cold_sink('public'::text, 'cf_stream'::text, (string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text)), (count(*))) -> HashAggregate - Output: string_agg((('('::text || _cold_row_literal(to_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch + Output: string_agg((('('::text || _cold_row_literal(_canonical_jsonb(r.*), '{id,ts,status,n,data,b,tags}'::text[], '{bigint,"timestamp with time zone",text,integer,jsonb,bytea,integer[]}'::text[], 'public'::text, 'cf_stream'::text)) || ')'::text), ', '::text), count(*), r.coldfront_batch Group Key: r.coldfront_batch -> Subquery Scan on r Output: r.coldfront_batch, r.* diff --git a/extension/coldfront/test/sql/cold_render_canonical.sql b/extension/coldfront/test/sql/cold_render_canonical.sql new file mode 100644 index 0000000..fcbcee4 --- /dev/null +++ b/extension/coldfront/test/sql/cold_render_canonical.sql @@ -0,0 +1,80 @@ +-- A value that leaves the caller's session as text reads back as the same +-- value whatever that session's DateStyle, IntervalStyle and extra_float_digits +-- are, and a cold interval is stored as ISO 8601 text. A tiered INSERT carries +-- a timestamp, an interval and a double to the cold tier through the stream +-- (its parameters, then its columns), through the sink, and a cold UPDATE +-- carries them as its parameters. Observed end to end: a DuckDB temporary +-- table stands in for the Iceberg table and is read back with duckdb.query. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; + +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_canon (id BIGINT, ts TIMESTAMPTZ, iv VARCHAR, x DOUBLE)') IS NOT NULL AS created; + +CREATE TABLE public._cf_canon (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz NOT NULL, + iv interval, x double precision); +CREATE VIEW public.cf_canon AS SELECT * FROM public._cf_canon; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'cf_canon', 'public._cf_canon', 'temp.main.cf_canon', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'cf_canon', '2026-03-15'::timestamptz); +CREATE TABLE public.cf_canon_src (tag text, ts timestamptz, iv interval, x double precision); +INSERT INTO public.cf_canon_src VALUES + ('cold', '2026-03-01 08:00:00+00', '-1 days -02:03:04', 0.1::float8 + 0.2::float8), + ('hot', '2026-04-01 08:00:00+00', '1 year 2 mons -3 days 04:05:06.5', 1e-7); +CREATE VIEW public.cf_canon_cold AS + SELECT r['ts']::timestamptz AS ts, r['iv']::text AS iv, r['x']::float8 AS x + FROM duckdb.query('SELECT * FROM temp.main.cf_canon') AS t(r); + +-- The caller's settings: day-first dates, SQL-standard intervals, and floats +-- printed with fewer digits than they hold. +SET DateStyle = 'SQL, DMY'; +SET IntervalStyle = sql_standard; +SET extra_float_digits = -3; + +-- (1) The stream's parameters. +PREPARE cf_canon_ins(timestamptz, interval, double precision) AS + INSERT INTO public.cf_canon (ts, iv, x) VALUES ($1, $2, $3); +EXECUTE cf_canon_ins('2026-03-01 08:00:00+00', '-1 days -02:03:04', 0.1::float8 + 0.2::float8); + +-- (2) The stream's columns. +INSERT INTO public.cf_canon (ts, iv, x) SELECT ts, iv, x FROM public.cf_canon_src WHERE tag = 'cold'; + +-- (3) The sink, for both. +SET coldfront.local_pg_dsn = ''; +EXECUTE cf_canon_ins('2026-03-01 08:00:00+00', '-1 days -02:03:04', 0.1::float8 + 0.2::float8); +INSERT INTO public.cf_canon (ts, iv, x) SELECT ts, iv, x FROM public.cf_canon_src; + +-- Every cold row holds the values the caller gave, the interval in the +-- canonical shape. The rows are copied to a heap table first, so the checks +-- run in PostgreSQL: pg_duckdb renders a query's interval literal in the +-- session's IntervalStyle, which DuckDB does not read. +CREATE TEMP TABLE cf_canon_got AS SELECT * FROM public.cf_canon_cold; +SELECT ts = '2026-03-01 08:00:00+00'::timestamptz AS ts_kept, iv, + iv::interval = interval '-1 days -02:03:04' AS iv_kept, + x = 0.1::float8 + 0.2::float8 AS x_kept + FROM cf_canon_got ORDER BY 1, 2, 3, 4; +SELECT count(*) AS hot_rows FROM public._cf_canon; + +-- (4) A cold UPDATE's parameters: the interval it sets and the timestamp it +-- matches on. +PREPARE cf_canon_upd(interval, timestamptz) AS UPDATE public.cf_canon SET iv = $1 WHERE ts = $2; +EXECUTE cf_canon_upd('1 year 2 mons -3 days 04:05:06.5', '2026-03-01 08:00:00+00'); +SELECT iv, count(*) FROM public.cf_canon_cold GROUP BY iv ORDER BY iv; + +RESET DateStyle; +RESET IntervalStyle; +RESET extra_float_digits; + +-- The interval shape is the one DuckDB 1.5.4 reads. When its INTERVAL cast +-- takes an ISO 8601 duration, this stops failing and the shape can become +-- ISO 8601. +SELECT duckdb.raw_query('SELECT ''PT2H''::INTERVAL'); From f1c4fd9c0a749fbb9a6371f5e6cfc7cd534d6ff3 Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 9 Oct 2026 14:26:11 +0100 Subject: [PATCH 6/9] fix: stream a tiered INSERT's cold rows at the statement's snapshot --- docs/architecture_tiered.md | 39 ++- docs/changelog.md | 16 +- docs/usage.md | 26 +- extension/coldfront/Makefile | 2 +- extension/coldfront/coldfront--1.0.sql | 78 +++-- extension/coldfront/src/coldfront.c | 55 +++- .../test/expected/array_cold_render.out | 23 +- .../test/expected/array_cold_render_1.out | 23 +- .../expected/async_write_before_claim.out | 5 + .../test/expected/cold_render_canonical.out | 5 + .../test/expected/param_cold_via_plpgsql.out | 23 +- .../expected/param_cold_via_plpgsql_1.out | 23 +- .../test/expected/settings_registered.out | 3 +- .../test/expected/tiered_insert_isolation.out | 300 ++++++++++++++++++ .../expected/tiered_insert_single_pass.out | 40 ++- .../expected/tiered_insert_single_pass_1.out | 40 ++- .../test/expected/tiered_insert_stream.out | 34 +- .../test/expected/tiered_insert_stream_1.out | 34 +- .../test/sql/tiered_insert_isolation.sql | 90 ++++++ .../test/sql/tiered_insert_stream.sql | 12 +- 20 files changed, 703 insertions(+), 168 deletions(-) create mode 100644 extension/coldfront/test/expected/tiered_insert_isolation.out create mode 100644 extension/coldfront/test/sql/tiered_insert_isolation.sql diff --git a/docs/architecture_tiered.md b/docs/architecture_tiered.md index 8368618..e827449 100644 --- a/docs/architecture_tiered.md +++ b/docs/architecture_tiered.md @@ -339,26 +339,35 @@ The query is deparsed with an empty `search_path`, so every relation, function and operator is schema-qualified whatever the second session's path is, and its literals and parameters are rendered by `coldfront._canonical_text`, whose ISO dates, full-precision floats and fixed interval shape read the same under -that session's `DateStyle`, `IntervalStyle` and `extra_float_digits`. `pgstream` is attached at -READ COMMITTED, so each statement's read takes a snapshot of its own and sees -what the hot half saw; the `pglocal` attachment the other cold paths read -through stays at REPEATABLE READ, which the extension needs to give a parallel -table scan one snapshot. The hot half still reads the source as the caller, so +that session's `DateStyle`, `IntervalStyle` and `extra_float_digits`. The +second session reads at the statement's snapshot: the rewrite exports it with +`coldfront._stream_snapshot()` as the statement runs, and `pgstream`, attached +at REPEATABLE READ, imports it with `SET TRANSACTION SNAPSHOT` before its first +read, so both halves see one state of the tables however other sessions commit +meanwhile, and a REPEATABLE READ transaction's later statements stream from +the snapshot they share. The hot half reads the source as the caller, so PostgreSQL checks the caller's `SELECT` privilege on it; the second session reads as the `coldfront.local_pg_dsn` user. The stream reads the source twice, once per half, so it is used only when the -hook can prove the second read sees what the first did: the transaction is -READ COMMITTED; the `INSERT` has no `WITH` clause; every function in the source -is IMMUTABLE (a volatile one yields other rows, and a stable one such as -`now()`, `ts + interval '1 day'` or a text-to-timestamptz cast may yield other -values under the second session's settings); the source reads only tables, no +hook can prove the second read sees what the first did: the transaction is not +SERIALIZABLE, whose conflict detection would not see the second session's +read; the `INSERT` has no `WITH` clause; every function in the source is +IMMUTABLE (a volatile one yields other rows, and a stable one such as `now()`, +`ts + interval '1 day'` or a text-to-timestamptz cast may yield other values +under the second session's settings); the source reads only tables, no temporary table, no view over one or over such a function, no foreign table; -this transaction has written none of those tables (or their partitions); and -`coldfront.local_pg_dsn` is set. A cached plan runs in transactions the hook -never saw, so the stream checks the isolation level and the written tables -again at execution and, when either fails, runs the same query in the calling -session and feeds its rows to the sink below. +this transaction has written none of those tables (or their partitions), since +its own writes are not in the shared snapshot; and `coldfront.local_pg_dsn` is +set. The snapshot is shared once per transaction: a second table-reading +`INSERT` in one READ COMMITTED transaction, or one after a `VALUES` source +streamed in it, takes the sink. A `VALUES` source reads no table and streams +without one. A statement in a subtransaction (a `SAVEPOINT`, a plpgsql +`EXCEPTION` block) cannot export a snapshot, and pg_duckdb refuses every cold +write there, so it fails with pg_duckdb's error rather than a snapshot one. A +cached plan runs in transactions the hook never saw, so the stream +checks these conditions again at execution and, when one fails, runs the same +query in the calling session and feeds its rows to the sink below. Every other source takes the sink shape, which reads the source once: diff --git a/docs/changelog.md b/docs/changelog.md index 7fb2cdc..01bd20f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -81,13 +81,15 @@ this project adheres to ### Changed - A tiered `INSERT` streams its cold rows: the cold projection, identity values - and defaults included, runs in PostgreSQL in a second session and its rows go - straight into the Iceberg writer in one pass, with nothing rendered per row. - A source the hook cannot read a second time (a volatile or stable function, a - temporary table, a `WITH` entry, a table the transaction has written, a - transaction that is not READ COMMITTED) is read once for both tiers and its - cold rows go through `coldfront._cold_sink`, whose cost is in proportion to - the rows it renders. `cold_rows` in the + and defaults included, runs in PostgreSQL in a second session, at the + statement's own snapshot, and its rows go straight into the Iceberg writer in + one pass, with nothing rendered per row. Both tiers see one state of the + source however other sessions commit meanwhile, and a REPEATABLE READ + transaction reads its snapshot on both. A source the hook cannot read a + second time (a volatile or stable function, a temporary table, a `WITH` + entry, a table the transaction has written, a SERIALIZABLE transaction) is + read once for both tiers and its cold rows go through `coldfront._cold_sink`, + whose cost is in proportion to the rows it renders. `cold_rows` in the statement's result is `NULL` for a streamed `INSERT`. - Every append to the cold tier (`INSERT`, `COPY`, the archiver's exports, the tiered `INSERT`'s cold sink) uses the async ordering wherever the patched diff --git a/docs/usage.md b/docs/usage.md index d5d6a68..9a72197 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1120,17 +1120,21 @@ Keep the following caveats in mind when running either mode: `coldfront.local_pg_dsn`, and its rows go straight into the Iceberg writer in one pass. An omitted IDENTITY column takes `nextval()` on the hot table's sequence, so cold ids share it with the hot side, and an omitted column with - a DEFAULT takes it. The hot rows are one set-based `INSERT`. The stream - reads the source once per tier, so a source it cannot read twice (a volatile - or stable function such as `now()`, a temporary table, a `WITH` entry, a - table the transaction has written, or a transaction that is not READ - COMMITTED) goes through `coldfront._cold_sink` instead, which renders each - row in plpgsql and writes Iceberg in batches of - `coldfront.cold_write_batch_size` rows, as does a server without - `coldfront.local_pg_dsn`. The statement's `cold_rows` column is `NULL` when - the cold rows streamed. A value that reaches the second session or DuckDB - as text is rendered in a form every reader takes as the same value, whatever - the caller's `DateStyle`, `IntervalStyle` or `extra_float_digits`. + a DEFAULT takes it. The hot rows are one set-based `INSERT`. The second + session reads the source at the statement's own snapshot, so both tiers see + one state of the tables whatever another session commits meanwhile, and a + REPEATABLE READ transaction reads its snapshot on both tiers. A source the + stream cannot read the same way twice (a volatile or stable function such as + `now()`, a temporary table, a `WITH` entry, a table the transaction has + written, a SERIALIZABLE transaction, or a second table-reading `INSERT` in + one READ COMMITTED transaction) goes through + `coldfront._cold_sink` instead, which renders each row in plpgsql and writes + Iceberg in batches of `coldfront.cold_write_batch_size` rows, as does a + server without `coldfront.local_pg_dsn`. The statement's `cold_rows` column + is `NULL` when the cold rows streamed. A value that reaches the second + session or DuckDB as text is rendered in a form every reader takes as the + same value, whatever the caller's `DateStyle`, `IntervalStyle` or + `extra_float_digits`. - `COPY FROM` reads the rows with PostgreSQL's `COPY` reader and writes them through that same `INSERT` path, `coldfront.cold_write_batch_size` rows per `INSERT`. The format options are the reader's, and a supplied value for a diff --git a/extension/coldfront/Makefile b/extension/coldfront/Makefile index 9fae8ec..467857a 100644 --- a/extension/coldfront/Makefile +++ b/extension/coldfront/Makefile @@ -30,7 +30,7 @@ REGRESS = load_order update_unregistered_view update_heap_table \ cold_write_json_agg duckdb_temp_dir \ vector_centroids vector_type_map vector_cold_render array_cold_render vector_param_render vector_ops vector_probe vector_status vector_multicolumn \ duckdb_keyword_column partition_clause failed_txn_block tiered_insert_single_pass tiered_view_no_trigger copy_tiered_view param_in_subquery merge_on_view delete_without_where update_from_without_alias settings_registered ensure_replicated read_inner_product \ - async_write_before_claim tiered_insert_stream cold_render_canonical + async_write_before_claim tiered_insert_stream cold_render_canonical tiered_insert_isolation REGRESS_OPTS = --inputdir=test --outputdir=test PG_CONFIG ?= pg_config diff --git a/extension/coldfront/coldfront--1.0.sql b/extension/coldfront/coldfront--1.0.sql index b21cbeb..a201fe7 100644 --- a/extension/coldfront/coldfront--1.0.sql +++ b/extension/coldfront/coldfront--1.0.sql @@ -424,15 +424,14 @@ $$ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp; -- ensure_stream_attached() attaches the local instance a second time, as -- `pgstream`, for the tiered INSERT's cold stream (coldfront._tiered_cold_stream). --- pglocal runs REPEATABLE READ, which DuckDB's postgres extension needs to --- give a parallel table scan one snapshot, and that pins the first read's --- snapshot for the rest of the DuckDB transaction; the stream wants each --- statement to read what its hot half read, so its attachment runs READ --- COMMITTED, and a stream is one connection with no parallel scan to keep --- consistent. The stream's query draws identity values and defaults in that --- session, so unlike pglocal's uses it writes: nextval() on the hot table's --- sequences. application_name tags the session and keeps the connection --- string distinct from pglocal's. +-- The stream reads the source at the snapshot the calling statement exported, +-- which SET TRANSACTION SNAPSHOT takes only in a REPEATABLE READ or +-- SERIALIZABLE transaction, so the attachment runs REPEATABLE READ, and the +-- session holds that snapshot for the rest of the DuckDB transaction, as +-- pglocal's does. The stream's query draws identity values and defaults in +-- that session, so unlike pglocal's uses it writes: nextval() on the hot +-- table's sequences. application_name tags the session and keeps the +-- connection string distinct from pglocal's. CREATE OR REPLACE FUNCTION coldfront.ensure_stream_attached() RETURNS void AS $$ DECLARE dsn text := current_setting('coldfront.local_pg_dsn', true); @@ -440,7 +439,7 @@ BEGIN IF dsn IS NOT NULL AND dsn <> '' THEN PERFORM duckdb.raw_query('LOAD postgres'); PERFORM duckdb.raw_query(format( - 'ATTACH IF NOT EXISTS %L AS pgstream (TYPE postgres, isolation_level ''READ COMMITTED'')', + 'ATTACH IF NOT EXISTS %L AS pgstream (TYPE postgres, isolation_level ''REPEATABLE READ'')', dsn || ' application_name=coldfront_pgstream' )); END IF; @@ -525,8 +524,8 @@ BEGIN WHERE p.pronamespace = 'coldfront'::regnamespace AND p.proname IN ('ensure_attached', 'ensure_pg_attached', 'ensure_stream_attached', '_exec_iceberg_with_claim', - -- the tiered INSERT's cold stream and its execution-time check - '_tiered_cold_stream', '_written_in_xact', + -- the tiered INSERT's cold stream, its snapshot and its execution-time check + '_tiered_cold_stream', '_stream_snapshot', '_written_in_xact', -- the tiered INSERT's cold sink, its statement and its row renderer '_cold_sink', '_cold_sink_step', '_cold_sink_final', '_cold_sink_sql', '_cold_row_literal', @@ -1074,6 +1073,15 @@ CREATE AGGREGATE coldfront._cold_sink(text, text, text, bigint) ( CREATE FUNCTION coldfront._written_in_xact(p_rels regclass[]) RETURNS boolean LANGUAGE c AS 'coldfront', 'coldfront_written_in_xact'; +-- coldfront._stream_snapshot: the statement's snapshot, exported for the +-- stream's second session, or NULL where it cannot be shared: under +-- SERIALIZABLE, whose conflict detection would not see that session's read, +-- and inside a subtransaction, which cannot export one and where pg_duckdb +-- refuses every cold write, so the statement fails with pg_duckdb's error +-- rather than a snapshot one. +CREATE FUNCTION coldfront._stream_snapshot() RETURNS text +LANGUAGE c AS 'coldfront', 'coldfront_stream_snapshot'; + -- coldfront._tiered_cold_stream: the cold half of a tiered-view INSERT whose -- source the hook proved safe to read from another session (emit_tiered_insert -- in coldfront.c). p_pg_sql is that read: the cold projection, every hot-table @@ -1085,16 +1093,26 @@ LANGUAGE c AS 'coldfront', 'coldfront_written_in_xact'; -- clustered table and ordered by it so this write's row groups each hold -- roughly one cluster. p_any_cold is false when the source has no cold row, -- and nothing runs: an all-hot INSERT stages nothing and takes no claim. --- p_sources are the tables the source reads. A cached plan runs in --- transactions the hook never saw, so two of its conditions are checked again --- here: when the transaction is not READ COMMITTED, or has written a source --- table, the other session's snapshot would not match the hot half's, so the --- same read runs in this session and its rows go through the cold sink. --- Returns the cold row count where this session counted it, 0 with no cold --- row and the sink's count after the fallback, and NULL when the rows --- streamed, since DuckDB keeps that count. +-- p_sources are the tables the source reads, and p_snapshot is the statement's +-- snapshot, exported by coldfront._stream_snapshot in the same statement, or +-- NULL where it could not be. The pgstream session reads the source at that +-- snapshot, so both halves see one state of the tables however other sessions +-- commit meanwhile: SET TRANSACTION SNAPSHOT imports it as the first statement +-- of that session's transaction, which DuckDB holds open for the rest of this +-- one. The transaction-local coldfront._stream_state records what the session +-- then holds: 'imported', this transaction's snapshot, which a later statement +-- shares only under REPEATABLE READ, where the snapshot is the transaction's; +-- or 'used', a snapshot of its own, after a VALUES source, which reads no +-- table and streams without one. A source that reads tables streams when its +-- snapshot can be shared and this transaction has written none of the tables +-- (its own writes are not in the snapshot); otherwise, as in a transaction the +-- hook never saw (a cached plan), the same read runs in this session and its +-- rows go through the cold sink. Returns the cold row count where this session +-- counted it, 0 with no cold row and the sink's count after the fallback, and +-- NULL when the rows streamed, since DuckDB keeps that count. CREATE FUNCTION coldfront._tiered_cold_stream( - p_schema text, p_view text, p_pg_sql text, p_sources regclass[], p_any_cold boolean + p_schema text, p_view text, p_pg_sql text, p_sources regclass[], p_any_cold boolean, + p_snapshot text ) RETURNS bigint LANGUAGE plpgsql AS $$ DECLARE @@ -1104,6 +1122,8 @@ DECLARE v_types text[]; v_prefix text; v_count bigint; + v_state text := current_setting('coldfront._stream_state'); + v_shared boolean := cardinality(p_sources) > 0; BEGIN IF NOT p_any_cold THEN RETURN 0; @@ -1126,8 +1146,12 @@ BEGIN AND c.relname = (parse_ident(v_hot))[2] AND a.attnum > 0 AND NOT a.attisdropped AND NOT coldfront._is_vec_companion(a.attname, a.attgenerated); - IF current_setting('transaction_isolation') <> 'read committed' - OR coldfront._written_in_xact(p_sources) THEN + IF v_shared + AND (p_snapshot IS NULL + OR v_state = 'used' + OR (v_state = 'imported' + AND current_setting('transaction_isolation') <> 'repeatable read') + OR coldfront._written_in_xact(p_sources)) THEN EXECUTE coldfront._cold_sink_sql(p_schema, p_view, v_names, v_types, p_pg_sql) INTO v_count; RETURN v_count; @@ -1136,6 +1160,14 @@ BEGIN SET LOCAL duckdb.unsafe_allow_mixed_transactions = on; PERFORM coldfront.ensure_attached(); PERFORM coldfront.ensure_stream_attached(); + IF v_state = '' THEN + IF v_shared THEN + PERFORM duckdb.raw_query(format('CALL postgres_execute(%L, %L)', 'pgstream', + format('SET TRANSACTION SNAPSHOT %L', p_snapshot))); + END IF; + PERFORM set_config('coldfront._stream_state', + CASE WHEN v_shared THEN 'imported' ELSE 'used' END, true); + END IF; PERFORM coldfront._exec_iceberg_with_claim(v_iceberg, format( 'INSERT INTO %s SELECT %s%s FROM postgres_query(%L, %L) AS s%s', v_iceberg, v_prefix, diff --git a/extension/coldfront/src/coldfront.c b/extension/coldfront/src/coldfront.c index 569272f..7e427c1 100644 --- a/extension/coldfront/src/coldfront.c +++ b/extension/coldfront/src/coldfront.c @@ -269,6 +269,7 @@ static int coldfront_peer_alive_window_ms = 10000; static bool coldfront_iceberg_async_parquet = false; static bool coldfront_iceberg_bakery_patch = false; static char *coldfront_claimed = NULL; +static char *coldfront_stream_state = NULL; static bool coldfront_async_downgrade_warned = false; static post_parse_analyze_hook_type prev_post_parse_analyze_hook = NULL; @@ -2425,16 +2426,19 @@ build_tiered_hot_dml(const char *hot_table, const char *col_list, /* * Whether a tiered INSERT's cold half can stream, and the tables its source * reads. The stream reads the source again from another session over libpq - * (coldfront._tiered_cold_stream), so it is sound only when that session sees - * what the hot half saw: coldfront.local_pg_dsn names it; the transaction is - * READ COMMITTED, so the hot half's snapshot is the statement's and not an - * older one; the source has no WITH entry; every function in it is IMMUTABLE, - * since a volatile one yields other rows and a stable one (now(), the - * timezone-dependent timestamptz arithmetic, a text-to-timestamptz cast) may - * yield other values in a session with other settings; it reads no temporary - * table, no view that reads one or calls such a function, nothing but tables; - * and this transaction has written none of those tables. Anything else goes - * to the sink shape, which reads the source once for both halves. + * (coldfront._tiered_cold_stream), at this statement's snapshot, which the + * emitted SQL exports (coldfront._stream_snapshot) and that session imports + * before it reads. So it is sound when: coldfront.local_pg_dsn names the + * session; the transaction is not SERIALIZABLE, whose conflict detection would + * not see that session's read; the source has no WITH entry; every function in + * it is IMMUTABLE, since a volatile one yields other rows and a stable one + * (now(), the timezone-dependent timestamptz arithmetic, a text-to-timestamptz + * cast) may yield other values in a session with other settings; it reads no + * temporary table, no view that reads one or calls such a function, nothing + * but tables; and this transaction has written none of those tables, which + * the shared snapshot would not show. Anything else goes to the sink shape, + * which reads the source once for both halves, as does, at execution, a + * statement whose snapshot cannot be shared (_tiered_cold_stream). */ typedef struct { Oid result_relid; /* the view being written, or the view being expanded */ @@ -2588,6 +2592,20 @@ coldfront_written_in_xact(PG_FUNCTION_ARGS) PG_RETURN_BOOL(cf_written_in_xact(rels)); } +/* The statement's snapshot, exported for the stream's second session, or NULL + * where it cannot be shared: a SERIALIZABLE transaction's conflict detection + * would not see that session's read, and a subtransaction cannot export one + * (pg_duckdb refuses every cold write there, so the statement then fails with + * pg_duckdb's error rather than a snapshot one). */ +PG_FUNCTION_INFO_V1(coldfront_stream_snapshot); +Datum +coldfront_stream_snapshot(PG_FUNCTION_ARGS) +{ + if (IsSubTransaction() || XactIsoLevel == XACT_SERIALIZABLE) + PG_RETURN_NULL(); + PG_RETURN_TEXT_P(cstring_to_text(ExportSnapshot(GetActiveSnapshot()))); +} + static bool cf_tiered_insert_streams(Query *query, Oid view_relid, List **sources) { @@ -2596,7 +2614,7 @@ cf_tiered_insert_streams(Query *query, Oid view_relid, List **sources) *sources = NIL; if (coldfront_local_pg_dsn == NULL || coldfront_local_pg_dsn[0] == '\0') return false; - if (IsolationUsesXactSnapshot() || query->cteList != NIL) + if (XactIsoLevel == XACT_SERIALIZABLE || query->cteList != NIL) return false; if (contain_mutable_functions((Node *) query)) return false; @@ -2859,10 +2877,11 @@ emit_tiered_insert(Query *query, TieredViewInfo *info, ColdParamSet *ps, ? psprintf("(SELECT count(*) FROM coldfront_hot) < %d", nvalues) : psprintf("EXISTS (SELECT 1 FROM (%s) AS coldfront_src(%s) WHERE %s < %s)", source, col_list, pc_expr, cutoff_lit); - cold_select = psprintf("SELECT coldfront._tiered_cold_stream(%s, %s, %s, %s, %s) AS n", + cold_select = psprintf("SELECT coldfront._tiered_cold_stream(%s, %s, %s, %s, %s, " + "CASE WHEN %s THEN coldfront._stream_snapshot() END) AS n", quote_literal_cstr(vschema), quote_literal_cstr(vname), cf_stream_pg_select(query, info, &hc, col_list, ps), - cf_regclass_array(sources), any_cold); + cf_regclass_array(sources), any_cold, any_cold); return wrap_tiered_result(in_plpgsql, &dr, NULL, hot_dml, cold_select, "< 0"); } @@ -6187,6 +6206,16 @@ register_gucs(void) 0, NULL, NULL, NULL); + DefineCustomStringVariable( + "coldfront._stream_state", + "The snapshot the tiered stream's second session reads at in this transaction: none yet, one of its own (used), or this transaction's (imported).", + NULL, + &coldfront_stream_state, + "", + PGC_USERSET, + 0, + NULL, NULL, NULL); + DefineCustomBoolVariable( "coldfront._async_downgrade_warned", "This session has logged its fall-back from the async ordering.", diff --git a/extension/coldfront/test/expected/array_cold_render.out b/extension/coldfront/test/expected/array_cold_render.out index d37907e..38c63c5 100644 --- a/extension/coldfront/test/expected/array_cold_render.out +++ b/extension/coldfront/test/expected/array_cold_render.out @@ -108,10 +108,10 @@ VALUES ('public', 'tags', '2026-03-01'::timestamptz); -- The INSERT's cold half checks each array's shape in its projection. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.tags VALUES (1, '2026-01-05', '{a,b}', '{1,2}'); - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: (InitPlan 4).col1, (InitPlan 5).col1 + Output: (InitPlan 5).col1, (InitPlan 6).col1 CTE coldfront_hot -> Insert on public._tags Output: 1 @@ -120,21 +120,26 @@ EXPLAIN (COSTS OFF, VERBOSE) One-Time Filter: false CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'tags'::text, 'SELECT coldfront_src.id::integer AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront._cold_list(coldfront_src.tags::text[], ''tags'') AS tags, coldfront._cold_list(coldfront_src.nums::integer[], ''nums'') AS nums FROM (VALUES (1, ''2026-01-05 00:00:00+00''::timestamp with time zone, ''{a,b}''::text[], ''{1,2}''::integer[])) AS coldfront_src(id, ts, tags, nums) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ((InitPlan 2).col1 < 1)) + Output: _tiered_cold_stream('public'::text, 'tags'::text, 'SELECT coldfront_src.id::integer AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront._cold_list(coldfront_src.tags::text[], ''tags'') AS tags, coldfront._cold_list(coldfront_src.nums::integer[], ''nums'') AS nums FROM (VALUES (1, ''2026-01-05 00:00:00+00''::timestamp with time zone, ''{a,b}''::text[], ''{1,2}''::integer[])) AS coldfront_src(id, ts, tags, nums) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ((InitPlan 2).col1 < 1), CASE WHEN ((InitPlan 3).col1 < 1) THEN _stream_snapshot() ELSE NULL::text END) InitPlan 2 -> Aggregate Output: count(*) -> CTE Scan on coldfront_hot Output: coldfront_hot."?column?" - InitPlan 4 + InitPlan 3 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" + InitPlan 5 -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot coldfront_hot_1 - Output: coldfront_hot_1."?column?" - InitPlan 5 + -> CTE Scan on coldfront_hot coldfront_hot_2 + Output: coldfront_hot_2."?column?" + InitPlan 6 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(24 rows) +(29 rows) -- A cold write's bound array parameter is rendered by _render_cold_param at -- execution. force_generic_plan keeps $1 from folding to a constant. diff --git a/extension/coldfront/test/expected/array_cold_render_1.out b/extension/coldfront/test/expected/array_cold_render_1.out index 5a63481..7fbc379 100644 --- a/extension/coldfront/test/expected/array_cold_render_1.out +++ b/extension/coldfront/test/expected/array_cold_render_1.out @@ -108,10 +108,10 @@ VALUES ('public', 'tags', '2026-03-01'::timestamptz); -- The INSERT's cold half checks each array's shape in its projection. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.tags VALUES (1, '2026-01-05', '{a,b}', '{1,2}'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: $4, $5 + Output: $5, $6 CTE coldfront_hot -> Insert on public._tags Output: 1 @@ -120,21 +120,26 @@ EXPLAIN (COSTS OFF, VERBOSE) One-Time Filter: false CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'tags'::text, 'SELECT coldfront_src.id::integer AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront._cold_list(coldfront_src.tags::text[], ''tags'') AS tags, coldfront._cold_list(coldfront_src.nums::integer[], ''nums'') AS nums FROM (VALUES (1, ''2026-01-05 00:00:00+00''::timestamp with time zone, ''{a,b}''::text[], ''{1,2}''::integer[])) AS coldfront_src(id, ts, tags, nums) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ($2 < 1)) + Output: _tiered_cold_stream('public'::text, 'tags'::text, 'SELECT coldfront_src.id::integer AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront._cold_list(coldfront_src.tags::text[], ''tags'') AS tags, coldfront._cold_list(coldfront_src.nums::integer[], ''nums'') AS nums FROM (VALUES (1, ''2026-01-05 00:00:00+00''::timestamp with time zone, ''{a,b}''::text[], ''{1,2}''::integer[])) AS coldfront_src(id, ts, tags, nums) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ($2 < 1), CASE WHEN ($3 < 1) THEN _stream_snapshot() ELSE NULL::text END) InitPlan 2 (returns $2) -> Aggregate Output: count(*) -> CTE Scan on coldfront_hot Output: coldfront_hot."?column?" - InitPlan 4 (returns $4) + InitPlan 3 (returns $3) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" + InitPlan 5 (returns $5) -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot coldfront_hot_1 - Output: coldfront_hot_1."?column?" - InitPlan 5 (returns $5) + -> CTE Scan on coldfront_hot coldfront_hot_2 + Output: coldfront_hot_2."?column?" + InitPlan 6 (returns $6) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(24 rows) +(29 rows) -- A cold write's bound array parameter is rendered by _render_cold_param at -- execution. force_generic_plan keeps $1 from folding to a constant. diff --git a/extension/coldfront/test/expected/async_write_before_claim.out b/extension/coldfront/test/expected/async_write_before_claim.out index 3870dec..ab7cc7e 100644 --- a/extension/coldfront/test/expected/async_write_before_claim.out +++ b/extension/coldfront/test/expected/async_write_before_claim.out @@ -160,6 +160,11 @@ BOOLEAN [ Rows: 0] +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + NOTICE: result: Count BIGINT [ Rows: 1] diff --git a/extension/coldfront/test/expected/cold_render_canonical.out b/extension/coldfront/test/expected/cold_render_canonical.out index 3f75a08..a96946e 100644 --- a/extension/coldfront/test/expected/cold_render_canonical.out +++ b/extension/coldfront/test/expected/cold_render_canonical.out @@ -88,6 +88,11 @@ BOOLEAN [ Rows: 0] +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + NOTICE: result: Count BIGINT [ Rows: 1] diff --git a/extension/coldfront/test/expected/param_cold_via_plpgsql.out b/extension/coldfront/test/expected/param_cold_via_plpgsql.out index f0cb1ee..d422908 100644 --- a/extension/coldfront/test/expected/param_cold_via_plpgsql.out +++ b/extension/coldfront/test/expected/param_cold_via_plpgsql.out @@ -130,10 +130,10 @@ INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) VALUES ('public', 'eid', '2026-03-01'::timestamptz); PREPARE eid_ins(timestamptz, bytea) AS INSERT INTO public.eid (ts, data) VALUES ($1, $2); EXPLAIN (COSTS OFF, VERBOSE) EXECUTE eid_ins('2026-01-10 00:00+00', '\xcafe'::bytea); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result - Output: (InitPlan 4).col1, (InitPlan 5).col1 + Output: (InitPlan 5).col1, (InitPlan 6).col1 CTE coldfront_hot -> Insert on public._eid Output: 1 @@ -142,21 +142,26 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE eid_ins('2026-01-10 00:00+00', '\xcafe'::by One-Time Filter: ($1 >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'eid'::text, format('SELECT nextval(''public._eid_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.data::bytea AS data FROM (VALUES (%1$L, %2$L)) AS coldfront_src(ts, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, _canonical_text($1), _canonical_text($2)), '{}'::regclass[], ((InitPlan 2).col1 < 1)) + Output: _tiered_cold_stream('public'::text, 'eid'::text, format('SELECT nextval(''public._eid_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.data::bytea AS data FROM (VALUES (%1$L, %2$L)) AS coldfront_src(ts, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, _canonical_text($1), _canonical_text($2)), '{}'::regclass[], ((InitPlan 2).col1 < 1), CASE WHEN ((InitPlan 3).col1 < 1) THEN _stream_snapshot() ELSE NULL::text END) InitPlan 2 -> Aggregate Output: count(*) -> CTE Scan on coldfront_hot Output: coldfront_hot."?column?" - InitPlan 4 + InitPlan 3 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" + InitPlan 5 -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot coldfront_hot_1 - Output: coldfront_hot_1."?column?" - InitPlan 5 + -> CTE Scan on coldfront_hot coldfront_hot_2 + Output: coldfront_hot_2."?column?" + InitPlan 6 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(24 rows) +(29 rows) DEALLOCATE cold_upd; DEALLOCATE cold_del; diff --git a/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out b/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out index e857b90..ea9f364 100644 --- a/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out +++ b/extension/coldfront/test/expected/param_cold_via_plpgsql_1.out @@ -130,10 +130,10 @@ INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) VALUES ('public', 'eid', '2026-03-01'::timestamptz); PREPARE eid_ins(timestamptz, bytea) AS INSERT INTO public.eid (ts, data) VALUES ($1, $2); EXPLAIN (COSTS OFF, VERBOSE) EXECUTE eid_ins('2026-01-10 00:00+00', '\xcafe'::bytea); - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ Result - Output: $4, $5 + Output: $5, $6 CTE coldfront_hot -> Insert on public._eid Output: 1 @@ -142,21 +142,26 @@ EXPLAIN (COSTS OFF, VERBOSE) EXECUTE eid_ins('2026-01-10 00:00+00', '\xcafe'::by One-Time Filter: ($1 >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'eid'::text, format('SELECT nextval(''public._eid_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.data::bytea AS data FROM (VALUES (%1$L, %2$L)) AS coldfront_src(ts, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, _canonical_text($1), _canonical_text($2)), '{}'::regclass[], ($2 < 1)) + Output: _tiered_cold_stream('public'::text, 'eid'::text, format('SELECT nextval(''public._eid_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.data::bytea AS data FROM (VALUES (%1$L, %2$L)) AS coldfront_src(ts, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, _canonical_text($1), _canonical_text($2)), '{}'::regclass[], ($2 < 1), CASE WHEN ($3 < 1) THEN _stream_snapshot() ELSE NULL::text END) InitPlan 2 (returns $2) -> Aggregate Output: count(*) -> CTE Scan on coldfront_hot Output: coldfront_hot."?column?" - InitPlan 4 (returns $4) + InitPlan 3 (returns $3) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" + InitPlan 5 (returns $5) -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot coldfront_hot_1 - Output: coldfront_hot_1."?column?" - InitPlan 5 (returns $5) + -> CTE Scan on coldfront_hot coldfront_hot_2 + Output: coldfront_hot_2."?column?" + InitPlan 6 (returns $6) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(24 rows) +(29 rows) DEALLOCATE cold_upd; DEALLOCATE cold_del; diff --git a/extension/coldfront/test/expected/settings_registered.out b/extension/coldfront/test/expected/settings_registered.out index c73f663..a9b7185 100644 --- a/extension/coldfront/test/expected/settings_registered.out +++ b/extension/coldfront/test/expected/settings_registered.out @@ -12,6 +12,7 @@ SELECT name, vartype, context, boot_val, min_val -----------------------------------+---------+-----------+----------+--------- coldfront._async_downgrade_warned | bool | user | off | coldfront._claimed | string | user | | + coldfront._stream_state | string | user | | coldfront.allow_mixed_writes | bool | user | on | coldfront.cold_write_batch_size | integer | user | 10000 | 1 coldfront.iceberg_async_parquet | bool | user | off | @@ -23,7 +24,7 @@ SELECT name, vartype, context, boot_val, min_val coldfront.vector_nprobe | integer | user | 0 | 0 coldfront.vector_probe | bool | user | on | coldfront.warehouse | string | superuser | | -(13 rows) +(14 rows) -- The bakery's dead-peer window is superuser-only and at least 1 ms. CREATE ROLE cf_settings_app; diff --git a/extension/coldfront/test/expected/tiered_insert_isolation.out b/extension/coldfront/test/expected/tiered_insert_isolation.out new file mode 100644 index 0000000..b7dd09c --- /dev/null +++ b/extension/coldfront/test/expected/tiered_insert_isolation.out @@ -0,0 +1,300 @@ +-- Both tiers of a tiered INSERT see one snapshot, the one PostgreSQL gave the +-- statement, whatever another session commits meanwhile, and the transaction's +-- isolation level holds across the tiers: READ COMMITTED streams each statement +-- from its own snapshot, REPEATABLE READ streams from the transaction's, and +-- SERIALIZABLE fails as native tables do. The other session is the bakery's +-- loopback, pointed at this server. cold_rows is NULL when the cold rows +-- streamed and their count when the sink wrote them. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; + dsn_set +--------- + t +(1 row) + +SELECT set_config('coldfront.loopback_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS loopback_set; + loopback_set +-------------- + t +(1 row) + +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_iso (id BIGINT, ts TIMESTAMPTZ, status VARCHAR)') IS NOT NULL AS created; +NOTICE: result: Count +BIGINT +[ Rows: 0] + + + created +--------- + t +(1 row) + +CREATE TABLE public._cf_iso (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz NOT NULL, status text); +CREATE VIEW public.cf_iso AS SELECT * FROM public._cf_iso; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'cf_iso', 'public._cf_iso', 'temp.main.cf_iso', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'cf_iso', '2026-03-01'::timestamptz); +CREATE TABLE public.cf_iso_src (ts timestamptz, status text); +INSERT INTO public.cf_iso_src VALUES ('2026-01-01', 'cold'), ('2026-05-01', 'hot'); +CREATE VIEW public.cf_iso_cold AS + SELECT r['ts']::timestamptz AS ts, r['status']::text AS status + FROM duckdb.query('SELECT * FROM temp.main.cf_iso') AS t(r); +-- (1) READ COMMITTED: another session commits a change to the source while +-- the statement runs, between its hot half and its cold half. A statement +-- trigger on the hot table fires before the hot INSERT and commits the +-- change through the loopback. The cold half reads the statement's +-- snapshot, so the cold tier holds the row as the hot half saw it. +CREATE FUNCTION public.cf_iso_meddle() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + PERFORM coldfront._loopback('UPDATE public.cf_iso_src SET status = ''late'' WHERE status = ''cold'''); + RETURN NULL; +END; +$$; +CREATE TRIGGER cf_iso_meddle BEFORE INSERT ON public._cf_iso + FOR EACH STATEMENT EXECUTE FUNCTION public.cf_iso_meddle(); +INSERT INTO public.cf_iso (ts, status) SELECT ts, status FROM public.cf_iso_src; +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 1 | +(1 row) + +DROP TRIGGER cf_iso_meddle ON public._cf_iso; +SELECT status FROM public.cf_iso_cold; + status +-------- + cold +(1 row) + +SELECT status FROM public.cf_iso_src ORDER BY 1; + status +-------- + hot + late +(2 rows) + +-- (2) A second streamed statement in one READ COMMITTED transaction cannot +-- give the second session a fresh snapshot, so it goes through the sink; +-- a VALUES source reads no table and streams regardless. +BEGIN; +INSERT INTO public.cf_iso (ts, status) SELECT ts, 'rc1' FROM public.cf_iso_src; +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 1 | +(1 row) + +INSERT INTO public.cf_iso (ts, status) SELECT ts, 'rc2' FROM public.cf_iso_src; +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 1 | 1 +(1 row) + +INSERT INTO public.cf_iso (ts, status) VALUES ('2026-01-02', 'rc3'); +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 0 | +(1 row) + +COMMIT; +SELECT status FROM public.cf_iso_cold WHERE status LIKE 'rc%' ORDER BY 1; + status +-------- + rc1 + rc2 + rc3 +(3 rows) + +-- (3) REPEATABLE READ: the cold half reads the transaction's snapshot, so a +-- row another session commits after it is on neither tier, and the second +-- statement streams from the same snapshot. +BEGIN ISOLATION LEVEL REPEATABLE READ; +SELECT count(*) AS seen FROM public.cf_iso_src; + seen +------ + 2 +(1 row) + +SELECT coldfront._loopback('INSERT INTO public.cf_iso_src VALUES (''2026-01-03'', ''rr-late'') RETURNING status') AS committed_elsewhere; + committed_elsewhere +--------------------- + rr-late +(1 row) + +INSERT INTO public.cf_iso (ts, status) SELECT ts, status || '/rr1' FROM public.cf_iso_src; +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 1 | +(1 row) + +INSERT INTO public.cf_iso (ts, status) SELECT ts, status || '/rr2' FROM public.cf_iso_src; +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + hot_rows | cold_rows +----------+----------- + 1 | +(1 row) + +COMMIT; +SELECT status FROM public.cf_iso_cold WHERE status LIKE '%/rr%' ORDER BY 1; + status +---------- + late/rr1 + late/rr2 +(2 rows) + +SELECT status FROM public._cf_iso WHERE status LIKE '%/rr%' ORDER BY 1; + status +--------- + hot/rr1 + hot/rr2 +(2 rows) + +SELECT status FROM public.cf_iso_src ORDER BY 1; + status +--------- + hot + late + rr-late +(3 rows) + +-- (4) SERIALIZABLE: this transaction reads the source and writes the hot +-- table; the other session read the hot table and wrote the source, and +-- committed first. The cycle fails this transaction, as it would native +-- tables, and the cold tier keeps nothing from it. +BEGIN ISOLATION LEVEL SERIALIZABLE; +SELECT count(*) AS seen FROM public.cf_iso_src WHERE status IN ('late', 'hot'); + seen +------ + 2 +(1 row) + +SELECT coldfront._loopback('BEGIN ISOLATION LEVEL SERIALIZABLE; SELECT count(*) FROM public._cf_iso; UPDATE public.cf_iso_src SET status = ''skew'' WHERE status = ''late''; COMMIT; SELECT ''committed''') AS committed_elsewhere; + committed_elsewhere +--------------------- + committed +(1 row) + +INSERT INTO public.cf_iso (ts, status) SELECT ts, status || '/ser' FROM public.cf_iso_src WHERE status IN ('late', 'hot'); +ERROR: could not serialize access due to read/write dependencies among transactions +DETAIL: Reason code: Canceled on identification as a pivot, during write. +HINT: The transaction might succeed if retried. +COMMIT; +SELECT count(*) AS serializable_rows + FROM (SELECT status FROM public.cf_iso_cold UNION ALL SELECT status FROM public._cf_iso) AS u + WHERE status LIKE '%/ser'; + serializable_rows +------------------- + 0 +(1 row) + diff --git a/extension/coldfront/test/expected/tiered_insert_single_pass.out b/extension/coldfront/test/expected/tiered_insert_single_pass.out index 7383468..78b1b96 100644 --- a/extension/coldfront/test/expected/tiered_insert_single_pass.out +++ b/extension/coldfront/test/expected/tiered_insert_single_pass.out @@ -69,10 +69,10 @@ EXPLAIN (COSTS OFF, VERBOSE) EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (id, ts, status, data) OVERRIDING SYSTEM VALUE SELECT i, '2026-01-01'::timestamptz, 'old', '{}'::jsonb FROM generate_series(1, 3) i; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: (InitPlan 4).col1, (InitPlan 5).col1 + Output: (InitPlan 5).col1, (InitPlan 6).col1 CTE coldfront_hot -> Insert on public._events Output: 1 @@ -81,27 +81,30 @@ EXPLAIN (COSTS OFF, VERBOSE) One-Time Filter: false CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT coldfront_src.id::bigint AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (coldfront_src.data::jsonb)::text AS data FROM (SELECT i.i, ''2026-01-01 00:00:00+00''::timestamp with time zone AS timestamptz, ''old'', ''{}''::jsonb AS jsonb FROM generate_series(1, 3) i(i)) AS coldfront_src(id, ts, status, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], (InitPlan 2).col1) + Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT coldfront_src.id::bigint AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (coldfront_src.data::jsonb)::text AS data FROM (SELECT i.i, ''2026-01-01 00:00:00+00''::timestamp with time zone AS timestamptz, ''old'', ''{}''::jsonb AS jsonb FROM generate_series(1, 3) i(i)) AS coldfront_src(id, ts, status, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], (InitPlan 2).col1, CASE WHEN (InitPlan 3).col1 THEN _stream_snapshot() ELSE NULL::text END) InitPlan 2 -> Function Scan on pg_catalog.generate_series i Function Call: generate_series(1, 3) - InitPlan 4 + InitPlan 3 + -> Function Scan on pg_catalog.generate_series i_1 + Function Call: generate_series(1, 3) + InitPlan 5 -> Aggregate Output: count(*) -> CTE Scan on coldfront_hot Output: coldfront_hot."?column?" - InitPlan 5 + InitPlan 6 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(22 rows) +(25 rows) -- VALUES takes the same shape. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (ts, status) VALUES ('2026-01-01', 'a'), ('2026-05-01', 'b'); - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: (InitPlan 4).col1, (InitPlan 5).col1 + Output: (InitPlan 5).col1, (InitPlan 6).col1 CTE coldfront_hot -> Insert on public._events Output: 1 @@ -110,21 +113,26 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: ("*VALUES*".column1 >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT nextval(''public._events_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (NULL::jsonb)::text AS data FROM (VALUES (''2026-01-01 00:00:00+00''::timestamp with time zone,''a''::text), (''2026-05-01 00:00:00+00''::timestamp with time zone,''b''::text)) AS coldfront_src(ts, status) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ((InitPlan 2).col1 < 2)) + Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT nextval(''public._events_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (NULL::jsonb)::text AS data FROM (VALUES (''2026-01-01 00:00:00+00''::timestamp with time zone,''a''::text), (''2026-05-01 00:00:00+00''::timestamp with time zone,''b''::text)) AS coldfront_src(ts, status) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ((InitPlan 2).col1 < 2), CASE WHEN ((InitPlan 3).col1 < 2) THEN _stream_snapshot() ELSE NULL::text END) InitPlan 2 -> Aggregate Output: count(*) -> CTE Scan on coldfront_hot Output: coldfront_hot."?column?" - InitPlan 4 + InitPlan 3 + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" + InitPlan 5 -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot coldfront_hot_1 - Output: coldfront_hot_1."?column?" - InitPlan 5 + -> CTE Scan on coldfront_hot coldfront_hot_2 + Output: coldfront_hot_2."?column?" + InitPlan 6 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(24 rows) +(29 rows) -- Cleanup. DELETE FROM coldfront.tiered_views; diff --git a/extension/coldfront/test/expected/tiered_insert_single_pass_1.out b/extension/coldfront/test/expected/tiered_insert_single_pass_1.out index 47365ca..980009a 100644 --- a/extension/coldfront/test/expected/tiered_insert_single_pass_1.out +++ b/extension/coldfront/test/expected/tiered_insert_single_pass_1.out @@ -68,10 +68,10 @@ EXPLAIN (COSTS OFF, VERBOSE) EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (id, ts, status, data) OVERRIDING SYSTEM VALUE SELECT i, '2026-01-01'::timestamptz, 'old', '{}'::jsonb FROM generate_series(1, 3) i; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: $4, $5 + Output: $5, $6 CTE coldfront_hot -> Insert on public._events Output: 1 @@ -80,27 +80,30 @@ EXPLAIN (COSTS OFF, VERBOSE) One-Time Filter: false CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT coldfront_src.id::bigint AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (coldfront_src.data::jsonb)::text AS data FROM (SELECT i.i, ''2026-01-01 00:00:00+00''::timestamp with time zone AS timestamptz, ''old'', ''{}''::jsonb AS jsonb FROM generate_series(1, 3) i(i)) AS coldfront_src(id, ts, status, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], $2) + Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT coldfront_src.id::bigint AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (coldfront_src.data::jsonb)::text AS data FROM (SELECT i.i, ''2026-01-01 00:00:00+00''::timestamp with time zone AS timestamptz, ''old'', ''{}''::jsonb AS jsonb FROM generate_series(1, 3) i(i)) AS coldfront_src(id, ts, status, data) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], $2, CASE WHEN $3 THEN _stream_snapshot() ELSE NULL::text END) InitPlan 2 (returns $2) -> Function Scan on pg_catalog.generate_series i Function Call: generate_series(1, 3) - InitPlan 4 (returns $4) + InitPlan 3 (returns $3) + -> Function Scan on pg_catalog.generate_series i_1 + Function Call: generate_series(1, 3) + InitPlan 5 (returns $5) -> Aggregate Output: count(*) -> CTE Scan on coldfront_hot Output: coldfront_hot."?column?" - InitPlan 5 (returns $5) + InitPlan 6 (returns $6) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(22 rows) +(25 rows) -- VALUES takes the same shape. EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.events (ts, status) VALUES ('2026-01-01', 'a'), ('2026-05-01', 'b'); - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: $4, $5 + Output: $5, $6 CTE coldfront_hot -> Insert on public._events Output: 1 @@ -109,21 +112,26 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: ("*VALUES*".column1 >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT nextval(''public._events_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (NULL::jsonb)::text AS data FROM (VALUES (''2026-01-01 00:00:00+00''::timestamp with time zone,''a''::text), (''2026-05-01 00:00:00+00''::timestamp with time zone,''b''::text)) AS coldfront_src(ts, status) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ($2 < 2)) + Output: _tiered_cold_stream('public'::text, 'events'::text, 'SELECT nextval(''public._events_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (NULL::jsonb)::text AS data FROM (VALUES (''2026-01-01 00:00:00+00''::timestamp with time zone,''a''::text), (''2026-05-01 00:00:00+00''::timestamp with time zone,''b''::text)) AS coldfront_src(ts, status) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{}'::regclass[], ($2 < 2), CASE WHEN ($3 < 2) THEN _stream_snapshot() ELSE NULL::text END) InitPlan 2 (returns $2) -> Aggregate Output: count(*) -> CTE Scan on coldfront_hot Output: coldfront_hot."?column?" - InitPlan 4 (returns $4) + InitPlan 3 (returns $3) + -> Aggregate + Output: count(*) + -> CTE Scan on coldfront_hot coldfront_hot_1 + Output: coldfront_hot_1."?column?" + InitPlan 5 (returns $5) -> Aggregate Output: count(*) - -> CTE Scan on coldfront_hot coldfront_hot_1 - Output: coldfront_hot_1."?column?" - InitPlan 5 (returns $5) + -> CTE Scan on coldfront_hot coldfront_hot_2 + Output: coldfront_hot_2."?column?" + InitPlan 6 (returns $6) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(24 rows) +(29 rows) -- Cleanup. DELETE FROM coldfront.tiered_views; diff --git a/extension/coldfront/test/expected/tiered_insert_stream.out b/extension/coldfront/test/expected/tiered_insert_stream.out index e200301..af88578 100644 --- a/extension/coldfront/test/expected/tiered_insert_stream.out +++ b/extension/coldfront/test/expected/tiered_insert_stream.out @@ -5,8 +5,8 @@ -- defaults drawn in that session. Observed end to end: a DuckDB temporary table -- stands in for the Iceberg table and is read back with duckdb.query. A source -- the hook cannot read twice (a volatile or stable function, a temporary table, --- a WITH entry, a view over one, a table this transaction wrote, a transaction --- that is not READ COMMITTED, no local_pg_dsn) goes through the cold sink in one +-- a WITH entry, a view over one, a table this transaction wrote, a SERIALIZABLE +-- transaction, no local_pg_dsn) goes through the cold sink in one -- pass, and a cached plan that meets such a transaction falls back inside the -- stream. SET client_min_messages = warning; @@ -70,10 +70,10 @@ CREATE VIEW public.cf_stream_ids AS EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status, data, b, tags) SELECT ts, status, data, b, tags FROM public.cf_staging; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: (InitPlan 4).col1, (InitPlan 5).col1 + Output: (InitPlan 5).col1, (InitPlan 6).col1 CTE coldfront_hot -> Insert on public._cf_stream Output: 1 @@ -82,19 +82,22 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (cf_staging.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'cf_stream'::text, 'SELECT nextval(''public._cf_stream_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (7) AS n, (coldfront_src.data::jsonb)::text AS data, coldfront_src.b::bytea AS b, coldfront._cold_list(coldfront_src.tags::integer[], ''tags'') AS tags FROM (SELECT cf_staging.ts, cf_staging.status, cf_staging.data, cf_staging.b, cf_staging.tags FROM public.cf_staging) AS coldfront_src(ts, status, data, b, tags) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{cf_staging}'::regclass[], (InitPlan 2).col1) + Output: _tiered_cold_stream('public'::text, 'cf_stream'::text, 'SELECT nextval(''public._cf_stream_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (7) AS n, (coldfront_src.data::jsonb)::text AS data, coldfront_src.b::bytea AS b, coldfront._cold_list(coldfront_src.tags::integer[], ''tags'') AS tags FROM (SELECT cf_staging.ts, cf_staging.status, cf_staging.data, cf_staging.b, cf_staging.tags FROM public.cf_staging) AS coldfront_src(ts, status, data, b, tags) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{cf_staging}'::regclass[], (InitPlan 2).col1, CASE WHEN (InitPlan 3).col1 THEN _stream_snapshot() ELSE NULL::text END) InitPlan 2 -> Seq Scan on public.cf_staging cf_staging_1 Filter: (cf_staging_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) - InitPlan 4 + InitPlan 3 + -> Seq Scan on public.cf_staging cf_staging_2 + Filter: (cf_staging_2.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 5 -> Aggregate Output: count(*) -> CTE Scan on coldfront_hot Output: coldfront_hot."?column?" - InitPlan 5 + InitPlan 6 -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(22 rows) +(25 rows) INSERT INTO public.cf_stream (ts, status, data, b, tags) SELECT ts, status, data, b, tags FROM public.cf_staging; @@ -108,6 +111,11 @@ BOOLEAN [ Rows: 0] +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + NOTICE: result: Count BIGINT [ Rows: 1] @@ -202,7 +210,9 @@ SELECT * FROM public.cf_stream_ids; -- (4) A prepared statement with a bound parameter streams from its cached plan -- and keeps the parameter live. Run again in a transaction that wrote the -- source table, the stream's check falls back to reading in this session, --- so the row written in the transaction is tiered too. +-- so the row written in the transaction is tiered too, and a SERIALIZABLE +-- transaction, whose snapshot the stream cannot share, falls back the same +-- way. PREPARE cf_prep(text) AS INSERT INTO public.cf_stream (ts, status) SELECT ts, $1 FROM public.cf_staging WHERE ts < '2026-03-01'; EXECUTE cf_prep('prep1'); @@ -220,7 +230,7 @@ EXECUTE cf_prep('prep2'); (1 row) COMMIT; -BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN ISOLATION LEVEL SERIALIZABLE; EXECUTE cf_prep('prep3'); hot_rows | cold_rows ----------+----------- @@ -502,7 +512,7 @@ INSERT INTO public.cf_stream (ts, status) SELECT ts, 'txn' FROM public.cf_stagin (1 row) COMMIT; -BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN ISOLATION LEVEL SERIALIZABLE; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; QUERY PLAN diff --git a/extension/coldfront/test/expected/tiered_insert_stream_1.out b/extension/coldfront/test/expected/tiered_insert_stream_1.out index 2294e86..06f1c2d 100644 --- a/extension/coldfront/test/expected/tiered_insert_stream_1.out +++ b/extension/coldfront/test/expected/tiered_insert_stream_1.out @@ -5,8 +5,8 @@ -- defaults drawn in that session. Observed end to end: a DuckDB temporary table -- stands in for the Iceberg table and is read back with duckdb.query. A source -- the hook cannot read twice (a volatile or stable function, a temporary table, --- a WITH entry, a view over one, a table this transaction wrote, a transaction --- that is not READ COMMITTED, no local_pg_dsn) goes through the cold sink in one +-- a WITH entry, a view over one, a table this transaction wrote, a SERIALIZABLE +-- transaction, no local_pg_dsn) goes through the cold sink in one -- pass, and a cached plan that meets such a transaction falls back inside the -- stream. SET client_min_messages = warning; @@ -70,10 +70,10 @@ CREATE VIEW public.cf_stream_ids AS EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status, data, b, tags) SELECT ts, status, data, b, tags FROM public.cf_staging; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: $4, $5 + Output: $5, $6 CTE coldfront_hot -> Insert on public._cf_stream Output: 1 @@ -82,19 +82,22 @@ EXPLAIN (COSTS OFF, VERBOSE) Filter: (cf_staging.ts >= 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) CTE coldfront_cold -> Result - Output: _tiered_cold_stream('public'::text, 'cf_stream'::text, 'SELECT nextval(''public._cf_stream_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (7) AS n, (coldfront_src.data::jsonb)::text AS data, coldfront_src.b::bytea AS b, coldfront._cold_list(coldfront_src.tags::integer[], ''tags'') AS tags FROM (SELECT cf_staging.ts, cf_staging.status, cf_staging.data, cf_staging.b, cf_staging.tags FROM public.cf_staging) AS coldfront_src(ts, status, data, b, tags) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{cf_staging}'::regclass[], $2) + Output: _tiered_cold_stream('public'::text, 'cf_stream'::text, 'SELECT nextval(''public._cf_stream_id_seq''::regclass) AS id, coldfront_src.ts::timestamp with time zone AS ts, coldfront_src.status::text AS status, (7) AS n, (coldfront_src.data::jsonb)::text AS data, coldfront_src.b::bytea AS b, coldfront._cold_list(coldfront_src.tags::integer[], ''tags'') AS tags FROM (SELECT cf_staging.ts, cf_staging.status, cf_staging.data, cf_staging.b, cf_staging.tags FROM public.cf_staging) AS coldfront_src(ts, status, data, b, tags) WHERE coldfront_src.ts::timestamp with time zone < ''2026-03-01 00:00:00+00''::timestamptz'::text, '{cf_staging}'::regclass[], $2, CASE WHEN $3 THEN _stream_snapshot() ELSE NULL::text END) InitPlan 2 (returns $2) -> Seq Scan on public.cf_staging cf_staging_1 Filter: (cf_staging_1.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) - InitPlan 4 (returns $4) + InitPlan 3 (returns $3) + -> Seq Scan on public.cf_staging cf_staging_2 + Filter: (cf_staging_2.ts < 'Sun Mar 01 00:00:00 2026 UTC'::timestamp with time zone) + InitPlan 5 (returns $5) -> Aggregate Output: count(*) -> CTE Scan on coldfront_hot Output: coldfront_hot."?column?" - InitPlan 5 (returns $5) + InitPlan 6 (returns $6) -> CTE Scan on coldfront_cold Output: coldfront_cold.n -(22 rows) +(25 rows) INSERT INTO public.cf_stream (ts, status, data, b, tags) SELECT ts, status, data, b, tags FROM public.cf_staging; @@ -108,6 +111,11 @@ BOOLEAN [ Rows: 0] +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + NOTICE: result: Count BIGINT [ Rows: 1] @@ -202,7 +210,9 @@ SELECT * FROM public.cf_stream_ids; -- (4) A prepared statement with a bound parameter streams from its cached plan -- and keeps the parameter live. Run again in a transaction that wrote the -- source table, the stream's check falls back to reading in this session, --- so the row written in the transaction is tiered too. +-- so the row written in the transaction is tiered too, and a SERIALIZABLE +-- transaction, whose snapshot the stream cannot share, falls back the same +-- way. PREPARE cf_prep(text) AS INSERT INTO public.cf_stream (ts, status) SELECT ts, $1 FROM public.cf_staging WHERE ts < '2026-03-01'; EXECUTE cf_prep('prep1'); @@ -220,7 +230,7 @@ EXECUTE cf_prep('prep2'); (1 row) COMMIT; -BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN ISOLATION LEVEL SERIALIZABLE; EXECUTE cf_prep('prep3'); hot_rows | cold_rows ----------+----------- @@ -496,7 +506,7 @@ INSERT INTO public.cf_stream (ts, status) SELECT ts, 'txn' FROM public.cf_stagin (1 row) COMMIT; -BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN ISOLATION LEVEL SERIALIZABLE; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; QUERY PLAN diff --git a/extension/coldfront/test/sql/tiered_insert_isolation.sql b/extension/coldfront/test/sql/tiered_insert_isolation.sql new file mode 100644 index 0000000..d6558ea --- /dev/null +++ b/extension/coldfront/test/sql/tiered_insert_isolation.sql @@ -0,0 +1,90 @@ +-- Both tiers of a tiered INSERT see one snapshot, the one PostgreSQL gave the +-- statement, whatever another session commits meanwhile, and the transaction's +-- isolation level holds across the tiers: READ COMMITTED streams each statement +-- from its own snapshot, REPEATABLE READ streams from the transaction's, and +-- SERIALIZABLE fails as native tables do. The other session is the bakery's +-- loopback, pointed at this server. cold_rows is NULL when the cold rows +-- streamed and their count when the sink wrote them. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; + +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; +SELECT set_config('coldfront.loopback_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS loopback_set; +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_iso (id BIGINT, ts TIMESTAMPTZ, status VARCHAR)') IS NOT NULL AS created; + +CREATE TABLE public._cf_iso (id bigint GENERATED ALWAYS AS IDENTITY, ts timestamptz NOT NULL, status text); +CREATE VIEW public.cf_iso AS SELECT * FROM public._cf_iso; +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col) +VALUES ('public', 'cf_iso', 'public._cf_iso', 'temp.main.cf_iso', 'ts'); +INSERT INTO coldfront.archive_watermark(schema_name, table_name, cutoff_time) +VALUES ('public', 'cf_iso', '2026-03-01'::timestamptz); +CREATE TABLE public.cf_iso_src (ts timestamptz, status text); +INSERT INTO public.cf_iso_src VALUES ('2026-01-01', 'cold'), ('2026-05-01', 'hot'); +CREATE VIEW public.cf_iso_cold AS + SELECT r['ts']::timestamptz AS ts, r['status']::text AS status + FROM duckdb.query('SELECT * FROM temp.main.cf_iso') AS t(r); + +-- (1) READ COMMITTED: another session commits a change to the source while +-- the statement runs, between its hot half and its cold half. A statement +-- trigger on the hot table fires before the hot INSERT and commits the +-- change through the loopback. The cold half reads the statement's +-- snapshot, so the cold tier holds the row as the hot half saw it. +CREATE FUNCTION public.cf_iso_meddle() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + PERFORM coldfront._loopback('UPDATE public.cf_iso_src SET status = ''late'' WHERE status = ''cold'''); + RETURN NULL; +END; +$$; +CREATE TRIGGER cf_iso_meddle BEFORE INSERT ON public._cf_iso + FOR EACH STATEMENT EXECUTE FUNCTION public.cf_iso_meddle(); +INSERT INTO public.cf_iso (ts, status) SELECT ts, status FROM public.cf_iso_src; +DROP TRIGGER cf_iso_meddle ON public._cf_iso; +SELECT status FROM public.cf_iso_cold; +SELECT status FROM public.cf_iso_src ORDER BY 1; + +-- (2) A second streamed statement in one READ COMMITTED transaction cannot +-- give the second session a fresh snapshot, so it goes through the sink; +-- a VALUES source reads no table and streams regardless. +BEGIN; +INSERT INTO public.cf_iso (ts, status) SELECT ts, 'rc1' FROM public.cf_iso_src; +INSERT INTO public.cf_iso (ts, status) SELECT ts, 'rc2' FROM public.cf_iso_src; +INSERT INTO public.cf_iso (ts, status) VALUES ('2026-01-02', 'rc3'); +COMMIT; +SELECT status FROM public.cf_iso_cold WHERE status LIKE 'rc%' ORDER BY 1; + +-- (3) REPEATABLE READ: the cold half reads the transaction's snapshot, so a +-- row another session commits after it is on neither tier, and the second +-- statement streams from the same snapshot. +BEGIN ISOLATION LEVEL REPEATABLE READ; +SELECT count(*) AS seen FROM public.cf_iso_src; +SELECT coldfront._loopback('INSERT INTO public.cf_iso_src VALUES (''2026-01-03'', ''rr-late'') RETURNING status') AS committed_elsewhere; +INSERT INTO public.cf_iso (ts, status) SELECT ts, status || '/rr1' FROM public.cf_iso_src; +INSERT INTO public.cf_iso (ts, status) SELECT ts, status || '/rr2' FROM public.cf_iso_src; +COMMIT; +SELECT status FROM public.cf_iso_cold WHERE status LIKE '%/rr%' ORDER BY 1; +SELECT status FROM public._cf_iso WHERE status LIKE '%/rr%' ORDER BY 1; +SELECT status FROM public.cf_iso_src ORDER BY 1; + +-- (4) SERIALIZABLE: this transaction reads the source and writes the hot +-- table; the other session read the hot table and wrote the source, and +-- committed first. The cycle fails this transaction, as it would native +-- tables, and the cold tier keeps nothing from it. +BEGIN ISOLATION LEVEL SERIALIZABLE; +SELECT count(*) AS seen FROM public.cf_iso_src WHERE status IN ('late', 'hot'); +SELECT coldfront._loopback('BEGIN ISOLATION LEVEL SERIALIZABLE; SELECT count(*) FROM public._cf_iso; UPDATE public.cf_iso_src SET status = ''skew'' WHERE status = ''late''; COMMIT; SELECT ''committed''') AS committed_elsewhere; +INSERT INTO public.cf_iso (ts, status) SELECT ts, status || '/ser' FROM public.cf_iso_src WHERE status IN ('late', 'hot'); +COMMIT; +SELECT count(*) AS serializable_rows + FROM (SELECT status FROM public.cf_iso_cold UNION ALL SELECT status FROM public._cf_iso) AS u + WHERE status LIKE '%/ser'; diff --git a/extension/coldfront/test/sql/tiered_insert_stream.sql b/extension/coldfront/test/sql/tiered_insert_stream.sql index 0d68ebb..869a0df 100644 --- a/extension/coldfront/test/sql/tiered_insert_stream.sql +++ b/extension/coldfront/test/sql/tiered_insert_stream.sql @@ -5,8 +5,8 @@ -- defaults drawn in that session. Observed end to end: a DuckDB temporary table -- stands in for the Iceberg table and is read back with duckdb.query. A source -- the hook cannot read twice (a volatile or stable function, a temporary table, --- a WITH entry, a view over one, a table this transaction wrote, a transaction --- that is not READ COMMITTED, no local_pg_dsn) goes through the cold sink in one +-- a WITH entry, a view over one, a table this transaction wrote, a SERIALIZABLE +-- transaction, no local_pg_dsn) goes through the cold sink in one -- pass, and a cached plan that meets such a transaction falls back inside the -- stream. SET client_min_messages = warning; @@ -83,7 +83,9 @@ SELECT * FROM public.cf_stream_ids; -- (4) A prepared statement with a bound parameter streams from its cached plan -- and keeps the parameter live. Run again in a transaction that wrote the -- source table, the stream's check falls back to reading in this session, --- so the row written in the transaction is tiered too. +-- so the row written in the transaction is tiered too, and a SERIALIZABLE +-- transaction, whose snapshot the stream cannot share, falls back the same +-- way. PREPARE cf_prep(text) AS INSERT INTO public.cf_stream (ts, status) SELECT ts, $1 FROM public.cf_staging WHERE ts < '2026-03-01'; EXECUTE cf_prep('prep1'); @@ -91,7 +93,7 @@ BEGIN; INSERT INTO public.cf_staging VALUES ('2026-01-20', 'cold4', NULL, NULL, NULL); EXECUTE cf_prep('prep2'); COMMIT; -BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN ISOLATION LEVEL SERIALIZABLE; EXECUTE cf_prep('prep3'); COMMIT; SELECT status, count(*) FROM public.cf_stream_cold WHERE status LIKE 'prep%' GROUP BY status ORDER BY status; @@ -131,7 +133,7 @@ EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; INSERT INTO public.cf_stream (ts, status) SELECT ts, 'txn' FROM public.cf_staging WHERE status = 'cold5'; COMMIT; -BEGIN ISOLATION LEVEL REPEATABLE READ; +BEGIN ISOLATION LEVEL SERIALIZABLE; EXPLAIN (COSTS OFF, VERBOSE) INSERT INTO public.cf_stream (ts, status) SELECT ts, status FROM public.cf_staging; COMMIT; From 325d5d836b6144f12f4fe9547a69365205bd2a96 Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 9 Oct 2026 16:45:38 +0100 Subject: [PATCH 7/9] fix: fail a clustered append whose centroid generation a retrain replaced --- docs/architecture.md | 2 +- docs/architecture_vectors.md | 32 ++-- docs/changelog.md | 5 + docs/usage_vectors.md | 5 +- extension/coldfront/Makefile | 2 +- extension/coldfront/coldfront--1.0.sql | 146 +++++++++++++-- extension/coldfront/src/coldfront.c | 26 ++- .../expected/async_write_before_claim.out | 4 +- .../expected/clustered_append_generation.out | 173 ++++++++++++++++++ .../coldfront/test/expected/cte_on_insert.out | 6 +- .../test/expected/cte_on_insert_1.out | 6 +- .../test/expected/settings_registered.out | 3 +- .../test/expected/vector_cold_render.out | 12 +- .../test/expected/vector_multicolumn.out | 18 +- .../test/sql/clustered_append_generation.sql | 66 +++++++ 15 files changed, 447 insertions(+), 59 deletions(-) create mode 100644 extension/coldfront/test/expected/clustered_append_generation.out create mode 100644 extension/coldfront/test/sql/clustered_append_generation.sql diff --git a/docs/architecture.md b/docs/architecture.md index 4935eea..d37880e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -361,7 +361,7 @@ The following table shows the two orderings: | Ordering | Applies to | Behavior | |---|---|---| | stock (default) | every write on stock upstream duckdb-iceberg; a `DELETE`, `UPDATE` or `MERGE` (the cross-tier move and `vector_train`'s assignment included) on every build | The writer takes the serializer first, *then* uploads parquet and commits inside it. On an unpatched binary this is what keeps a second writer from capturing a stale parent. A write that references rows the table holds needs it on every build: its position deletes name data files by path, and a compaction or another such write committing between its scan and its commit would rewrite or remove those files, after which the deletes apply to nothing and the rows come back. | -| async (both GUCs `on`) | appends (`INSERT`, `COPY`, the archiver's exports, the tiered `INSERT`'s cold sink) on the patched build (`iceberg-bakery-aware-commit-refresh-v15.patch`) | The statement uploads its parquet and queues the claim; the extension's transaction callback takes it at PRE_COMMIT, just before pg_duckdb's own PRE_COMMIT callback commits the Iceberg transaction, and releases it at COMMIT. Uploads overlap on one node and across nodes, an open transaction blocks no other writer, and only the short commit POST is serialized. A new data file conflicts with nothing committed in between. | +| async (both GUCs `on`) | appends (`INSERT`, `COPY`, the archiver's exports, the tiered `INSERT`'s cold sink) on the patched build (`iceberg-bakery-aware-commit-refresh-v15.patch`) | The statement uploads its parquet and queues the claim; the extension's transaction callback takes it at PRE_COMMIT, just before pg_duckdb's own PRE_COMMIT callback commits the Iceberg transaction, and releases it at COMMIT. Uploads overlap on one node and across nodes, an open transaction blocks no other writer, and only the short commit POST is serialized. A new data file conflicts with nothing committed in between. An append to a clustered table is checked at commit against the live centroid generation and fails with `serialization_failure` if a retrain replaced it (see [architecture_vectors.md](architecture_vectors.md)). | The code path and the application-visible behavior are identical, so the GUCs are purely a performance knob. The patch refreshes the table from the catalog diff --git a/docs/architecture_vectors.md b/docs/architecture_vectors.md index f93ad6f..3667845 100644 --- a/docs/architecture_vectors.md +++ b/docs/architecture_vectors.md @@ -160,8 +160,7 @@ expression that yields the vector as DuckDB sees it, it returns: (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, )) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = … AND c.table_name = … AND c.column_name = … - AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc - WHERE …)) + AND c.generation = getvariable('cf_gen_')) ``` A row whose cluster disagrees with its vector is invisible to its own search @@ -180,19 +179,28 @@ with the DSN from the `coldfront.local_pg_dsn` GUC. As a consequence, the PostgreSQL table stays the only copy of the centroids, no path inlines a centroid set, and no path keeps a session copy it has no way to check. -**The generation is resolved by the emitted SQL, not baked into it.** A -statement generated once, such as a trigger body, keeps assigning against the -live generation after a retrain instead of filtering on one that no longer -exists. - -Before any training the config has no generation, the inner query matches -nothing, and the expression yields `NULL`. Unassigned is a legitimate value: +**The generation is a DuckDB session variable the write sets at execution,** +from PostgreSQL's own read of the config (`coldfront._pin_claim_generation`, +run by `coldfront._exec_iceberg_with_claim` before the statement). A statement +generated once, such as a trigger body, keeps assigning against the live +generation after a retrain, and the assignment uses the generation PostgreSQL +chose whatever the second session's snapshot shows: a generation's centroids +never change once trained, and a session that cannot see them yet leaves the +row unassigned. + +Before any training the variable is `0` or unset, no centroid matches, and the +expression yields `NULL`. Unassigned is a legitimate value: rows a foreign engine appended straight to Iceberg have none either, and the read path handles them explicitly. -A retrain cannot interleave with a cold write, because an operation that -rewrites the table holds the table's claim and every cold write serializes on -that same claim. +A write that takes the claim before it assigns cannot interleave with a retrain, +because the retrain holds the table's claim from its sample to its commit. An +append in the async ordering pins the generation when it stages and takes the +claim at commit; there, under the claim, the commit callback compares the +pinned generation with the live one and fails the transaction with +`serialization_failure` (SQLSTATE 40001) if a retrain committed in between, so +no row lands with a cluster of a replaced generation and the client retries, as +under SERIALIZABLE. ### The Six Paths diff --git a/docs/changelog.md b/docs/changelog.md index 01bd20f..4245a03 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -101,6 +101,11 @@ this project adheres to `UPDATE` or `MERGE`, the cross-tier move and `coldfront.vector_train` keep the claim-first ordering on every build, since their position deletes name the data files a concurrent compaction could rewrite. +- An append to a clustered vector table in the async ordering is checked at + commit against the live centroid generation: a retrain that committed while + its rows were being assigned fails the transaction with + `serialization_failure`, for the client to retry, so no row lands with a + cluster of a replaced generation. - The mesh bakery no longer needs the `dblink` extension. Claims, acks, releases and orphan reaping run over a libpq loopback connection that the extension opens from `coldfront.loopback_dsn`, which must name a unix socket. diff --git a/docs/usage_vectors.md b/docs/usage_vectors.md index 52e06fc..df63b51 100644 --- a/docs/usage_vectors.md +++ b/docs/usage_vectors.md @@ -171,7 +171,10 @@ Call `vector_train` again when the data has grown or you want a different keep their identities, so only the rows whose nearest centroid changed are rewritten; with a new `nlist`, nearly every row is. Either way the centroids and the assignments change together, so a search is never wrong in between; it -is only slower until the next compaction. +is only slower until the next compaction. A cold write that overlaps the +retrain, with its rows assigned before the retrain commits and its own commit +after, fails with `serialization_failure` (SQLSTATE 40001) and nothing of it +lands; retry it, as a client retries under SERIALIZABLE. ### Time Partitioning diff --git a/extension/coldfront/Makefile b/extension/coldfront/Makefile index 467857a..c072d2b 100644 --- a/extension/coldfront/Makefile +++ b/extension/coldfront/Makefile @@ -30,7 +30,7 @@ REGRESS = load_order update_unregistered_view update_heap_table \ cold_write_json_agg duckdb_temp_dir \ vector_centroids vector_type_map vector_cold_render array_cold_render vector_param_render vector_ops vector_probe vector_status vector_multicolumn \ duckdb_keyword_column partition_clause failed_txn_block tiered_insert_single_pass tiered_view_no_trigger copy_tiered_view param_in_subquery merge_on_view delete_without_where update_from_without_alias settings_registered ensure_replicated read_inner_product \ - async_write_before_claim tiered_insert_stream cold_render_canonical tiered_insert_isolation + async_write_before_claim tiered_insert_stream cold_render_canonical tiered_insert_isolation clustered_append_generation REGRESS_OPTS = --inputdir=test --outputdir=test PG_CONFIG ?= pg_config diff --git a/extension/coldfront/coldfront--1.0.sql b/extension/coldfront/coldfront--1.0.sql index a201fe7..a4fe949 100644 --- a/extension/coldfront/coldfront--1.0.sql +++ b/extension/coldfront/coldfront--1.0.sql @@ -514,7 +514,7 @@ BEGIN -- coldfront schema + registry read + the dual-write anchor table. EXECUTE format('GRANT USAGE ON SCHEMA coldfront TO %s', tgt); - EXECUTE format('GRANT SELECT ON coldfront.tiered_views, coldfront.archive_watermark TO %s', tgt); + EXECUTE format('GRANT SELECT ON coldfront.tiered_views, coldfront.archive_watermark, coldfront.vector_config TO %s', tgt); EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON coldfront._dummy_dml_target TO %s', tgt); -- EXECUTE on the runtime cold-path functions only (allow-list mirrors coldfront.c). @@ -541,6 +541,9 @@ BEGIN -- cold rows via _move_row_literal. '_cross_tier_move', '_move_row_literal', '_move_pg_row_literal', '_enqueue_release', '_enqueue_claim', + -- a clustered write's generation, pinned at staging and checked at commit + '_pin_claim_generation', '_verify_claim_generation', '_generation_fingerprint', + '_vec_generation_var', -- cold-write preconditions + serializer gate wrappers (the -- standby refusal, the armed-predicate, the Spock-without- -- bakery refusal, the claim-or-advisory acquire); app-role @@ -2340,19 +2343,24 @@ $$; -- -- Two properties are deliberate. The centroids are read over pglocal, so the -- PostgreSQL table is the only copy and no path has to inline a centroid set or --- keep a session copy it has no way to check. And the generation is resolved by --- the emitted SQL rather than baked into it, so a statement generated once (a --- trigger body) keeps assigning against the live generation after a retrain --- instead of filtering on a generation that no longer exists. +-- keep a session copy it has no way to check. And the generation is a DuckDB +-- session variable (_vec_generation_var) that the write sets at execution from +-- PostgreSQL's own read of the config (_pin_claim_generation, in +-- _exec_iceberg_with_claim), so a statement generated once (a trigger body) +-- keeps assigning against the live generation after a retrain, and the +-- assignment uses the generation PostgreSQL chose whatever pglocal's snapshot +-- shows: a generation's centroids never change once trained, and a session that +-- cannot see them yet leaves the row unassigned. -- --- Before any training the config carries no generation, the inner query matches --- nothing, and the expression yields NULL: unassigned, which a probe reads --- through the null arm of its predicate rather than missing. A retrain cannot --- interleave with a cold write, since vector_train holds the table's claim from --- its sample to its commit and every cold write serialises on that same claim; --- an append staged in the async ordering assigns against the generation live --- when it stages, so one staged during a retrain lands with the generation the --- retrain replaced, and the next retrain reassigns it. +-- Before any training the variable is 0 or unset, no centroid matches, and the +-- expression yields NULL: unassigned, which a probe reads through the null arm +-- of its predicate rather than missing. A write that claims before it assigns +-- cannot interleave with a retrain, since vector_train holds the table's claim +-- from its sample to its commit; an append staged in the async ordering claims +-- at commit, where the callback compares the generation it pinned with the live +-- one and fails the transaction if a retrain replaced it +-- (_verify_claim_generation), so no row lands with a cluster of a replaced +-- generation. -- -- The formula itself is _vec_nearest_expr, shared with vector_train, which scores -- the generation it has just trained from its own session: one formula, so a @@ -2364,6 +2372,15 @@ LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ p_vec_expr, p_from); $$; +-- The DuckDB session variable that holds a column's centroid generation for +-- the assignment, a plain identifier whatever the names hold. +CREATE OR REPLACE FUNCTION coldfront._vec_generation_var( + p_schema text, p_table text, p_column text) +RETURNS text +LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT 'cf_gen_' || md5(p_schema || '.' || p_table || '.' || p_column); +$$; + CREATE OR REPLACE FUNCTION coldfront._vec_list_expr( p_schema text, p_table text, p_column text, p_vec_expr text) RETURNS text @@ -2371,10 +2388,9 @@ LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ SELECT coldfront._vec_nearest_expr(format( 'pglocal.coldfront.vector_centroids c ' 'WHERE c.schema_name = %L AND c.table_name = %L AND c.column_name = %L ' - 'AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc ' - 'WHERE vc.schema_name = %L AND vc.table_name = %L ' - 'AND vc.column_name = %L)', - p_schema, p_table, p_column, p_schema, p_table, p_column), p_vec_expr); + 'AND c.generation = getvariable(%L)', + p_schema, p_table, p_column, + coldfront._vec_generation_var(p_schema, p_table, p_column)), p_vec_expr); $$; -- The cluster columns for an Iceberg ref, quoted and comma-joined in schema order. @@ -4229,6 +4245,81 @@ CREATE FUNCTION coldfront._enqueue_claim(p_iceberg_table text) RETURNS void LANGUAGE c AS 'coldfront', 'coldfront_enqueue_claim'; +-- coldfront._generation_fingerprint: the trained generations of a table's +-- vector columns as one text, "=" per column, NULL before +-- any training. +CREATE FUNCTION coldfront._generation_fingerprint(p_schema text, p_table text) +RETURNS text +LANGUAGE sql STABLE AS $$ + SELECT string_agg(column_name || '=' || generation, ',' ORDER BY column_name) + FROM coldfront.vector_config + WHERE schema_name = p_schema AND table_name = p_table AND generation > 0; +$$; + +-- coldfront._pin_claim_generation: before a cold write to a clustered table +-- stages, the generation of each vector column, read here from the config, is +-- set as the DuckDB session variable the assignment filters the centroids by +-- (_vec_list_expr), so the rows are assigned against the generation this +-- session sees. Returns the table's fingerprint, "=" per +-- trained column, NULL for an untrained or unclustered table. +CREATE FUNCTION coldfront._pin_claim_generation(p_iceberg_table text) +RETURNS text +LANGUAGE plpgsql AS $$ +DECLARE + v_schema text; + v_table text; + v_fp text; + r record; +BEGIN + SELECT tv.schema_name, tv.relname INTO v_schema, v_table + FROM coldfront.tiered_views tv + WHERE tv.iceberg_table = p_iceberg_table AND tv.vec_columns IS NOT NULL; + IF v_table IS NULL THEN + RETURN NULL; + END IF; + FOR r IN SELECT column_name, generation + FROM coldfront.vector_config + WHERE schema_name = v_schema AND table_name = v_table + ORDER BY column_name + LOOP + PERFORM duckdb.raw_query(format('SET VARIABLE %s = %s', + coldfront._vec_generation_var(v_schema, v_table, r.column_name), r.generation)); + IF r.generation > 0 THEN + v_fp := concat_ws(',', v_fp, r.column_name || '=' || r.generation); + END IF; + END LOOP; + RETURN v_fp; +END; +$$; + +-- coldfront._verify_claim_generation: the commit callback's check, under the +-- claim and at the latest snapshot, that the generation an async append pinned +-- (coldfront._claim_generations, " " per line) is the live +-- one. Where a retrain replaced it, the transaction fails with +-- serialization_failure, for the client to retry, and its ABORT releases the +-- claims. +CREATE FUNCTION coldfront._verify_claim_generation(p_iceberg_table text) +RETURNS void +LANGUAGE plpgsql STABLE AS $$ +DECLARE + v_schema text; + v_table text; +BEGIN + SELECT tv.schema_name, tv.relname INTO v_schema, v_table + FROM coldfront.tiered_views tv + WHERE tv.iceberg_table = p_iceberg_table; + IF EXISTS (SELECT 1 + FROM unnest(string_to_array(current_setting('coldfront._claim_generations'), E'\n')) AS l + WHERE split_part(l, ' ', 1) = p_iceberg_table + AND split_part(l, ' ', 2) + IS DISTINCT FROM coldfront._generation_fingerprint(v_schema, v_table)) THEN + RAISE EXCEPTION 'coldfront: a retrain of %.% committed while this transaction''s rows were being assigned to its clusters', + quote_ident(v_schema), quote_ident(v_table) + USING ERRCODE = 'serialization_failure', HINT = 'Retry the transaction.'; + END IF; +END; +$$; + -- coldfront._reject_on_standby: refuse a cold-tier mutation on a physical standby. -- PostgreSQL's read-only enforcement covers only its own writes; a cold write leaves -- PG entirely (DuckDB to object storage), so the cold tier relies on this explicit @@ -4344,7 +4435,10 @@ $$; -- claim. The patch re-stamps parent_snapshot_id at the POST, under the -- claim, and a new data file conflicts with nothing a peer or a compaction -- commits in between (docs/formal Bakery_async.cfg, --- Bakery_async_samenode.cfg and Bakery_async_single.cfg). +-- Bakery_async_samenode.cfg and Bakery_async_single.cfg). An append to a +-- clustered table pins the centroid generation its assignment uses +-- (_pin_claim_generation); the callback checks that generation under the +-- claim and fails the transaction where a retrain replaced it. -- * A write that references rows already in the table (DELETE, UPDATE, MERGE, -- the cross-tier move's bundle) takes the claim FIRST, whatever the build: -- its position deletes name data files by path, and a compaction or another @@ -4371,7 +4465,13 @@ DECLARE -- Every statement here is generated by coldfront, and an append starts -- with its verb; a DELETE-led bundle or an UPDATE does not. v_append boolean := upper(ltrim(p_sql)) LIKE 'INSERT %'; + v_pinned text; BEGIN + -- A cluster assignment filters the centroids by the generation this session + -- sets before the statement runs (_pin_claim_generation). + IF strpos(p_sql, 'getvariable(''cf_gen_') > 0 THEN + v_pinned := coldfront._pin_claim_generation(p_iceberg_table); + END IF; -- A cluster assignment reads the centroids over pglocal, so a statement that -- names it needs pglocal attached in this backend. Tested on the statement -- rather than the table: this wrapper is the one chokepoint every C-generated @@ -4398,9 +4498,17 @@ BEGIN PERFORM set_config('coldfront._async_downgrade_warned', 'on', false); END IF; IF v_async AND v_append THEN - -- Patched iceberg: stage the parquet now, claim at PRE_COMMIT. + -- Patched iceberg: stage the parquet now, claim at PRE_COMMIT, where a + -- clustered table's pinned generation is checked against the live one + -- (_verify_claim_generation). PERFORM coldfront._refuse_spock_without_bakery(); PERFORM duckdb.raw_query(p_sql); + IF v_pinned IS NOT NULL THEN + PERFORM set_config('coldfront._claim_generations', + concat_ws(E'\n', NULLIF(current_setting('coldfront._claim_generations'), ''), + p_iceberg_table || ' ' || v_pinned), + true); + END IF; PERFORM coldfront._enqueue_claim(p_iceberg_table); ELSE -- Acquire FIRST, then stage+commit inside the claim. diff --git a/extension/coldfront/src/coldfront.c b/extension/coldfront/src/coldfront.c index 7e427c1..a3b8829 100644 --- a/extension/coldfront/src/coldfront.c +++ b/extension/coldfront/src/coldfront.c @@ -270,6 +270,7 @@ static bool coldfront_iceberg_async_parquet = false; static bool coldfront_iceberg_bakery_patch = false; static char *coldfront_claimed = NULL; static char *coldfront_stream_state = NULL; +static char *coldfront_claim_generations = NULL; static bool coldfront_async_downgrade_warned = false; static post_parse_analyze_hook_type prev_post_parse_analyze_hook = NULL; @@ -4811,7 +4812,9 @@ cf_statement_timeout_applies(void) * at PRE_COMMIT: the transaction is still open, so SPI runs here under a * snapshot pushed for it (no statement is active at this point), and the ERROR * a timeout or a refused claim raises aborts the transaction, whose ABORT - * event then drops the queue. PostgreSQL switches statement_timeout off before + * event then drops the queue. With the claims held, a clustered table's append + * is checked against the live centroid generation (_verify_claim_generation). + * PostgreSQL switches statement_timeout off before * commit processing (finish_xact_command), so the wait for the claim, whose * protocol has no timeout of its own, would be the one part of the statement * the setting does not bound: the timer runs again, at the statement's own @@ -4839,6 +4842,17 @@ cf_take_pending_claims(void) 1, argtypes, values, NULL, false, 0) != SPI_OK_SELECT) elog(ERROR, "coldfront: taking the claim on %s failed", (char *) lfirst(lc)); } + PopActiveSnapshot(); + PushActiveSnapshot(GetLatestSnapshot()); + foreach(lc, coldfront_pending_claims) + { + Oid argtypes[1] = { TEXTOID }; + Datum values[1] = { CStringGetTextDatum((char *) lfirst(lc)) }; + + if (SPI_execute_with_args("SELECT coldfront._verify_claim_generation($1)", + 1, argtypes, values, NULL, true, 0) != SPI_OK_SELECT) + elog(ERROR, "coldfront: checking the centroid generation of %s failed", (char *) lfirst(lc)); + } SPI_finish(); PopActiveSnapshot(); if (bounded) @@ -6206,6 +6220,16 @@ register_gucs(void) 0, NULL, NULL, NULL); + DefineCustomStringVariable( + "coldfront._claim_generations", + "Per clustered table this transaction appended to in the async ordering, the centroid generation its rows were assigned against, one per line.", + NULL, + &coldfront_claim_generations, + "", + PGC_USERSET, + 0, + NULL, NULL, NULL); + DefineCustomStringVariable( "coldfront._stream_state", "The snapshot the tiered stream's second session reads at in this transaction: none yet, one of its own (used), or this transaction's (imported).", diff --git a/extension/coldfront/test/expected/async_write_before_claim.out b/extension/coldfront/test/expected/async_write_before_claim.out index ab7cc7e..09d9aa4 100644 --- a/extension/coldfront/test/expected/async_write_before_claim.out +++ b/extension/coldfront/test/expected/async_write_before_claim.out @@ -74,7 +74,7 @@ ERROR: canceling statement due to lock timeout CONTEXT: SQL statement "SELECT pg_advisory_xact_lock(hashtext('coldfront_iceberg:' || p_iceberg_ref))" PL/pgSQL function _take_iceberg_claim(text) line 13 at PERFORM SQL statement "SELECT coldfront._take_iceberg_claim(p_iceberg_table)" -PL/pgSQL function _exec_iceberg_with_claim(text,text) line 40 at PERFORM +PL/pgSQL function _exec_iceberg_with_claim(text,text) line 54 at PERFORM -- (3) The stock ordering claims first for an append too. SET coldfront.iceberg_async_parquet = off; SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_order', @@ -83,7 +83,7 @@ ERROR: canceling statement due to lock timeout CONTEXT: SQL statement "SELECT pg_advisory_xact_lock(hashtext('coldfront_iceberg:' || p_iceberg_ref))" PL/pgSQL function _take_iceberg_claim(text) line 13 at PERFORM SQL statement "SELECT coldfront._take_iceberg_claim(p_iceberg_table)" -PL/pgSQL function _exec_iceberg_with_claim(text,text) line 40 at PERFORM +PL/pgSQL function _exec_iceberg_with_claim(text,text) line 54 at PERFORM SET coldfront.iceberg_async_parquet = on; -- (4) The cold sink: a tiered INSERT's cold rows, one row per batch, are all -- staged in the statement; the claim at COMMIT meets the lock. diff --git a/extension/coldfront/test/expected/clustered_append_generation.out b/extension/coldfront/test/expected/clustered_append_generation.out new file mode 100644 index 0000000..94bd7b0 --- /dev/null +++ b/extension/coldfront/test/expected/clustered_append_generation.out @@ -0,0 +1,173 @@ +-- An append to a clustered table in the async ordering assigns its rows to the +-- centroid generation live when it stages and takes the claim at commit. A +-- retrain that commits in between replaces that generation, so the commit +-- callback, under the claim, compares the generation the assignment used with +-- the live one and fails the transaction with serialization_failure where they +-- differ, for the client to retry; a commit that meets no retrain lands. +-- Observed end to end: a DuckDB temporary table stands in for the Iceberg +-- table, the registry holds a clustered table of one generation, and the +-- retrain is the bakery's loopback, pointed at this server, bumping it. Each +-- staged INSERT stores the generation it is assigned against, read as the +-- assignment reads it, from the session variable the write pins. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; + dsn_set +--------- + t +(1 row) + +SELECT set_config('coldfront.loopback_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS loopback_set; + loopback_set +-------------- + t +(1 row) + +SET coldfront.iceberg_async_parquet = on; +SET coldfront.iceberg_bakery_patch = on; +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_gen (id INTEGER, v FLOAT[], cf_v_list INTEGER)') IS NOT NULL AS created; +NOTICE: result: Count +BIGINT +[ Rows: 0] + + + created +--------- + t +(1 row) + +CREATE TABLE public._cf_gen (id int, v real[]); +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col, vec_columns) +VALUES ('public', 'cf_gen', 'public._cf_gen', 'temp.main.cf_gen', 'id', '{v}'); +INSERT INTO coldfront.vector_config(schema_name, table_name, column_name, nlist, nprobe, generation) +VALUES ('public', 'cf_gen', 'v', 2, 1, 1); +-- (1) No retrain between staging and commit: the append lands. +BEGIN; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_gen', + format('INSERT INTO temp.main.cf_gen SELECT 1, [0.1, 0.2], getvariable(%L)', coldfront._vec_generation_var('public', 'cf_gen', 'v'))); +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + _exec_iceberg_with_claim +-------------------------- + +(1 row) + +COMMIT; +-- (2) A retrain commits in between: the commit fails and nothing lands. +BEGIN; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_gen', + format('INSERT INTO temp.main.cf_gen SELECT 2, [0.3, 0.4], getvariable(%L)', coldfront._vec_generation_var('public', 'cf_gen', 'v'))); +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + _exec_iceberg_with_claim +-------------------------- + +(1 row) + +SELECT coldfront._loopback('UPDATE coldfront.vector_config SET generation = 2 WHERE table_name = ''cf_gen'' RETURNING generation') AS retrained_elsewhere; + retrained_elsewhere +--------------------- + 2 +(1 row) + +COMMIT; +ERROR: coldfront: a retrain of public.cf_gen committed while this transaction's rows were being assigned to its clusters +HINT: Retry the transaction. +CONTEXT: PL/pgSQL function _verify_claim_generation(text) line 14 at RAISE +SQL statement "SELECT coldfront._verify_claim_generation($1)" +-- (3) The check reads the live generation under REPEATABLE READ too, whose +-- statements would see the one the transaction started with. +BEGIN ISOLATION LEVEL REPEATABLE READ; +SELECT generation FROM coldfront.vector_config WHERE table_name = 'cf_gen'; + generation +------------ + 2 +(1 row) + +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_gen', + format('INSERT INTO temp.main.cf_gen SELECT 3, [0.5, 0.6], getvariable(%L)', coldfront._vec_generation_var('public', 'cf_gen', 'v'))); +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + _exec_iceberg_with_claim +-------------------------- + +(1 row) + +SELECT coldfront._loopback('UPDATE coldfront.vector_config SET generation = 3 WHERE table_name = ''cf_gen'' RETURNING generation') AS retrained_elsewhere; + retrained_elsewhere +--------------------- + 3 +(1 row) + +COMMIT; +ERROR: coldfront: a retrain of public.cf_gen committed while this transaction's rows were being assigned to its clusters +HINT: Retry the transaction. +CONTEXT: PL/pgSQL function _verify_claim_generation(text) line 14 at RAISE +SQL statement "SELECT coldfront._verify_claim_generation($1)" +-- (4) The next attempt, assigned against the live generation, lands. +BEGIN; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_gen', + format('INSERT INTO temp.main.cf_gen SELECT 4, [0.7, 0.8], getvariable(%L)', coldfront._vec_generation_var('public', 'cf_gen', 'v'))); +NOTICE: result: Success +BOOLEAN +[ Rows: 0] + + +NOTICE: result: Count +BIGINT +[ Rows: 1] +1 + + + _exec_iceberg_with_claim +-------------------------- + +(1 row) + +COMMIT; +SELECT r['id']::int AS id, r['cf_v_list']::int AS generation_used + FROM duckdb.query('SELECT id, cf_v_list FROM temp.main.cf_gen') AS t(r) ORDER BY 1; + id | generation_used +----+----------------- + 1 | 1 + 4 | 3 +(2 rows) + diff --git a/extension/coldfront/test/expected/cte_on_insert.out b/extension/coldfront/test/expected/cte_on_insert.out index cce4df7..edc2de7 100644 --- a/extension/coldfront/test/expected/cte_on_insert.out +++ b/extension/coldfront/test/expected/cte_on_insert.out @@ -238,10 +238,10 @@ VALUES ('public', 'icevec', 'ice.default.icevec', true, ARRAY['embedding']); EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT 7 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, ARRAY[1,0,0]::real[] AS embedding) INSERT INTO public.icevec SELECT id, ts, embedding FROM s; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: _exec_iceberg_with_claim('ice.default.icevec'::text, 'INSERT INTO ice.default.icevec (_cf_vec_list_embedding, id, ts, embedding) SELECT (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, coldfront_src.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = ''public'' AND c.table_name = ''icevec'' AND c.column_name = ''embedding'' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = ''public'' AND vc.table_name = ''icevec'' AND vc.column_name = ''embedding'')), id, ts, embedding FROM (WITH s AS ( SELECT 7 AS id, ''Fri May 01 00:00:00 2026 UTC''::timestamptz AS ts, ARRAY[(1)::real, (0)::real, (0)::real] AS embedding ) SELECT s.id, s.ts, s.embedding FROM s) AS coldfront_src(id, ts, embedding) ORDER BY 1'::text) + Output: _exec_iceberg_with_claim('ice.default.icevec'::text, 'INSERT INTO ice.default.icevec (_cf_vec_list_embedding, id, ts, embedding) SELECT (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, coldfront_src.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = ''public'' AND c.table_name = ''icevec'' AND c.column_name = ''embedding'' AND c.generation = getvariable(''cf_gen_b1aee26bff21bd9e8c268e03c2e0c089'')), id, ts, embedding FROM (WITH s AS ( SELECT 7 AS id, ''Fri May 01 00:00:00 2026 UTC''::timestamptz AS ts, ARRAY[(1)::real, (0)::real, (0)::real] AS embedding ) SELECT s.id, s.ts, s.embedding FROM s) AS coldfront_src(id, ts, embedding) ORDER BY 1'::text) (2 rows) -- Cleanup. diff --git a/extension/coldfront/test/expected/cte_on_insert_1.out b/extension/coldfront/test/expected/cte_on_insert_1.out index 1050f62..6eec8af 100644 --- a/extension/coldfront/test/expected/cte_on_insert_1.out +++ b/extension/coldfront/test/expected/cte_on_insert_1.out @@ -237,10 +237,10 @@ VALUES ('public', 'icevec', 'ice.default.icevec', true, ARRAY['embedding']); EXPLAIN (COSTS OFF, VERBOSE) WITH s AS (SELECT 7 AS id, '2026-05-01 00:00:00+00'::timestamptz AS ts, ARRAY[1,0,0]::real[] AS embedding) INSERT INTO public.icevec SELECT id, ts, embedding FROM s; - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Result - Output: _exec_iceberg_with_claim('ice.default.icevec'::text, 'INSERT INTO ice.default.icevec (_cf_vec_list_embedding, id, ts, embedding) SELECT (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, coldfront_src.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = ''public'' AND c.table_name = ''icevec'' AND c.column_name = ''embedding'' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = ''public'' AND vc.table_name = ''icevec'' AND vc.column_name = ''embedding'')), id, ts, embedding FROM (WITH s AS ( SELECT 7 AS id, ''Fri May 01 00:00:00 2026 UTC''::timestamptz AS ts, ARRAY[(1)::real, (0)::real, (0)::real] AS embedding ) SELECT s.id, s.ts, s.embedding FROM s) AS coldfront_src(id, ts, embedding) ORDER BY 1'::text) + Output: _exec_iceberg_with_claim('ice.default.icevec'::text, 'INSERT INTO ice.default.icevec (_cf_vec_list_embedding, id, ts, embedding) SELECT (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, coldfront_src.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = ''public'' AND c.table_name = ''icevec'' AND c.column_name = ''embedding'' AND c.generation = getvariable(''cf_gen_b1aee26bff21bd9e8c268e03c2e0c089'')), id, ts, embedding FROM (WITH s AS ( SELECT 7 AS id, ''Fri May 01 00:00:00 2026 UTC''::timestamptz AS ts, ARRAY[(1)::real, (0)::real, (0)::real] AS embedding ) SELECT s.id, s.ts, s.embedding FROM s) AS coldfront_src(id, ts, embedding) ORDER BY 1'::text) (2 rows) -- Cleanup. diff --git a/extension/coldfront/test/expected/settings_registered.out b/extension/coldfront/test/expected/settings_registered.out index a9b7185..3add0eb 100644 --- a/extension/coldfront/test/expected/settings_registered.out +++ b/extension/coldfront/test/expected/settings_registered.out @@ -11,6 +11,7 @@ SELECT name, vartype, context, boot_val, min_val name | vartype | context | boot_val | min_val -----------------------------------+---------+-----------+----------+--------- coldfront._async_downgrade_warned | bool | user | off | + coldfront._claim_generations | string | user | | coldfront._claimed | string | user | | coldfront._stream_state | string | user | | coldfront.allow_mixed_writes | bool | user | on | @@ -24,7 +25,7 @@ SELECT name, vartype, context, boot_val, min_val coldfront.vector_nprobe | integer | user | 0 | 0 coldfront.vector_probe | bool | user | on | coldfront.warehouse | string | superuser | | -(14 rows) +(15 rows) -- The bakery's dead-peer window is superuser-only and at least 1 ms. CREATE ROLE cf_settings_app; diff --git a/extension/coldfront/test/expected/vector_cold_render.out b/extension/coldfront/test/expected/vector_cold_render.out index 71b491b..95ecee2 100644 --- a/extension/coldfront/test/expected/vector_cold_render.out +++ b/extension/coldfront/test/expected/vector_cold_render.out @@ -38,9 +38,9 @@ SELECT coldfront._move_row_literal( ARRAY['id','ts','embedding'], ARRAY['bigint','timestamp with time zone','vector(3)'], 'ts', 'public', 'chunks') AS vec_row; - vec_row -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, CAST('[1,2,3]' AS FLOAT[]))) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'chunks' AND c.column_name = 'embedding' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'chunks' AND vc.column_name = 'embedding')), '8', '2026-06-01 00:00:00+00', CAST('[1,2,3]' AS FLOAT[]) + vec_row +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, CAST('[1,2,3]' AS FLOAT[]))) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'chunks' AND c.column_name = 'embedding' AND c.generation = getvariable('cf_gen_87c15a8ce2c1f39338e9f1e80a32b7b6')), '8', '2026-06-01 00:00:00+00', CAST('[1,2,3]' AS FLOAT[]) (1 row) SELECT coldfront._move_row_literal( @@ -48,8 +48,8 @@ SELECT coldfront._move_row_literal( ARRAY['id','ts','embedding'], ARRAY['bigint','timestamp with time zone','vector(3)'], 'ts', 'public', 'chunks') AS null_vec_row; - null_vec_row ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ - (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, NULL)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'chunks' AND c.column_name = 'embedding' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'chunks' AND vc.column_name = 'embedding')), '7', '2026-06-01 00:00:00+00', NULL + null_vec_row +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, NULL)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'chunks' AND c.column_name = 'embedding' AND c.generation = getvariable('cf_gen_87c15a8ce2c1f39338e9f1e80a32b7b6')), '7', '2026-06-01 00:00:00+00', NULL (1 row) diff --git a/extension/coldfront/test/expected/vector_multicolumn.out b/extension/coldfront/test/expected/vector_multicolumn.out index 95bbea6..32d4869 100644 --- a/extension/coldfront/test/expected/vector_multicolumn.out +++ b/extension/coldfront/test/expected/vector_multicolumn.out @@ -42,17 +42,17 @@ SELECT coldfront._vec_sort_key('embedding', ARRAY['id','ts']) AS sort_key; SELECT coldfront._vec_list_prefix('public', 'docs', ARRAY['embedding','summary'], ARRAY['s.embedding','s.summary']) AS prefix; - prefix ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, s.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'embedding' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'docs' AND vc.column_name = 'embedding')), (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, s.summary)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'summary' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'docs' AND vc.column_name = 'summary')), + prefix +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, s.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'embedding' AND c.generation = getvariable('cf_gen_79c2d62beeebe93ab46a505ad9717b3d')), (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, s.summary)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'summary' AND c.generation = getvariable('cf_gen_e4410281f87a36742e678dde63ab1249')), (1 row) -- A single vector column yields a prefix of one expression. SELECT coldfront._vec_list_prefix('public', 'docs', ARRAY['embedding'], ARRAY['s.embedding']) AS one_column; - one_column ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, s.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'embedding' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'docs' AND vc.column_name = 'embedding')), + one_column +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, s.embedding)) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'embedding' AND c.generation = getvariable('cf_gen_79c2d62beeebe93ab46a505ad9717b3d')), (1 row) -- No vector column, no prefix: a table without vectors pays nothing. @@ -152,9 +152,9 @@ SELECT coldfront._cold_row_literal( ARRAY['id', 'ts', 'body', 'body_len', 'embedding', 'summary'], ARRAY['bigint', 'timestamp with time zone', 'text', 'integer', 'vector(3)', 'vector(2)'], 'public', 'docs') AS two_vector_row; - two_vector_row ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, CAST('[0.9,0.1,0]' AS FLOAT[]))) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'embedding' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'docs' AND vc.column_name = 'embedding')), (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, CAST('[-1,1]' AS FLOAT[]))) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'summary' AND c.generation = (SELECT vc.generation FROM pglocal.coldfront.vector_config vc WHERE vc.schema_name = 'public' AND vc.table_name = 'docs' AND vc.column_name = 'summary')), NULL, '2026-06-01 00:00:00+00', 'b', '1', CAST('[0.9,0.1,0]' AS FLOAT[]), CAST('[-1,1]' AS FLOAT[]) + two_vector_row +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, CAST('[0.9,0.1,0]' AS FLOAT[]))) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'embedding' AND c.generation = getvariable('cf_gen_79c2d62beeebe93ab46a505ad9717b3d')), (SELECT arg_min(c.centroid_id, list_cosine_distance(c.centroid, CAST('[-1,1]' AS FLOAT[]))) FROM pglocal.coldfront.vector_centroids c WHERE c.schema_name = 'public' AND c.table_name = 'docs' AND c.column_name = 'summary' AND c.generation = getvariable('cf_gen_e4410281f87a36742e678dde63ab1249')), NULL, '2026-06-01 00:00:00+00', 'b', '1', CAST('[0.9,0.1,0]' AS FLOAT[]), CAST('[-1,1]' AS FLOAT[]) (1 row) -- Cleanup. Unregister before dropping: the DDL hook blocks DROP of a registered diff --git a/extension/coldfront/test/sql/clustered_append_generation.sql b/extension/coldfront/test/sql/clustered_append_generation.sql new file mode 100644 index 0000000..6b1553d --- /dev/null +++ b/extension/coldfront/test/sql/clustered_append_generation.sql @@ -0,0 +1,66 @@ +-- An append to a clustered table in the async ordering assigns its rows to the +-- centroid generation live when it stages and takes the claim at commit. A +-- retrain that commits in between replaces that generation, so the commit +-- callback, under the claim, compares the generation the assignment used with +-- the live one and fails the transaction with serialization_failure where they +-- differ, for the client to retry; a commit that meets no retrain lands. +-- Observed end to end: a DuckDB temporary table stands in for the Iceberg +-- table, the registry holds a clustered table of one generation, and the +-- retrain is the bakery's loopback, pointed at this server, bumping it. Each +-- staged INSERT stores the generation it is assigned against, read as the +-- assignment reads it, from the session variable the write pins. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS pg_duckdb; +CREATE EXTENSION IF NOT EXISTS coldfront; +RESET client_min_messages; + +SET TIME ZONE 'UTC'; +SET coldfront.warehouse = ''; +SET coldfront.lakekeeper_endpoint = ''; +SELECT set_config('coldfront.local_pg_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS dsn_set; +SELECT set_config('coldfront.loopback_dsn', + format('host=%s dbname=%s user=%s', + trim(split_part(current_setting('unix_socket_directories'), ',', 1)), + current_database(), current_user), false) <> '' AS loopback_set; +SET coldfront.iceberg_async_parquet = on; +SET coldfront.iceberg_bakery_patch = on; +SELECT duckdb.raw_query('CREATE TEMP TABLE cf_gen (id INTEGER, v FLOAT[], cf_v_list INTEGER)') IS NOT NULL AS created; + +CREATE TABLE public._cf_gen (id int, v real[]); +INSERT INTO coldfront.tiered_views(schema_name, relname, hot_table, iceberg_table, partition_col, vec_columns) +VALUES ('public', 'cf_gen', 'public._cf_gen', 'temp.main.cf_gen', 'id', '{v}'); +INSERT INTO coldfront.vector_config(schema_name, table_name, column_name, nlist, nprobe, generation) +VALUES ('public', 'cf_gen', 'v', 2, 1, 1); + +-- (1) No retrain between staging and commit: the append lands. +BEGIN; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_gen', + format('INSERT INTO temp.main.cf_gen SELECT 1, [0.1, 0.2], getvariable(%L)', coldfront._vec_generation_var('public', 'cf_gen', 'v'))); +COMMIT; + +-- (2) A retrain commits in between: the commit fails and nothing lands. +BEGIN; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_gen', + format('INSERT INTO temp.main.cf_gen SELECT 2, [0.3, 0.4], getvariable(%L)', coldfront._vec_generation_var('public', 'cf_gen', 'v'))); +SELECT coldfront._loopback('UPDATE coldfront.vector_config SET generation = 2 WHERE table_name = ''cf_gen'' RETURNING generation') AS retrained_elsewhere; +COMMIT; + +-- (3) The check reads the live generation under REPEATABLE READ too, whose +-- statements would see the one the transaction started with. +BEGIN ISOLATION LEVEL REPEATABLE READ; +SELECT generation FROM coldfront.vector_config WHERE table_name = 'cf_gen'; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_gen', + format('INSERT INTO temp.main.cf_gen SELECT 3, [0.5, 0.6], getvariable(%L)', coldfront._vec_generation_var('public', 'cf_gen', 'v'))); +SELECT coldfront._loopback('UPDATE coldfront.vector_config SET generation = 3 WHERE table_name = ''cf_gen'' RETURNING generation') AS retrained_elsewhere; +COMMIT; + +-- (4) The next attempt, assigned against the live generation, lands. +BEGIN; +SELECT coldfront._exec_iceberg_with_claim('temp.main.cf_gen', + format('INSERT INTO temp.main.cf_gen SELECT 4, [0.7, 0.8], getvariable(%L)', coldfront._vec_generation_var('public', 'cf_gen', 'v'))); +COMMIT; +SELECT r['id']::int AS id, r['cf_v_list']::int AS generation_used + FROM duckdb.query('SELECT id, cf_v_list FROM temp.main.cf_gen') AS t(r) ORDER BY 1; From 2ecb23476348ef1b64cec42d796e3283278bc29d Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 9 Oct 2026 16:45:38 +0100 Subject: [PATCH 8/9] test: require no orphan deleted beside an open write --- ci/journey.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/journey.sh b/ci/journey.sh index bce8e6e..9780095 100755 --- a/ci/journey.sh +++ b/ci/journey.sh @@ -2421,7 +2421,7 @@ story_compactor_claim_matches_writes() { compactor_runs_beside_write TC-183 cf_cw 2 'expired [1-9][0-9]* snapshot' \ --expire-snapshots --expire-older-than 0s --expire-retain-last 1 --expire-keep-files compactor_runs_beside_write TC-184 cf_ad 5 'compacted: [0-9]+ files' - compactor_runs_beside_write TC-185 cf_or 1 'deleted [0-9]+ orphan' --orphans + compactor_runs_beside_write TC-185 cf_or 1 'deleted 0 orphan' --orphans for t in cf_cw cf_ad cf_or; do q "$HOST" "SELECT coldfront.drop_iceberg_table('public','$t', true);" >/dev/null 2>&1 done From 14de5e0096e6481b328c4f7907b7a4eef6cbb778 Mon Sep 17 00:00:00 2001 From: Jimmy Angelakos Date: Fri, 9 Oct 2026 17:07:33 +0100 Subject: [PATCH 9/9] docs: qualify the async ordering's identical behaviour for clustered appends --- docs/architecture.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index d37880e..4e436de 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -343,8 +343,8 @@ ColdFront coordinates concurrent writes across the cluster as follows: iceberg-only `INSERT`/`UPDATE`/`DELETE`/`MERGE` wraps in `coldfront._exec_iceberg_with_claim`, which holds a globally-ordered Snowflake ticket via the Spock-replicated `coldfront.claims` table and waits - for its turn before issuing the iceberg commit. There are no 409s and no - app-level retry. See the + for its turn before issuing the iceberg commit. The bakery produces no 409s + and needs no retry in the application. See the [Concurrency](architecture_decoupled.md#concurrency-horizontal-scaling-the-bakery-protocol) section of the Decoupled Mode page for the full design and benchmarks. @@ -363,8 +363,11 @@ The following table shows the two orderings: | stock (default) | every write on stock upstream duckdb-iceberg; a `DELETE`, `UPDATE` or `MERGE` (the cross-tier move and `vector_train`'s assignment included) on every build | The writer takes the serializer first, *then* uploads parquet and commits inside it. On an unpatched binary this is what keeps a second writer from capturing a stale parent. A write that references rows the table holds needs it on every build: its position deletes name data files by path, and a compaction or another such write committing between its scan and its commit would rewrite or remove those files, after which the deletes apply to nothing and the rows come back. | | async (both GUCs `on`) | appends (`INSERT`, `COPY`, the archiver's exports, the tiered `INSERT`'s cold sink) on the patched build (`iceberg-bakery-aware-commit-refresh-v15.patch`) | The statement uploads its parquet and queues the claim; the extension's transaction callback takes it at PRE_COMMIT, just before pg_duckdb's own PRE_COMMIT callback commits the Iceberg transaction, and releases it at COMMIT. Uploads overlap on one node and across nodes, an open transaction blocks no other writer, and only the short commit POST is serialized. A new data file conflicts with nothing committed in between. An append to a clustered table is checked at commit against the live centroid generation and fails with `serialization_failure` if a retrain replaced it (see [architecture_vectors.md](architecture_vectors.md)). | -The code path and the application-visible behavior are identical, so the GUCs -are purely a performance knob. The patch refreshes the table from the catalog +For every write but a clustered table's append, the code path and the +application-visible behavior are identical, so the GUCs are a performance knob; +a clustered append in the async ordering can fail with `serialization_failure` +when a retrain overlaps it, which the claim-first ordering, waiting for the +retrain instead, cannot. The patch refreshes the table from the catalog in PG's pre-commit phase (inside the serializer) and re-derives every pending snapshot of the transaction from that head, so overlapping uploads and several batches alike cannot commit a stale parent. A staged file is referenced by no