diff --git a/.clangd b/.clangd new file mode 100644 index 0000000000000..490220f56a7c0 --- /dev/null +++ b/.clangd @@ -0,0 +1,43 @@ +Diagnostics: + MissingIncludes: None +InlayHints: + Enabled: true + ParameterNames: true + DeducedTypes: true +CompileFlags: + CompilationDatabase: build/ # Search build/ directory for compile_commands.json + Remove: [ -Werror ] + Add: + - -DDEBUG + - -DLOCAL + - -DPGDLLIMPORT= + - -DPIC + - -O2 + - -Wall + - -Wcast-function-type + - -Wconversion + - -Wdeclaration-after-statement + - -Wendif-labels + - -Werror=vla + - -Wextra + - -Wfloat-equal + - -Wformat-security + - -Wimplicit-fallthrough=3 + - -Wmissing-format-attribute + - -Wmissing-prototypes + - -Wno-format-truncation + - -Wno-sign-conversion + - -Wno-stringop-truncation + - -Wno-unused-const-variable + - -Wpointer-arith + - -Wshadow + - -Wshadow=compatible-local + - -fPIC + - -fexcess-precision=standard + - -fno-strict-aliasing + - -fvisibility=hidden + - -fwrapv + - -g + - -std=c11 + - -I. + - -I../../../../src/include diff --git a/.gdbinit b/.gdbinit new file mode 100644 index 0000000000000..854e5ecbaf69c --- /dev/null +++ b/.gdbinit @@ -0,0 +1,156 @@ +# HOT Indexed Updates — GDB breakpoints for code review +# +# Usage: gdb -x .gdbinit +# Or from gdb: source .gdbinit +# +# These breakpoints cover the major code paths introduced or modified by +# the HOT indexed updates patch series. They are organized by subsystem +# to make it easy to enable/disable groups during debugging. +# +# Tip: to skip to a specific subsystem, disable all then enable selectively: +# disable breakpoints +# enable 1 2 3 # just the update-decision group + +# ========================================================================= +# 1. UPDATE DECISION — heap_update() HOT/HOT-indexed/non-HOT choice +# src/backend/access/heap/heapam.c +# ========================================================================= + +# Main entry: heap_update +break heapam.c:3210 + +# HOT decision block: pure HOT vs HOT indexed vs non-HOT +# Line 4019: pure HOT (no indexed columns changed) +# Line 4024: HOT indexed path (non-catalog, some indexed columns changed) +# Line 4031: predict augmented tuple size +# Line 4033: size+space check before creating augmented tuple +break heapam.c:4019 +break heapam.c:4024 +break heapam.c:4033 + +# Set HEAP_INDEXED_UPDATED flag on new tuple before page insertion +break heapam.c:4101 + +# Restore HEAP_INDEXED_UPDATED on old tuple (only if it previously had it) +break heapam.c:4147 + +# ========================================================================= +# 2. TUPLE CREATION — building the augmented tuple with embedded bitmap +# src/backend/access/heap/heapam.c +# ========================================================================= + +# Predict augmented tuple size (returns 0 if t_hoff would overflow) +break heap_hot_indexed_tuple_size + +# Create augmented tuple with embedded modified-column bitmap +break heap_hot_indexed_create_tuple + +# Serialize Bitmapset into raw bytes in tuple header +break heap_hot_indexed_serialize_bitmap + +# ========================================================================= +# 3. BITMAP UTILITIES — raw bitmap operations for chain following +# src/backend/access/heap/heapam.c +# ========================================================================= + +# Compute raw bitmap byte size from natts +break heap_hot_indexed_bitmap_raw_size + +# Check if tuple header has room for bitmap between null bitmap and data +break heap_hot_indexed_has_bitmap_space + +# Read HOT indexed bitmap from tuple header (returns Bitmapset) +break heap_hot_indexed_read_bitmap + +# Fast overlap check: does tuple's raw bitmap overlap with indexed_attrs? +break heap_hot_indexed_bitmap_overlaps_raw + +# OR a tuple's raw bitmap into an accumulator buffer +break heap_hot_indexed_bitmap_or_raw + +# Check if accumulated raw bitmap overlaps with indexed_attrs +break heap_hot_indexed_accum_overlaps + +# Merge bitmaps from dead tuples into a target tuple on the page +break heap_hot_indexed_merge_bitmaps_raw + +# Deserialize raw bytes back to Bitmapset +break heap_hot_indexed_deserialize_bitmap + +# ========================================================================= +# 4. INDEX SCAN — HOT chain following with stale-entry detection +# src/backend/access/heap/heapam_indexscan.c +# ========================================================================= + +# Main HOT chain search with indexed update awareness +break heap_hot_search_buffer + +# Redirect-with-data: initialize bitmap accumulator from collapsed redirect +break heapam_indexscan.c:182 + +# Accumulate bitmap from INDEXED_UPDATED tuple in chain +break heapam_indexscan.c:250 + +# Stale entry detection: accumulated bitmap overlaps this index's attrs +break heapam_indexscan.c:297 + +# ========================================================================= +# 5. INDEX SCAN SETUP — indexed_attrs bitmap computation +# src/backend/access/index/indexam.c +# ========================================================================= + +# Compute indexed_attrs for HOT indexed update chain following +break indexam.c:299 + +# ========================================================================= +# 6. INDEX INSERTION — skip unchanged indexes for HOT indexed updates +# src/backend/executor/execIndexing.c +# ========================================================================= + +# Entry: insert/update index tuples +break ExecInsertIndexTuples + +# Index skip decision: skip indexes whose attrs don't overlap modified set +break execIndexing.c:370 + +# ========================================================================= +# 7. PRUNING — chain collapsing and redirect-with-data +# src/backend/access/heap/pruneheap.c +# ========================================================================= + +# Main prune function +break heap_page_prune_and_freeze + +# Per-chain pruning entry +break heap_prune_chain + +# Chain collapsing: collect bitmaps from dead INDEXED_UPDATED intermediates +break pruneheap.c:1802 + +# OR dead tuple bitmaps into combined bitmap +break pruneheap.c:1836 + +# Record redirect-with-data for execute phase +break pruneheap.c:1863 + +# Execute phase: apply redirect-with-data entries on the page +break pruneheap.c:1287 + +# ========================================================================= +# 8. WAL REPLAY — recovery of HOT indexed updates +# src/backend/access/heap/heapam_xlog.c +# ========================================================================= + +# WAL replay for XLOG_HEAP2_INDEXED_UPDATE +break heap_xlog_indexed_update + +# ========================================================================= +# 9. WAL LOGGING — writing HOT indexed update records +# src/backend/access/heap/heapam.c +# ========================================================================= + +# WAL logging for heap updates (handles indexed_update flag) +break log_heap_update + +# Serialize redirect-with-data into WAL record (pruneheap.c) +break pruneheap.c:2936 diff --git a/.github/workflows/fork-ocr-model-check.yml b/.github/workflows/fork-ocr-model-check.yml new file mode 100644 index 0000000000000..b85a990c66fd7 --- /dev/null +++ b/.github/workflows/fork-ocr-model-check.yml @@ -0,0 +1,18 @@ +# Weekly check for a newer Claude Opus on Bedrock; opens/updates one issue here +# when OCR_BEDROCK_MODEL is behind. Notify-only (GITHUB_TOKEN cannot write +# Actions variables). Logic lives in gburd/ci-workflows. + +name: OCR model self-check + +on: + schedule: + - cron: '0 12 * * 1' # Mondays 12:00 UTC + workflow_dispatch: + +jobs: + check: + uses: gburd/ci-workflows/.github/workflows/ocr-model-check.yml@v1 + with: + aws_role_arn: ${{ vars.AWS_ROLE_ARN }} + aws_region: ${{ vars.AWS_REGION }} + bedrock_model: ${{ vars.OCR_BEDROCK_MODEL }} diff --git a/.github/workflows/fork-ocr-review.yml b/.github/workflows/fork-ocr-review.yml new file mode 100644 index 0000000000000..ae938754339b4 --- /dev/null +++ b/.github/workflows/fork-ocr-review.yml @@ -0,0 +1,43 @@ +# AI PR review (Claude Opus on Bedrock). Logic and review rules live in +# gburd/ci-workflows so this fork's master carries a stub instead of ~1000 +# lines that have to be rebased onto upstream forever. +# +# Needs repo variables AWS_ROLE_ARN, AWS_REGION, OCR_BEDROCK_MODEL. Auth is +# GitHub OIDC; no static AWS keys. See the ci-workflows README. + +name: OCR AI Review + +on: + pull_request: + # Note: no draft filter — drafts are reviewed too. + types: [opened, synchronize, reopened, ready_for_review] + issue_comment: + types: [created] + workflow_dispatch: + inputs: + pr_number: + description: 'PR number to review' + required: true + type: number + +# One review per PR; cancel superseded runs to save Bedrock spend. +concurrency: + group: ocr-review-${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} + cancel-in-progress: true + +jobs: + review: + # Comment events only when the comment is on a PR and starts with the + # trigger keyword; PR events and manual dispatch always. + if: | + github.event_name == 'pull_request' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (startsWith(github.event.comment.body, '/open-code-review') || + startsWith(github.event.comment.body, '@open-code-review') || + startsWith(github.event.comment.body, '/pg-history'))) + uses: gburd/ci-workflows/.github/workflows/ocr-review.yml@v1 + with: + aws_role_arn: ${{ vars.AWS_ROLE_ARN }} + aws_region: ${{ vars.AWS_REGION }} + bedrock_model: ${{ vars.OCR_BEDROCK_MODEL }} diff --git a/.github/workflows/fork-sync-upstream.yml b/.github/workflows/fork-sync-upstream.yml new file mode 100644 index 0000000000000..022b3101b5a9d --- /dev/null +++ b/.github/workflows/fork-sync-upstream.yml @@ -0,0 +1,48 @@ +# Keep this fork's master rebased on postgres/postgres master, carrying only +# the fork-owned CI stubs (.github/workflows/fork-*.yml). The workflows they +# call, and the OCR review rules, live in gburd/ci-workflows. +# +# Requires the SYNC_TOKEN secret: a PAT with repo+workflow scope. The default +# GITHUB_TOKEN is not allowed to push commits that touch .github/workflows/, +# and every commit we carry does. + +name: Sync from upstream + +on: + schedule: + - cron: '17 */3 * * *' + workflow_dispatch: + +concurrency: + group: fork-sync-upstream + +permissions: + contents: read + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + token: ${{ secrets.SYNC_TOKEN }} + + - name: Rebase master onto upstream + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git remote add upstream https://github.com/postgres/postgres.git + git fetch --no-tags upstream master + + # Master carries fork CI only. Anything else means work landed here + # that belongs on a branch; force-rebasing it on a timer would be wrong. + if git diff --name-only upstream/master...HEAD | + grep -v '^\.github/workflows/fork-'; then + echo "::error::master carries non-CI changes (listed above); move them to a branch" + exit 1 + fi + + git rebase upstream/master + git push --force-with-lease diff --git a/contrib/pg_buffercache/Makefile b/contrib/pg_buffercache/Makefile index 0e618f66aec6e..7fd5cdfc43d66 100644 --- a/contrib/pg_buffercache/Makefile +++ b/contrib/pg_buffercache/Makefile @@ -9,7 +9,8 @@ EXTENSION = pg_buffercache DATA = pg_buffercache--1.2.sql pg_buffercache--1.2--1.3.sql \ pg_buffercache--1.1--1.2.sql pg_buffercache--1.0--1.1.sql \ pg_buffercache--1.3--1.4.sql pg_buffercache--1.4--1.5.sql \ - pg_buffercache--1.5--1.6.sql pg_buffercache--1.6--1.7.sql + pg_buffercache--1.5--1.6.sql pg_buffercache--1.6--1.7.sql \ + pg_buffercache--1.7--1.8.sql PGFILEDESC = "pg_buffercache - monitoring of shared buffer cache in real-time" REGRESS = pg_buffercache pg_buffercache_numa diff --git a/contrib/pg_buffercache/meson.build b/contrib/pg_buffercache/meson.build index e681205abb2d8..361628b8bea42 100644 --- a/contrib/pg_buffercache/meson.build +++ b/contrib/pg_buffercache/meson.build @@ -25,6 +25,7 @@ install_data( 'pg_buffercache--1.4--1.5.sql', 'pg_buffercache--1.5--1.6.sql', 'pg_buffercache--1.6--1.7.sql', + 'pg_buffercache--1.7--1.8.sql', 'pg_buffercache.control', kwargs: contrib_data_args, ) diff --git a/contrib/pg_buffercache/pg_buffercache--1.7--1.8.sql b/contrib/pg_buffercache/pg_buffercache--1.7--1.8.sql new file mode 100644 index 0000000000000..9d8f4969555af --- /dev/null +++ b/contrib/pg_buffercache/pg_buffercache--1.7--1.8.sql @@ -0,0 +1,40 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION pg_buffercache UPDATE TO '1.8'" to load this file. \quit + +-- Register the new functions. +CREATE OR REPLACE FUNCTION pg_buffercache_partitions() +RETURNS SETOF RECORD +AS 'MODULE_PATHNAME', 'pg_buffercache_partitions' +LANGUAGE C PARALLEL SAFE; + +-- Create a view for convenient access. +CREATE VIEW pg_buffercache_partitions AS + SELECT P.* FROM pg_buffercache_partitions() AS P + (partition integer, -- partition index + numa_node integer, -- NUMA node of the partitioon + num_buffers integer, -- number of buffers in the partition + first_buffer integer, -- first buffer of partition + last_buffer integer, -- last buffer of partition + + -- clocksweep counters + num_passes bigint, -- clocksweep passes + next_buffer integer, -- next victim buffer for clocksweep + total_allocs bigint, -- handled allocs (running total) + num_allocs bigint, -- handled allocs (current cycle) + total_req_allocs bigint, -- requested allocs (running total) + num_req_allocs bigint, -- handled allocs (current cycle) + weights int[]); -- balancing weights + +-- Register the function to set clock-sweep balance weights. +CREATE FUNCTION pg_buffercache_set_partition(IN partition int, IN weights int[]) +RETURNS void +AS 'MODULE_PATHNAME', 'pg_buffercache_set_partition' +LANGUAGE C VOLATILE PARALLEL UNSAFE; + +-- Don't want these to be available to public. +REVOKE ALL ON FUNCTION pg_buffercache_partitions() FROM PUBLIC; +REVOKE ALL ON pg_buffercache_partitions FROM PUBLIC; +REVOKE ALL ON FUNCTION pg_buffercache_set_partition(int, int[]) FROM PUBLIC; + +GRANT EXECUTE ON FUNCTION pg_buffercache_partitions() TO pg_monitor; +GRANT SELECT ON pg_buffercache_partitions TO pg_monitor; diff --git a/contrib/pg_buffercache/pg_buffercache.control b/contrib/pg_buffercache/pg_buffercache.control index 11499550945ee..d2fa8ba53ba9f 100644 --- a/contrib/pg_buffercache/pg_buffercache.control +++ b/contrib/pg_buffercache/pg_buffercache.control @@ -1,5 +1,5 @@ # pg_buffercache extension comment = 'examine the shared buffer cache' -default_version = '1.7' +default_version = '1.8' module_pathname = '$libdir/pg_buffercache' relocatable = true diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index 510455998aa74..79766c0cb11e1 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -15,14 +15,23 @@ #include "port/pg_numa.h" #include "storage/buf_internals.h" #include "storage/bufmgr.h" +#include "utils/array.h" +#include "utils/builtins.h" #include "utils/rel.h" #include "utils/tuplestore.h" +#include "utils/typcache.h" #define NUM_BUFFERCACHE_PAGES_MIN_ELEM 8 #define NUM_BUFFERCACHE_PAGES_ELEM 9 #define NUM_BUFFERCACHE_SUMMARY_ELEM 5 #define NUM_BUFFERCACHE_USAGE_COUNTS_ELEM 4 + +/* + * Highest value the usagecount columns can report. The clock sweep tracks one + * second chance per buffer, so the reported count is 0 or 1. + */ +#define PGBUFFERCACHE_MAX_USAGE_COUNT 1 #define NUM_BUFFERCACHE_EVICT_ELEM 2 #define NUM_BUFFERCACHE_EVICT_RELATION_ELEM 3 #define NUM_BUFFERCACHE_EVICT_ALL_ELEM 3 @@ -31,6 +40,7 @@ #define NUM_BUFFERCACHE_MARK_DIRTY_ALL_ELEM 3 #define NUM_BUFFERCACHE_OS_PAGES_ELEM 3 +#define NUM_BUFFERCACHE_PARTITIONS_ELEM 12 PG_MODULE_MAGIC_EXT( .name = "pg_buffercache", @@ -77,6 +87,8 @@ PG_FUNCTION_INFO_V1(pg_buffercache_evict_all); PG_FUNCTION_INFO_V1(pg_buffercache_mark_dirty); PG_FUNCTION_INFO_V1(pg_buffercache_mark_dirty_relation); PG_FUNCTION_INFO_V1(pg_buffercache_mark_dirty_all); +PG_FUNCTION_INFO_V1(pg_buffercache_partitions); +PG_FUNCTION_INFO_V1(pg_buffercache_set_partition); /* Only need to touch memory once per backend process lifetime */ @@ -161,7 +173,7 @@ pg_buffercache_pages(PG_FUNCTION_ARGS) reldatabase = bufHdr->tag.dbOid; forknum = BufTagGetForkNum(&bufHdr->tag); blocknum = bufHdr->tag.blockNum; - usagecount = BUF_STATE_GET_USAGECOUNT(buf_state); + usagecount = (BUF_STATE_HAS_SECOND_CHANCE(buf_state) ? 1 : 0); pinning_backends = BUF_STATE_GET_REFCOUNT(buf_state); if (buf_state & BM_DIRTY) @@ -605,7 +617,7 @@ pg_buffercache_summary(PG_FUNCTION_ARGS) if (buf_state & BM_VALID) { buffers_used++; - usagecount_total += BUF_STATE_GET_USAGECOUNT(buf_state); + usagecount_total += (BUF_STATE_HAS_SECOND_CHANCE(buf_state) ? 1 : 0); if (buf_state & BM_DIRTY) buffers_dirty++; @@ -639,9 +651,9 @@ Datum pg_buffercache_usage_counts(PG_FUNCTION_ARGS) { ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - int usage_counts[BM_MAX_USAGE_COUNT + 1] = {0}; - int dirty[BM_MAX_USAGE_COUNT + 1] = {0}; - int pinned[BM_MAX_USAGE_COUNT + 1] = {0}; + int usage_counts[PGBUFFERCACHE_MAX_USAGE_COUNT + 1] = {0}; + int dirty[PGBUFFERCACHE_MAX_USAGE_COUNT + 1] = {0}; + int pinned[PGBUFFERCACHE_MAX_USAGE_COUNT + 1] = {0}; Datum values[NUM_BUFFERCACHE_USAGE_COUNTS_ELEM]; bool nulls[NUM_BUFFERCACHE_USAGE_COUNTS_ELEM] = {0}; @@ -655,7 +667,7 @@ pg_buffercache_usage_counts(PG_FUNCTION_ARGS) CHECK_FOR_INTERRUPTS(); - usage_count = BUF_STATE_GET_USAGECOUNT(buf_state); + usage_count = (BUF_STATE_HAS_SECOND_CHANCE(buf_state) ? 1 : 0); usage_counts[usage_count]++; if (buf_state & BM_DIRTY) @@ -665,7 +677,7 @@ pg_buffercache_usage_counts(PG_FUNCTION_ARGS) pinned[usage_count]++; } - for (int i = 0; i < BM_MAX_USAGE_COUNT + 1; i++) + for (int i = 0; i < PGBUFFERCACHE_MAX_USAGE_COUNT + 1; i++) { values[0] = Int32GetDatum(i); values[1] = Int32GetDatum(usage_counts[i]); @@ -922,3 +934,190 @@ pg_buffercache_mark_dirty_all(PG_FUNCTION_ARGS) PG_RETURN_DATUM(result); } + +/* + * Inquire about partitioning of shared buffers. + */ +Datum +pg_buffercache_partitions(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + MemoryContext oldcontext; + TupleDesc tupledesc; + TupleDesc expected_tupledesc; + HeapTuple tuple; + Datum result; + + if (SRF_IS_FIRSTCALL()) + { + TypeCacheEntry *typentry = lookup_type_cache(INT4OID, 0); + + funcctx = SRF_FIRSTCALL_INIT(); + + /* Switch context when allocating stuff to be used in later calls */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + if (get_call_result_type(fcinfo, NULL, &expected_tupledesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + if (expected_tupledesc->natts != NUM_BUFFERCACHE_PARTITIONS_ELEM) + elog(ERROR, "incorrect number of output arguments"); + + /* Construct a tuple descriptor for the result rows. */ + tupledesc = CreateTemplateTupleDesc(expected_tupledesc->natts); + TupleDescInitEntry(tupledesc, (AttrNumber) 1, "partition", + INT4OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 2, "numa_node", + INT4OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 3, "num_buffers", + INT4OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 4, "first_buffer", + INT4OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 5, "last_buffer", + INT4OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 6, "num_passes", + INT8OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 7, "next_buffer", + INT4OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 8, "total_allocs", + INT8OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 9, "num_allocs", + INT8OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 10, "total_req_allocs", + INT8OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 11, "num_req_allocs", + INT8OID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 12, "weigths", + typentry->typarray, -1, 0); + + funcctx->user_fctx = BlessTupleDesc(tupledesc); + + /* Return to original context when allocating transient memory */ + MemoryContextSwitchTo(oldcontext); + + /* Set max calls and remember the user function context. */ + funcctx->max_calls = BufferPartitionCount(); + } + + funcctx = SRF_PERCALL_SETUP(); + + if (funcctx->call_cntr < funcctx->max_calls) + { + uint32 i = funcctx->call_cntr; + + int numa_node, + num_buffers, + first_buffer, + last_buffer; + + uint64 buffer_total_allocs, + buffer_total_req_allocs; + + uint32 complete_passes, + next_victim_buffer, + buffer_allocs, + buffer_req_allocs; + + int *weights; + Datum *dweights; + ArrayType *array; + + Datum values[NUM_BUFFERCACHE_PARTITIONS_ELEM]; + bool nulls[NUM_BUFFERCACHE_PARTITIONS_ELEM]; + + BufferPartitionGet(i, &numa_node, &num_buffers, + &first_buffer, &last_buffer); + + ClockSweepPartitionGetInfo(i, + &complete_passes, &next_victim_buffer, + &buffer_total_allocs, &buffer_allocs, + &buffer_total_req_allocs, &buffer_req_allocs, + &weights); + + dweights = palloc_array(Datum, funcctx->max_calls); + for (int j = 0; j < funcctx->max_calls; j++) + dweights[j] = Int32GetDatum(weights[j]); + + array = construct_array_builtin(dweights, funcctx->max_calls, INT4OID); + + values[0] = Int32GetDatum(i); + nulls[0] = false; + + values[1] = Int32GetDatum(numa_node); + nulls[1] = false; + + values[2] = Int32GetDatum(num_buffers); + nulls[2] = false; + + values[3] = Int32GetDatum(first_buffer); + nulls[3] = false; + + values[4] = Int32GetDatum(last_buffer); + nulls[4] = false; + + values[5] = Int64GetDatum(complete_passes); + nulls[5] = false; + + values[6] = Int32GetDatum(next_victim_buffer); + nulls[6] = false; + + values[7] = Int64GetDatum(buffer_total_allocs); + nulls[7] = false; + + values[8] = Int64GetDatum(buffer_allocs); + nulls[8] = false; + + values[9] = Int64GetDatum(buffer_total_req_allocs); + nulls[9] = false; + + values[10] = Int64GetDatum(buffer_req_allocs); + nulls[10] = false; + + values[11] = PointerGetDatum(array); + nulls[11] = false; + + /* Build and return the tuple. */ + tuple = heap_form_tuple((TupleDesc) funcctx->user_fctx, values, nulls); + result = HeapTupleGetDatum(tuple); + + SRF_RETURN_NEXT(funcctx, result); + } + else + SRF_RETURN_DONE(funcctx); +} + +/* + * Set the clock-sweep balance weights for a single partition. + */ +Datum +pg_buffercache_set_partition(PG_FUNCTION_ARGS) +{ + int partition = PG_GETARG_INT32(0); + ArrayType *array = PG_GETARG_ARRAYTYPE_P(1); + Datum *elems; + bool *nulls; + int nelems; + int *weights; + + if (ARR_NDIM(array) > 1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("weights must be a one-dimensional array"))); + + deconstruct_array_builtin(array, INT4OID, &elems, &nulls, &nelems); + + weights = palloc_array(int, nelems); + for (int i = 0; i < nelems; i++) + { + if (nulls[i]) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("weights must not contain NULL values"))); + + weights[i] = DatumGetInt32(elems[i]); + } + + ClockSweepSetWeights(partition, weights, nelems); + + PG_RETURN_VOID(); +} diff --git a/doc/src/sgml/pgbuffercache.sgml b/doc/src/sgml/pgbuffercache.sgml index 1e9aee10275f2..f751b55e75406 100644 --- a/doc/src/sgml/pgbuffercache.sgml +++ b/doc/src/sgml/pgbuffercache.sgml @@ -99,7 +99,9 @@ The pg_buffercache_usage_counts() function returns a set of records, each row describing the number of buffers with a given usage - count. + count. The clock sweep tracks one access per buffer, so there are two rows: + usage count 0 for buffers that are candidates for replacement, and 1 for + those that will be passed over once more. @@ -240,7 +242,9 @@ usagecount smallint - Clock-sweep access count + Clock-sweep access count: 1 if the buffer has been accessed since the + clock hand last passed it, and so will be passed over once more, or 0 + if it is a candidate for replacement diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000000..b8e8a1fdb750f --- /dev/null +++ b/flake.lock @@ -0,0 +1,78 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1767313136, + "narHash": "sha256-16KkgfdYqjaeRGBaYsNrhPRRENs0qzkQVUooNHtoy2w=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ac62194c3917d5f474c1a844b6fd6da2db95077d", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-25.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1777270315, + "narHash": "sha256-yKB4G6cKsQsWN7M6rZGk6gkJPDNPIzT05y4qzRyCDlI=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "6368eda62c9775c38ef7f714b2555a741c20c72d", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "nixpkgs-unstable": "nixpkgs-unstable" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000000..aae6d54c4c8cf --- /dev/null +++ b/flake.nix @@ -0,0 +1,45 @@ +{ + description = "PostgreSQL development environment"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; + nixpkgs-unstable.url = "github:nixos/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { + self, + nixpkgs, + nixpkgs-unstable, + flake-utils, + }: + flake-utils.lib.eachDefaultSystem ( + system: let + pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; + }; + pkgs-unstable = import nixpkgs-unstable { + inherit system; + config.allowUnfree = true; + }; + + shellConfig = import ./shell.nix {inherit pkgs pkgs-unstable system;}; + in { + formatter = pkgs.alejandra; + devShells = { + default = shellConfig.devShell; + gcc = shellConfig.devShell; + clang = shellConfig.clangDevShell; + gcc-musl = shellConfig.muslDevShell; + clang-musl = shellConfig.clangMuslDevShell; + }; + + packages = { + inherit (shellConfig) gdbConfig flameGraphScript pgbenchScript; + }; + + environment.localBinInPath = true; + } + ); +} diff --git a/glibc-no-fortify-warning.patch b/glibc-no-fortify-warning.patch new file mode 100644 index 0000000000000..4657a12adbcc5 --- /dev/null +++ b/glibc-no-fortify-warning.patch @@ -0,0 +1,24 @@ +From 130c231020f97e5eb878cc9fdb2bd9b186a5aa04 Mon Sep 17 00:00:00 2001 +From: Greg Burd +Date: Fri, 24 Oct 2025 11:58:24 -0400 +Subject: [PATCH] no warnings with -O0 and fortify source please + +--- + include/features.h | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/include/features.h b/include/features.h +index 673c4036..a02c8a3f 100644 +--- a/include/features.h ++++ b/include/features.h +@@ -432,7 +432,6 @@ + + #if defined _FORTIFY_SOURCE && _FORTIFY_SOURCE > 0 + # if !defined __OPTIMIZE__ || __OPTIMIZE__ <= 0 +-# warning _FORTIFY_SOURCE requires compiling with optimization (-O) + # elif !__GNUC_PREREQ (4, 1) + # warning _FORTIFY_SOURCE requires GCC 4.1 or later + # elif _FORTIFY_SOURCE > 2 && (__glibc_clang_prereq (9, 0) \ +-- +2.50.1 + diff --git a/pg-aliases.sh b/pg-aliases.sh new file mode 100644 index 0000000000000..0c13adc8f903a --- /dev/null +++ b/pg-aliases.sh @@ -0,0 +1,658 @@ +# PostgreSQL Development Aliases + +# ============================================================ +# Build helpers shared by every variant. +# ============================================================ +pg_clean_for_compiler() { + local current_compiler="$(basename $CC)" + local build_dir="${1:-$PG_BUILD_DIR}" + + if [ -f "$build_dir/compile_commands.json" ]; then + local last_compiler=$(grep -o '/[^/]*/bin/[gc]cc\|/[^/]*/bin/clang' "$build_dir/compile_commands.json" | head -1 | xargs basename 2>/dev/null || echo "unknown") + + if [ "$last_compiler" != "$current_compiler" ] && [ "$last_compiler" != "unknown" ]; then + echo "Detected compiler change from $last_compiler to $current_compiler" + echo "Cleaning build directory..." + trash "$build_dir" 2>/dev/null || rm -rf "$build_dir" + mkdir -p "$build_dir" + fi + fi + + mkdir -p "$build_dir" + echo "$current_compiler" >"$build_dir/.compiler_used" +} + +# ============================================================ +# Core PostgreSQL commands (default/debug build) +# ============================================================ +alias pg-setup=' + if [ -z "$PERL_CORE_DIR" ]; then + echo "Error: Could not find perl CORE directory" >&2 + return 1 + fi + + pg_clean_for_compiler "$PG_BUILD_DIR" + + echo "=== PostgreSQL Build Configuration ===" + echo "Compiler: $CC" + echo "LLVM: $(llvm-config --version 2>/dev/null || echo disabled)" + echo "Source: $PG_SOURCE_DIR" + echo "Build: $PG_BUILD_DIR" + echo "Install: $PG_INSTALL_DIR" + echo "======================================" + + env CFLAGS="-I$PERL_CORE_DIR $CFLAGS" \ + LDFLAGS="-L$PERL_CORE_DIR -lperl $LDFLAGS" \ + meson setup $MESON_EXTRA_SETUP \ + --reconfigure \ + -Doptimization=g \ + -Ddebug=true \ + -Db_sanitize=none \ + -Db_lundef=false \ + -Dlz4=enabled \ + -Dzstd=enabled \ + -Dllvm=disabled \ + -Dplperl=enabled \ + -Dplpython=enabled \ + -Dpltcl=enabled \ + -Dlibxml=enabled \ + -Duuid=e2fs \ + -Dlibxslt=enabled \ + -Dssl=openssl \ + -Dldap=disabled \ + -Dcassert=true \ + -Dtap_tests=enabled \ + -Dinjection_points=true \ + -Ddocs_pdf=enabled \ + -Ddocs_html_style=website \ + --prefix="$PG_INSTALL_DIR" \ + "$PG_BUILD_DIR" \ + "$PG_SOURCE_DIR"' + +alias pg-compdb='compdb -p build/ list > compile_commands.json' +alias pg-build='meson compile -C "$PG_BUILD_DIR"' +alias pg-install='meson install -C "$PG_BUILD_DIR"' +alias pg-test='meson test -q --print-errorlogs -C "$PG_BUILD_DIR"' + +# Clean commands +alias pg-clean='ninja -C "$PG_BUILD_DIR" clean' +alias pg-full-clean='trash "$PG_BUILD_DIR" "$PG_INSTALL_DIR" 2>/dev/null || rm -rf "$PG_BUILD_DIR" "$PG_INSTALL_DIR"; echo "Build and install directories cleaned"' + +# Database management +alias pg-init='trash "$PG_DATA_DIR" 2>/dev/null || rm -rf "$PG_DATA_DIR"; "$PG_INSTALL_DIR/bin/initdb" --debug --no-clean "$PG_DATA_DIR"' + +alias pg-start='ulimit -c unlimited && "$PG_INSTALL_DIR/bin/postgres" -D "$PG_DATA_DIR" -k "$PG_DATA_DIR"' + +alias pg-stop='pkill -f "postgres.*-D.*$PG_DATA_DIR" || true' +alias pg-restart='pg-stop && sleep 2 && pg-start' +alias pg-status='pgrep -f "postgres.*-D.*$PG_DATA_DIR" && echo "PostgreSQL is running" || echo "PostgreSQL is not running"' + +# Client connections +alias pg-psql='"$PG_INSTALL_DIR/bin/psql" -h "$PG_DATA_DIR" postgres' +alias pg-createdb='"$PG_INSTALL_DIR/bin/createdb" -h "$PG_DATA_DIR"' +alias pg-dropdb='"$PG_INSTALL_DIR/bin/dropdb" -h "$PG_DATA_DIR"' + +# ============================================================ +# Debugger attachments +# ============================================================ +alias pg-debug-gdb='gdb -x "$GDBINIT" -x .gdbinit "$PG_INSTALL_DIR/bin/postgres"' +alias pg-debug-lldb='lldb "$PG_INSTALL_DIR/bin/postgres"' +alias pg-debug=' + if command -v gdb >/dev/null 2>&1; then + pg-debug-gdb + elif command -v lldb >/dev/null 2>&1; then + pg-debug-lldb + else + echo "No debugger available (gdb or lldb required)" + fi' + +alias pg-attach-gdb=' + PG_PID=$(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1) + if [ -n "$PG_PID" ]; then + echo "Attaching GDB to PostgreSQL process $PG_PID" + gdb -x "$GDBINIT" -x .gdbinit -p "$PG_PID" + else + echo "No PostgreSQL process found" + fi' + +alias pg-attach-lldb=' + PG_PID=$(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1) + if [ -n "$PG_PID" ]; then + echo "Attaching LLDB to PostgreSQL process $PG_PID" + lldb -p "$PG_PID" + else + echo "No PostgreSQL process found" + fi' + +alias pg-attach=' + if command -v gdb >/dev/null 2>&1; then + pg-attach-gdb + elif command -v lldb >/dev/null 2>&1; then + pg-attach-lldb + else + echo "No debugger available (gdb or lldb required)" + fi' + +# ============================================================ +# Valgrind-instrumented build and tests +# +# The valgrind build lives in a separate directory so the normal +# build stays warm. Runs use a wrapper dir that shadows `postgres` +# with a valgrind wrapper -- pg_regress finds it via PATH. +# ============================================================ +pg-build-valgrind() { + local bdir="$PG_BUILD_DIR_VALGRIND" + if [ -z "$PERL_CORE_DIR" ]; then + echo "Error: PERL_CORE_DIR is not set" >&2 + return 1 + fi + + pg_clean_for_compiler "$bdir" + + echo "=== Configuring Valgrind build in $bdir ===" + env CFLAGS="-Og -ggdb3 -fno-omit-frame-pointer -DUSE_VALGRIND -I$PERL_CORE_DIR $CFLAGS" \ + LDFLAGS="-L$PERL_CORE_DIR -lperl $LDFLAGS" \ + meson setup --reconfigure \ + -Doptimization=g \ + -Ddebug=true \ + -Dcassert=true \ + -Dtap_tests=enabled \ + -Dinjection_points=true \ + -Dllvm=disabled \ + -Dplperl=enabled -Dplpython=enabled -Dpltcl=enabled \ + -Dlz4=enabled -Dzstd=enabled \ + -Dlibxml=enabled -Dlibxslt=enabled -Dssl=openssl -Duuid=e2fs \ + -Dldap=disabled \ + --prefix="$PG_INSTALL_DIR-valgrind" \ + "$bdir" "$PG_SOURCE_DIR" || return 1 + + meson compile -C "$bdir" +} + +# Drop a wrapper directory that shadows the real binaries; `postgres` +# exec's into valgrind, everything else is a symlink. Writes to the +# supplied wrap dir and echoes its path. +_pg_make_valgrind_wrapper() { + local bindir="$1" + local wrapdir="$2" + + mkdir -p "$wrapdir" + cat >"$wrapdir/postgres" <&2 + return 1 + fi + + local tmpbin="$bdir/tmp_install$PG_INSTALL_DIR-valgrind/bin" + if [ ! -x "$tmpbin/postgres" ]; then + echo "Populating tmp_install..." + meson test -C "$bdir" tmp_install install_test_files initdb_cache >/dev/null || return 1 + fi + + local wrap + wrap=$(mktemp -d /tmp/pg-vg-wrap-XXXXXX) + _pg_make_valgrind_wrapper "$tmpbin" "$wrap" + + mkdir -p "$PG_BENCH_DIR" + echo "Valgrind logs: $PG_BENCH_DIR/valgrind-*.log" + echo "Wrapper dir: $wrap (will be removed on exit)" + echo "Expect the regress suite to take 15-45 minutes under valgrind." + + local rc=0 + (cd "$bdir" && PATH="$wrap:$PATH" meson test -t 60 --print-errorlogs regress/regress) || rc=$? + + trash "$wrap" 2>/dev/null || rm -rf "$wrap" + return "$rc" +} + +pg-valgrind-test() { + local bdir="$PG_BUILD_DIR_VALGRIND" + if [ ! -x "$bdir/src/backend/postgres" ]; then + echo "Valgrind build not found; run 'pg-build-valgrind' first." >&2 + return 1 + fi + + echo "This runs the FULL postgres test suite under valgrind." + echo "Expect many hours, and tens of GB of valgrind log output." + echo "Logs: $PG_BENCH_DIR/valgrind-*.log" + local yn + read -r -p "Continue? [y/N] " yn + case "$yn" in + y | Y | yes) ;; + *) echo "Aborted."; return 0 ;; + esac + + local tmpbin="$bdir/tmp_install$PG_INSTALL_DIR-valgrind/bin" + if [ ! -x "$tmpbin/postgres" ]; then + echo "Populating tmp_install..." + meson test -C "$bdir" tmp_install install_test_files initdb_cache >/dev/null || return 1 + fi + + local wrap + wrap=$(mktemp -d /tmp/pg-vg-wrap-XXXXXX) + _pg_make_valgrind_wrapper "$tmpbin" "$wrap" + mkdir -p "$PG_BENCH_DIR" + + local rc=0 + (cd "$bdir" && PATH="$wrap:$PATH" meson test -t 60 --print-errorlogs) || rc=$? + + trash "$wrap" 2>/dev/null || rm -rf "$wrap" + return "$rc" +} + +# ============================================================ +# AddressSanitizer / UndefinedBehaviorSanitizer build and tests +# ============================================================ +pg-build-asan() { + local bdir="$PG_BUILD_DIR_ASAN" + if [ -z "$PERL_CORE_DIR" ]; then + echo "Error: PERL_CORE_DIR is not set" >&2 + return 1 + fi + + pg_clean_for_compiler "$bdir" + + echo "=== Configuring ASan+UBSan build in $bdir ===" + env CFLAGS="-Og -ggdb3 -fno-omit-frame-pointer -fsanitize=address,undefined -fno-sanitize-recover=all -I$PERL_CORE_DIR $CFLAGS" \ + LDFLAGS="-fsanitize=address,undefined -L$PERL_CORE_DIR -lperl $LDFLAGS" \ + meson setup --reconfigure \ + -Doptimization=g \ + -Ddebug=true \ + -Dcassert=true \ + -Dtap_tests=enabled \ + -Dinjection_points=true \ + -Dllvm=disabled \ + -Dplperl=enabled -Dplpython=enabled -Dpltcl=enabled \ + -Dlz4=enabled -Dzstd=enabled \ + -Dlibxml=enabled -Dlibxslt=enabled -Dssl=openssl -Duuid=e2fs \ + -Dldap=disabled \ + --prefix="$PG_INSTALL_DIR-asan" \ + "$bdir" "$PG_SOURCE_DIR" || return 1 + + meson compile -C "$bdir" +} + +pg-asan-regress() { + local bdir="$PG_BUILD_DIR_ASAN" + if [ ! -x "$bdir/src/backend/postgres" ]; then + echo "ASan build not found; run 'pg-build-asan' first." >&2 + return 1 + fi + + # halt_on_error=0 lets regress continue past the first diagnostic so + # the whole suite runs; abort_on_error=1 makes each hit fail the test. + ASAN_OPTIONS="halt_on_error=0:abort_on_error=1:detect_leaks=0:print_summary=1:print_stacktrace=1" \ + UBSAN_OPTIONS="halt_on_error=1:abort_on_error=1:print_stacktrace=1:print_summary=1" \ + meson test -t 5 --print-errorlogs -C "$bdir" regress/regress +} + +# ============================================================ +# rr (deterministic record-and-replay) +# Requires kernel.perf_event_paranoid <= 1. rr is the single most +# effective tool for postgres bugs that reproduce intermittently. +# ============================================================ +pg-rr-check() { + if ! command -v rr >/dev/null; then + echo "rr is not installed (expected in the dev shell)." >&2 + return 1 + fi + local paranoid + paranoid=$(cat /proc/sys/kernel/perf_event_paranoid 2>/dev/null || echo 99) + if [ "$paranoid" -gt 1 ]; then + echo "rr requires kernel.perf_event_paranoid <= 1; currently $paranoid" + echo "To enable (root needed):" + echo " echo 1 | sudo tee /proc/sys/kernel/perf_event_paranoid" + return 1 + fi + echo "rr ready (perf_event_paranoid=$paranoid)" +} + +pg-rr-record() { + pg-rr-check >/dev/null || { + pg-rr-check + return 1 + } + ulimit -c unlimited + rr record -- "$PG_INSTALL_DIR/bin/postgres" -D "$PG_DATA_DIR" -k "$PG_DATA_DIR" +} + +pg-rr-replay() { + rr replay "$@" +} + +# ============================================================ +# perf wrappers (parallel to the flame-graph helper) +# ============================================================ +pg-perf-record() { + local pid + pid=$(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1) + if [ -z "$pid" ]; then + echo "No postgres running under $PG_DATA_DIR" >&2 + return 1 + fi + mkdir -p "$PG_BENCH_DIR" + local out="$PG_BENCH_DIR/perf-$(date +%Y%m%d_%H%M%S).data" + echo "Recording to $out (Ctrl-C to stop)" + perf record -F 997 --call-graph dwarf -p "$pid" -o "$out" "$@" + echo "Saved: $out" +} + +pg-perf-report() { + local data + data=$(ls -t "$PG_BENCH_DIR"/perf-*.data 2>/dev/null | head -1) + if [ -z "$data" ]; then + echo "No perf data in $PG_BENCH_DIR" >&2 + return 1 + fi + echo "Reading $data" + perf report -i "$data" "$@" +} + +pg-perf-annotate() { + local data + data=$(ls -t "$PG_BENCH_DIR"/perf-*.data 2>/dev/null | head -1) + if [ -z "$data" ]; then + echo "No perf data in $PG_BENCH_DIR" >&2 + return 1 + fi + perf annotate -i "$data" "$@" +} + +# ============================================================ +# Single regression test / group runner. +# Runs pg_regress directly against the existing build so you skip the +# full meson-driven suite wrapper. Usage: pg-test-one boolean [name ...] +# ============================================================ +pg-test-one() { + if [ $# -eq 0 ]; then + echo "usage: pg-test-one TESTNAME [TESTNAME ...]" + echo "example: pg-test-one boolean" + return 2 + fi + local bdir="${PG_BUILD_DIR_ONE:-$PG_BUILD_DIR}" + local tmpbin="$bdir/tmp_install$PG_INSTALL_DIR/bin" + if [ ! -x "$tmpbin/postgres" ]; then + echo "Populating tmp_install..." + meson test -C "$bdir" tmp_install install_test_files initdb_cache >/dev/null || return 1 + fi + local outdir + outdir=$(mktemp -d /tmp/pg-test-one-XXXXXX) + echo "Test output: $outdir" + "$bdir/src/test/regress/pg_regress" \ + --bindir="$tmpbin" \ + --inputdir="$PG_SOURCE_DIR/src/test/regress" \ + --expecteddir="$PG_SOURCE_DIR/src/test/regress" \ + --dlpath="$bdir/src/test/regress" \ + --outputdir="$outdir" \ + --temp-instance="$outdir/tmp" \ + --port=40099 \ + "$@" +} + +# Full flame graph / benchmark aliases +alias pg-flame='pg-flame-generate' +alias pg-flame-30='pg-flame-generate 30' +alias pg-flame-60='pg-flame-generate 60' +alias pg-flame-120='pg-flame-generate 120' + +pg-flame-custom() { + local duration=${1:-30} + local output_dir=${2:-$PG_FLAME_DIR} + echo "Generating flame graph for ${duration}s, output to: $output_dir" + pg-flame-generate "$duration" "$output_dir" +} + +alias pg-bench='pg-bench-run' +alias pg-bench-quick='pg-bench-run 5 1 100 1 30 select-only' +alias pg-bench-standard='pg-bench-run 10 2 1000 10 60 tpcb-like' +alias pg-bench-heavy='pg-bench-run 50 4 5000 100 300 tpcb-like' +alias pg-bench-readonly='pg-bench-run 20 4 2000 50 120 select-only' + +pg-bench-custom() { + local clients=${1:-10} + local threads=${2:-2} + local transactions=${3:-1000} + local scale=${4:-10} + local duration=${5:-60} + local test_type=${6:-tpcb-like} + + echo "Running custom benchmark:" + echo " Clients: $clients, Threads: $threads" + echo " Transactions: $transactions, Scale: $scale" + echo " Duration: ${duration}s, Type: $test_type" + + pg-bench-run "$clients" "$threads" "$transactions" "$scale" "$duration" "$test_type" +} + +pg-bench-flame() { + local duration=${1:-60} + local clients=${2:-10} + local scale=${3:-10} + + echo "Running benchmark with flame graph generation" + echo "Duration: ${duration}s, Clients: $clients, Scale: $scale" + + pg-bench-run "$clients" 2 1000 "$scale" "$duration" tpcb-like & + local bench_pid=$! + + sleep 5 + + local flame_duration=$((duration - 10)) + if [ $flame_duration -gt 10 ]; then + pg-flame-generate "$flame_duration" & + local flame_pid=$! + fi + + wait $bench_pid + if [ -n "${flame_pid:-}" ]; then + wait $flame_pid + fi + + echo "Benchmark and flame graph generation completed" +} + +# Live monitoring +alias pg-perf='perf top -p $(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | head -1)' +alias pg-htop='htop -p $(pgrep -f "postgres.*-D.*$PG_DATA_DIR" | tr "\n" "," | sed "s/,$//")' + +pg-stats() { + local duration=${1:-30} + echo "Collecting system stats for ${duration}s..." + + iostat -x 1 "$duration" >"$PG_BENCH_DIR/iostat_$(date +%Y%m%d_%H%M%S).log" & + vmstat 1 "$duration" >"$PG_BENCH_DIR/vmstat_$(date +%Y%m%d_%H%M%S).log" & + + wait + echo "System stats saved to $PG_BENCH_DIR" +} + +# ============================================================ +# Code quality helpers +# ============================================================ +pg-format() { + local since=${1:-HEAD} + + if [ ! -f "$PG_SOURCE_DIR/src/tools/pgindent/pgindent" ]; then + echo "Error: pgindent not found at $PG_SOURCE_DIR/src/tools/pgindent/pgindent" + else + + modified_files=$(git diff --name-only "${since}" | grep -E "\.c$|\.h$") + + if [ -z "$modified_files" ]; then + echo "No modified .c or .h files found" + else + + echo "Formatting modified files with pgindent:" + for file in $modified_files; do + if [ -f "$file" ]; then + echo " Formatting: $file" + "$PG_SOURCE_DIR/src/tools/pgindent/pgindent" "$file" + else + echo " Warning: File not found: $file" + fi + done + + echo "Checking files for whitespace:" + git diff --check "${since}" + fi + fi +} + +pg-tidy() { + local since=${1:-HEAD} + local files + files=$(git diff --name-only "$since" | grep -E "\.(c|h)$") + if [ -z "$files" ]; then + echo "No modified .c or .h files." + return 0 + fi + for f in $files; do + [ -f "$f" ] || continue + echo "clang-tidy: $f" + clang-tidy -p "$PG_BUILD_DIR" "$f" 2>&1 | head -50 + done +} + +pg-spell() { + local since=${1:-HEAD} + local files=$(git diff --name-only "$since" | grep -E '\.(c|h|sgml|md)$') + if [ -z "$files" ]; then + echo "No .c/.h/.sgml/.md files changed since $since" + return 0 + fi + for f in $files; do + [ -f "$f" ] || continue + case "$f" in + *.c | *.h) + grep -nE '^\s*(/\*|\*|//)' "$f" | codespell --stdin-single-line - 2>/dev/null \ + && echo " $f: ok" || true + ;; + *.sgml | *.md) + codespell "$f" || true + ;; + esac + done +} + +# ============================================================ +# Core dump one-shots (one-time, requires root). kernel.core_pattern +# is a system-wide sysctl -- we don't touch it on every shell entry. +# ============================================================ +pg-cores-status() { + echo "ulimit -c: $(ulimit -c)" + echo "kernel.core_pattern: $(cat /proc/sys/kernel/core_pattern 2>/dev/null || echo unreadable)" + echo "cwd: $(pwd)" +} + +pg-enable-cores() { + ulimit -c unlimited + if ! [ -w /proc/sys/kernel/core_pattern ]; then + echo "Setting kernel.core_pattern (requires sudo)..." + echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern >/dev/null || { + echo "Failed to write /proc/sys/kernel/core_pattern" >&2 + return 1 + } + else + echo "core.%p" >/proc/sys/kernel/core_pattern + fi + pg-cores-status +} + +pg-disable-cores() { + ulimit -c 0 + if ! [ -w /proc/sys/kernel/core_pattern ]; then + echo "Restoring kernel.core_pattern to 'core' (requires sudo)..." + echo "core" | sudo tee /proc/sys/kernel/core_pattern >/dev/null || { + echo "Failed to restore /proc/sys/kernel/core_pattern" >&2 + return 1 + } + else + echo "core" >/proc/sys/kernel/core_pattern + fi + pg-cores-status +} + +# ============================================================ +# Logs and results +# ============================================================ +alias pg-log='tail -f "$PG_DATA_DIR/log/postgresql-$(date +%Y-%m-%d).log" 2>/dev/null || echo "No log file found"' +alias pg-log-errors='grep -i error "$PG_DATA_DIR/log/"*.log 2>/dev/null || echo "No error logs found"' + +alias pg-build-log='cat "$PG_BUILD_DIR/meson-logs/meson-log.txt"' +alias pg-build-errors='grep -i error "$PG_BUILD_DIR/meson-logs/meson-log.txt" 2>/dev/null || echo "No build errors found"' + +alias pg-bench-results='ls -la "$PG_BENCH_DIR" && echo "Latest results:" && tail -20 "$PG_BENCH_DIR"/results_*.txt 2>/dev/null | tail -20' +alias pg-flame-results='ls -la "$PG_FLAME_DIR" && echo "Open flame graphs with: firefox $PG_FLAME_DIR/*.svg"' + +pg-clean-results() { + local days=${1:-7} + echo "Cleaning benchmark and flame graph results older than $days days..." + find "$PG_BENCH_DIR" -type f -mtime +$days -delete 2>/dev/null || true + find "$PG_FLAME_DIR" -type f -mtime +$days -delete 2>/dev/null || true + echo "Cleanup completed" +} + +# ============================================================ +# Info +# ============================================================ +alias pg-info=' + echo "=== PostgreSQL Development Environment ===" + echo "Source: $PG_SOURCE_DIR" + echo "Build (default): $PG_BUILD_DIR" + echo "Build (valgrind):$PG_BUILD_DIR_VALGRIND" + echo "Build (asan): $PG_BUILD_DIR_ASAN" + echo "Install: $PG_INSTALL_DIR" + echo "Data: $PG_DATA_DIR" + echo "Benchmarks: $PG_BENCH_DIR" + echo "Flame graphs: $PG_FLAME_DIR" + echo "Compiler: $CC" + echo "" + echo "Available commands:" + echo " Setup/build: pg-setup, pg-build, pg-install" + echo " Database: pg-init, pg-start, pg-stop, pg-psql" + echo " Tests: pg-test, pg-test-one NAME" + echo " Valgrind: pg-build-valgrind, pg-valgrind-regress, pg-valgrind-test" + echo " ASan/UBSan: pg-build-asan, pg-asan-regress" + echo " Debug: pg-debug, pg-attach" + echo " Record/replay: pg-rr-check, pg-rr-record, pg-rr-replay" + echo " Perf: pg-perf-record, pg-perf-report, pg-perf-annotate, pg-perf" + echo " Flame graphs: pg-flame, pg-flame-30, pg-flame-60, pg-flame-custom" + echo " Benchmarks: pg-bench-quick, pg-bench-standard, pg-bench-heavy" + echo " Combined: pg-bench-flame" + echo " Results: pg-bench-results, pg-flame-results" + echo " Logs: pg-log, pg-build-log" + echo " Clean: pg-clean, pg-full-clean, pg-clean-results" + echo " Code quality: pg-format, pg-tidy, pg-spell" + echo " Cores: pg-enable-cores, pg-disable-cores, pg-cores-status" + echo "=========================================="' + +echo "PostgreSQL aliases loaded. Run 'pg-info' for available commands." diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000000..8c738afa789a7 --- /dev/null +++ b/shell.nix @@ -0,0 +1,745 @@ +{ + pkgs, + pkgs-unstable, + system, +}: let + # Create a patched glibc only for the dev shell. + # + # Glibc's features.h emits a `-Wcpp` diagnostic when _FORTIFY_SOURCE is + # defined without an optimization level. Meson's dependency probes + # (notably the libcurl thread-safety check) compile small snippets with + # `-O0 -Werror`, which turns that cpp warning into a hard error and + # breaks reconfigure under our default CFLAGS. The patch simply drops + # the warning. It is scoped to this dev shell only and never leaks + # into system glibc or release builds. + patchedGlibc = pkgs.glibc.overrideAttrs (oldAttrs: { + patches = (oldAttrs.patches or []) ++ [ + ./glibc-no-fortify-warning.patch + ]; + }); + + # Use LLVM for modern PostgreSQL development + llvmPkgs = pkgs-unstable.llvmPackages_21; + + # Configuration constants + config = { + pgSourceDir = "$PWD"; + pgBuildDir = "$PWD/build"; + pgBuildDirValgrind = "$PWD/build-valgrind"; + pgBuildDirAsan = "$PWD/build-asan"; + pgInstallDir = "$PWD/install"; + pgDataDir = "/tmp/test-db-$(basename $PWD)"; + pgBenchDir = "/tmp/pgbench-results-$(basename $PWD)"; + pgFlameDir = "/tmp/flame-graphs-$(basename $PWD)"; + }; + + # Single dependency function that can be used for all environments + getPostgreSQLDeps = muslLibs: + with pkgs; + [ + # Build system (always use host tools) + pkgs-unstable.meson + pkgs-unstable.ninja + pkg-config + autoconf + git + which + binutils + gnumake + mold # fast linker, big wins on large postgres links + + # Parser/lexer tools + bison + flex + + # Perl with required packages + (perl.withPackages (ps: with ps; [IPCRun])) + + # Documentation + docbook_xml_dtd_45 + docbook-xsl-nons + libxslt + libxml2 + fop + + # Development tools (always use host tools) + coreutils + shellcheck + ripgrep + valgrind + curl + uv + pylint + black + lcov + strace + ltrace + perf-tools + linuxPackages.perf + flamegraph + bpftrace # kernel-level tracing (probes, uprobes) + rr # record-and-replay deterministic debugger + htop + iotop + sysstat + ccache + cppcheck + compdb + + # Spell checking + aspell + aspellDicts.en + codespell + + # GCC/GDB + gcc + gdb + + # LLVM toolchain + llvmPkgs.llvm + llvmPkgs.llvm.dev + llvmPkgs.clang-tools + llvmPkgs.lldb + + # Language support + (python3.withPackages (ps: with ps; [requests browser-cookie3])) + tcl + ] + ++ ( + if muslLibs + then [ + # Musl target libraries for cross-compilation + pkgs.pkgsMusl.readline + pkgs.pkgsMusl.zlib + pkgs.pkgsMusl.openssl + pkgs.pkgsMusl.icu + pkgs.pkgsMusl.lz4 + pkgs.pkgsMusl.zstd + pkgs.pkgsMusl.libuuid + pkgs.pkgsMusl.libkrb5 + pkgs.pkgsMusl.linux-pam + pkgs.pkgsMusl.libxcrypt + ] + else [ + # Glibc target libraries + readline + zlib + openssl + icu + lz4 + zstd + libuuid + libkrb5 + linux-pam + libxcrypt + numactl + openldap + liburing + libselinux + patchedGlibc + patchedGlibc.dev + ] + ); + + # GDB configuration for PostgreSQL debugging + gdbConfig = pkgs.writeText "gdbinit-postgres" '' + # PostgreSQL-specific GDB configuration + + # Pretty-print PostgreSQL data structures + define print_node + if $arg0 + printf "Node type: %s\n", nodeTagNames[$arg0->type] + print *$arg0 + else + printf "NULL node\n" + end + end + document print_node + Print a PostgreSQL Node with type information + Usage: print_node + end + + define print_list + set $list = (List*)$arg0 + if $list + printf "List length: %d\n", $list->length + set $cell = $list->head + set $i = 0 + while $cell && $i < $list->length + printf " [%d]: ", $i + print_node $cell->data.ptr_value + set $cell = $cell->next + set $i = $i + 1 + end + else + printf "NULL list\n" + end + end + document print_list + Print a PostgreSQL List structure + Usage: print_list + end + + define print_query + set $query = (Query*)$arg0 + if $query + printf "Query type: %d, command type: %d\n", $query->querySource, $query->commandType + print *$query + else + printf "NULL query\n" + end + end + document print_query + Print a PostgreSQL Query structure + Usage: print_query + end + + define print_relcache + set $rel = (Relation)$arg0 + if $rel + printf "Relation: %s.%s (OID: %u)\n", $rel->rd_rel->relnamespace, $rel->rd_rel->relname.data, $rel->rd_id + printf " natts: %d, relkind: %c\n", $rel->rd_rel->relnatts, $rel->rd_rel->relkind + else + printf "NULL relation\n" + end + end + document print_relcache + Print relation cache entry information + Usage: print_relcache + end + + define print_tupdesc + set $desc = (TupleDesc)$arg0 + if $desc + printf "TupleDesc: %d attributes\n", $desc->natts + set $i = 0 + while $i < $desc->natts + set $attr = $desc->attrs[$i] + printf " [%d]: %s (type: %u, len: %d)\n", $i, $attr->attname.data, $attr->atttypid, $attr->attlen + set $i = $i + 1 + end + else + printf "NULL tuple descriptor\n" + end + end + document print_tupdesc + Print tuple descriptor information + Usage: print_tupdesc + end + + define print_slot + set $slot = (TupleTableSlot*)$arg0 + if $slot + printf "TupleTableSlot: %s\n", $slot->tts_ops->name + printf " empty: %d, shouldFree: %d\n", $slot->tts_empty, $slot->tts_shouldFree + if $slot->tts_tupleDescriptor + print_tupdesc $slot->tts_tupleDescriptor + end + else + printf "NULL slot\n" + end + end + document print_slot + Print tuple table slot information + Usage: print_slot + end + + # Memory context debugging + define print_mcxt + set $context = (MemoryContext)$arg0 + if $context + printf "MemoryContext: %s\n", $context->name + printf " type: %s, parent: %p\n", $context->methods->name, $context->parent + printf " total: %zu, free: %zu\n", $context->mem_allocated, $context->freep - $context->freeptr + else + printf "NULL memory context\n" + end + end + document print_mcxt + Print memory context information + Usage: print_mcxt + end + + # Process debugging + define print_proc + set $proc = (PGPROC*)$arg0 + if $proc + printf "PGPROC: pid=%d, database=%u\n", $proc->pid, $proc->databaseId + printf " waiting: %d, waitStatus: %d\n", $proc->waiting, $proc->waitStatus + else + printf "NULL process\n" + end + end + document print_proc + Print process information + Usage: print_proc + end + + # Set useful defaults + set print pretty on + set print object on + set print static-members off + set print vtbl on + set print demangle on + set demangle-style gnu-v3 + set print sevenbit-strings off + set history save on + set history size 1000 + set history filename ~/.gdb_history_postgres + + # Common breakpoints for PostgreSQL debugging + define pg_break_common + break elog + break errfinish + break ExceptionalCondition + break ProcessInterrupts + end + document pg_break_common + Set common PostgreSQL debugging breakpoints + end + + printf "PostgreSQL GDB configuration loaded.\n" + printf "Available commands: print_node, print_list, print_query, print_relcache,\n" + printf " print_tupdesc, print_slot, print_mcxt, print_proc, pg_break_common\n" + ''; + + # Flame graph generation script + flameGraphScript = pkgs.writeScriptBin "pg-flame-generate" '' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + DURATION=''${1:-30} + OUTPUT_DIR=''${2:-${config.pgFlameDir}} + TIMESTAMP=$(date +%Y%m%d_%H%M%S) + + mkdir -p "$OUTPUT_DIR" + + echo "Generating flame graph for PostgreSQL (duration: ''${DURATION}s)" + + # Find PostgreSQL processes + PG_PIDS=$(pgrep -f "postgres.*-D.*${config.pgDataDir}" || true) + + if [ -z "$PG_PIDS" ]; then + echo "Error: No PostgreSQL processes found" + exit 1 + fi + + echo "Found PostgreSQL processes: $PG_PIDS" + + # Record perf data + PERF_DATA="$OUTPUT_DIR/perf_$TIMESTAMP.data" + echo "Recording perf data to $PERF_DATA" + + ${pkgs.linuxPackages.perf}/bin/perf record \ + -F 997 \ + -g \ + --call-graph dwarf \ + -p "$(echo $PG_PIDS | tr ' ' ',')" \ + -o "$PERF_DATA" \ + sleep "$DURATION" + + # Generate flame graph + FLAME_SVG="$OUTPUT_DIR/postgres_flame_$TIMESTAMP.svg" + echo "Generating flame graph: $FLAME_SVG" + + ${pkgs.linuxPackages.perf}/bin/perf script -i "$PERF_DATA" | \ + ${pkgs.flamegraph}/bin/stackcollapse-perf.pl | \ + ${pkgs.flamegraph}/bin/flamegraph.pl \ + --title "PostgreSQL Flame Graph ($TIMESTAMP)" \ + --width 1200 \ + --height 800 \ + > "$FLAME_SVG" + + echo "Flame graph generated: $FLAME_SVG" + echo "Perf data saved: $PERF_DATA" + + # Generate summary report + REPORT="$OUTPUT_DIR/report_$TIMESTAMP.txt" + echo "Generating performance report: $REPORT" + + { + echo "PostgreSQL Performance Analysis Report" + echo "Generated: $(date)" + echo "Duration: ''${DURATION}s" + echo "Processes: $PG_PIDS" + echo "" + echo "=== Top Functions ===" + ${pkgs.linuxPackages.perf}/bin/perf report -i "$PERF_DATA" --stdio --sort comm,dso,symbol | head -50 + echo "" + echo "=== Call Graph ===" + ${pkgs.linuxPackages.perf}/bin/perf report -i "$PERF_DATA" --stdio -g --sort comm,dso,symbol | head -100 + } > "$REPORT" + + echo "Report generated: $REPORT" + echo "" + echo "Files created:" + echo " Flame graph: $FLAME_SVG" + echo " Perf data: $PERF_DATA" + echo " Report: $REPORT" + ''; + + # pgbench wrapper script + pgbenchScript = pkgs.writeScriptBin "pg-bench-run" '' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + # Default parameters + CLIENTS=''${1:-10} + THREADS=''${2:-2} + TRANSACTIONS=''${3:-1000} + SCALE=''${4:-10} + DURATION=''${5:-60} + TEST_TYPE=''${6:-tpcb-like} + + OUTPUT_DIR="${config.pgBenchDir}" + TIMESTAMP=$(date +%Y%m%d_%H%M%S) + + mkdir -p "$OUTPUT_DIR" + + echo "=== PostgreSQL Benchmark Configuration ===" + echo "Clients: $CLIENTS" + echo "Threads: $THREADS" + echo "Transactions: $TRANSACTIONS" + echo "Scale factor: $SCALE" + echo "Duration: ''${DURATION}s" + echo "Test type: $TEST_TYPE" + echo "Output directory: $OUTPUT_DIR" + echo "============================================" + + # Check if PostgreSQL is running + if ! pgrep -f "postgres.*-D.*${config.pgDataDir}" >/dev/null; then + echo "Error: PostgreSQL is not running. Start it with 'pg-start'" + exit 1 + fi + + PGBENCH="${config.pgInstallDir}/bin/pgbench" + PSQL="${config.pgInstallDir}/bin/psql" + CREATEDB="${config.pgInstallDir}/bin/createdb" + DROPDB="${config.pgInstallDir}/bin/dropdb" + + DB_NAME="pgbench_test_$TIMESTAMP" + RESULTS_FILE="$OUTPUT_DIR/results_$TIMESTAMP.txt" + LOG_FILE="$OUTPUT_DIR/pgbench_$TIMESTAMP.log" + + echo "Creating test database: $DB_NAME" + "$CREATEDB" -h "${config.pgDataDir}" "$DB_NAME" || { + echo "Failed to create database" + exit 1 + } + + # Initialize pgbench tables + echo "Initializing pgbench tables (scale factor: $SCALE)" + "$PGBENCH" -h "${config.pgDataDir}" -i -s "$SCALE" "$DB_NAME" || { + echo "Failed to initialize pgbench tables" + "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true + exit 1 + } + + # Run benchmark based on test type + echo "Running benchmark..." + + case "$TEST_TYPE" in + "tpcb-like"|"default") + BENCH_ARGS="" + ;; + "select-only") + BENCH_ARGS="-S" + ;; + "simple-update") + BENCH_ARGS="-N" + ;; + "read-write") + BENCH_ARGS="-b select-only@70 -b tpcb-like@30" + ;; + *) + echo "Unknown test type: $TEST_TYPE" + echo "Available types: tpcb-like, select-only, simple-update, read-write" + "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true + exit 1 + ;; + esac + + { + echo "PostgreSQL Benchmark Results" + echo "Generated: $(date)" + echo "Test type: $TEST_TYPE" + echo "Clients: $CLIENTS, Threads: $THREADS" + echo "Transactions: $TRANSACTIONS, Duration: ''${DURATION}s" + echo "Scale factor: $SCALE" + echo "Database: $DB_NAME" + echo "" + echo "=== System Information ===" + echo "CPU: $(nproc) cores" + echo "Memory: $(free -h | grep '^Mem:' | awk '{print $2}')" + echo "Compiler: $CC" + echo "PostgreSQL version: $("$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -t -c "SELECT version();" | head -1)" + echo "" + echo "=== Benchmark Results ===" + } > "$RESULTS_FILE" + + # Run the actual benchmark + "$PGBENCH" \ + -h "${config.pgDataDir}" \ + -c "$CLIENTS" \ + -j "$THREADS" \ + -T "$DURATION" \ + -P 5 \ + --log \ + --log-prefix="$OUTPUT_DIR/pgbench_$TIMESTAMP" \ + $BENCH_ARGS \ + "$DB_NAME" 2>&1 | tee -a "$RESULTS_FILE" + + # Collect additional statistics + { + echo "" + echo "=== Database Statistics ===" + "$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -c " + SELECT + schemaname, + relname, + n_tup_ins as inserts, + n_tup_upd as updates, + n_tup_del as deletes, + n_live_tup as live_tuples, + n_dead_tup as dead_tuples + FROM pg_stat_user_tables; + " + + echo "" + echo "=== Index Statistics ===" + "$PSQL" --no-psqlrc -h "${config.pgDataDir}" -d "$DB_NAME" -c " + SELECT + schemaname, + relname, + indexrelname, + idx_scan, + idx_tup_read, + idx_tup_fetch + FROM pg_stat_user_indexes; + " + } >> "$RESULTS_FILE" + + # Clean up + echo "Cleaning up test database: $DB_NAME" + "$DROPDB" -h "${config.pgDataDir}" "$DB_NAME" 2>/dev/null || true + + echo "" + echo "Benchmark completed!" + echo "Results saved to: $RESULTS_FILE" + echo "Transaction logs: $OUTPUT_DIR/pgbench_$TIMESTAMP*" + + # Show summary + echo "" + echo "=== Quick Summary ===" + grep -E "(tps|latency)" "$RESULTS_FILE" | tail -5 + ''; + + # Shared shellHook fragments. Each devShell prepends its own compiler/CFLAGS + # block, then appends the common tail via ${commonHookTail variant}. + commonHookHead = icon: '' + # History configuration + export HISTFILE=.history + export HISTSIZE=1000000 + export HISTFILESIZE=1000000 + + # Clean environment + unset LD_LIBRARY_PATH LD_PRELOAD LIBRARY_PATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH + + # Essential tools in PATH + export PATH="${pkgs.which}/bin:${pkgs.coreutils}/bin:$PATH" + export PS1="$(echo -e '\u${icon}') {\[$(tput sgr0)\]\[\033[38;5;228m\]\w\[$(tput sgr0)\]\[\033[38;5;15m\]} ($(git rev-parse --abbrev-ref HEAD)) \\$ \[$(tput sgr0)\]" + + # Ccache configuration + export PATH=${pkgs.ccache}/bin:$PATH + export CCACHE_COMPILERCHECK=content + # Loosen a few rules so ccache hits across rebuilds with touched headers. + export CCACHE_SLOPPINESS=pch_defines,time_macros,include_file_mtime,include_file_ctime + export CCACHE_DIR=$HOME/.ccache/pg/$(basename $PWD) + mkdir -p "$CCACHE_DIR" + + # Development tools in PATH + export PATH=${pkgs.clang-tools}/bin:$PATH + export PATH=${pkgs.cppcheck}/bin:$PATH + ''; + + # Tail shared by every devShell: PG env vars, GDB, tool PATH, per-process + # setup and alias load. Kernel core_pattern is NOT touched here -- + # run 'pg-enable-cores' explicitly if you need per-PID cores in CWD. + commonHookTail = label: '' + # PostgreSQL environment + export PG_SOURCE_DIR="${config.pgSourceDir}" + export PG_BUILD_DIR="${config.pgBuildDir}" + export PG_BUILD_DIR_VALGRIND="${config.pgBuildDirValgrind}" + export PG_BUILD_DIR_ASAN="${config.pgBuildDirAsan}" + export PG_INSTALL_DIR="${config.pgInstallDir}" + export PG_DATA_DIR="${config.pgDataDir}" + export PG_BENCH_DIR="${config.pgBenchDir}" + export PG_FLAME_DIR="${config.pgFlameDir}" + export PERL_CORE_DIR=$(find ${pkgs.perl} -maxdepth 5 -path "*/CORE" -type d) + + # GDB configuration + export GDBINIT="${gdbConfig}" + + # Performance tools in PATH + export PATH="${flameGraphScript}/bin:${pgbenchScript}/bin:$PATH" + + # Create output directories + mkdir -p "$PG_BENCH_DIR" "$PG_FLAME_DIR" + + # Per-process core dump size limit. Kernel core_pattern is NOT + # touched here -- run 'pg-enable-cores' explicitly when you need + # per-PID cores in CWD. + ulimit -c unlimited + + # Local git excludes + git config core.excludesFile .local-gitignore 2>/dev/null || true + + # Load PostgreSQL development aliases + if [ -f ./pg-aliases.sh ]; then + source ./pg-aliases.sh + else + echo "Warning: pg-aliases.sh not found in current directory" + fi + + echo "" + echo "PostgreSQL Development Environment Ready (${label})" + echo "Run 'pg-info' for available commands" + ''; + + # Development shell (GCC + glibc) + devShell = pkgs.mkShell { + name = "postgresql-dev"; + buildInputs = + (getPostgreSQLDeps false) + ++ [ + flameGraphScript + pgbenchScript + ]; + + shellHook = + (commonHookHead "f121") + + '' + # LLVM configuration + export LLVM_CONFIG="${llvmPkgs.llvm}/bin/llvm-config" + export PATH="${llvmPkgs.llvm}/bin:$PATH" + export PKG_CONFIG_PATH="${llvmPkgs.llvm.dev}/lib/pkgconfig:$PKG_CONFIG_PATH" + export LLVM_DIR="${llvmPkgs.llvm.dev}/lib/cmake/llvm" + export LLVM_ROOT="${llvmPkgs.llvm}" + + # PostgreSQL Development CFLAGS + export CFLAGS="" + export CXXFLAGS="" + + # Python UV + UV_PYTHON_DOWNLOADS=never + + # GCC configuration (default compiler) + export CC="${pkgs.gcc}/bin/gcc" + export CXX="${pkgs.gcc}/bin/g++" + + echo "Environment configured:" + echo " Compiler: $CC" + echo " libc: glibc" + echo " LLVM: $(llvm-config --version 2>/dev/null || echo 'not available')" + '' + + (commonHookTail "GCC + glibc"); + }; + + # Clang + glibc variant + clangDevShell = pkgs.mkShell { + name = "postgresql-clang-glibc"; + buildInputs = + (getPostgreSQLDeps false) + ++ [ + llvmPkgs.clang + llvmPkgs.lld + llvmPkgs.compiler-rt + flameGraphScript + pgbenchScript + ]; + + shellHook = + (commonHookHead "f121") + + '' + # LLVM configuration + export LLVM_CONFIG="${llvmPkgs.llvm}/bin/llvm-config" + export PATH="${llvmPkgs.llvm}/bin:$PATH" + export PKG_CONFIG_PATH="${llvmPkgs.llvm.dev}/lib/pkgconfig:$PKG_CONFIG_PATH" + export LLVM_DIR="${llvmPkgs.llvm.dev}/lib/cmake/llvm" + export LLVM_ROOT="${llvmPkgs.llvm}" + + # Clang + glibc configuration + export CC="${llvmPkgs.clang}/bin/clang" + export CXX="${llvmPkgs.clang}/bin/clang++" + + echo "Environment configured:" + echo " Compiler: $CC" + echo " libc: glibc" + echo " LLVM: $(llvm-config --version 2>/dev/null || echo 'not available')" + '' + + (commonHookTail "Clang + glibc"); + }; + + # GCC + musl variant (cross-compilation) + muslDevShell = pkgs.mkShell { + name = "postgresql-gcc-musl"; + buildInputs = + (getPostgreSQLDeps true) + ++ [ + pkgs.gcc + flameGraphScript + pgbenchScript + ]; + + shellHook = + (commonHookHead "f121") + + '' + # Cross-compilation to musl with GCC + export CC="${pkgs.gcc}/bin/gcc" + export CXX="${pkgs.gcc}/bin/g++" + + export PKG_CONFIG_PATH="${pkgs.pkgsMusl.openssl.dev}/lib/pkgconfig:${pkgs.pkgsMusl.zlib.dev}/lib/pkgconfig:${pkgs.pkgsMusl.icu.dev}/lib/pkgconfig" + export CFLAGS="-ggdb -Og -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export CXXFLAGS="-ggdb -Og -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export LDFLAGS="-L${pkgs.pkgsMusl.stdenv.cc.libc}/lib -static-libgcc" + + echo "Environment configured:" + echo " Compiler: $CC" + echo " libc: musl (cross-compilation)" + '' + + (commonHookTail "GCC + musl"); + }; + + # Clang + musl variant (cross-compilation) + clangMuslDevShell = pkgs.mkShell { + name = "postgresql-clang-musl"; + buildInputs = + (getPostgreSQLDeps true) + ++ [ + llvmPkgs.clang + llvmPkgs.lld + flameGraphScript + pgbenchScript + ]; + + shellHook = + (commonHookHead "f121") + + '' + # Cross-compilation to musl with clang + export CC="${llvmPkgs.clang}/bin/clang" + export CXX="${llvmPkgs.clang}/bin/clang++" + + export PKG_CONFIG_PATH="${pkgs.pkgsMusl.openssl.dev}/lib/pkgconfig:${pkgs.pkgsMusl.zlib.dev}/lib/pkgconfig:${pkgs.pkgsMusl.icu.dev}/lib/pkgconfig" + export CFLAGS="--target=x86_64-linux-musl -ggdb -Og -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export CXXFLAGS="--target=x86_64-linux-musl -ggdb -Og -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -I${pkgs.pkgsMusl.stdenv.cc.libc}/include" + export LDFLAGS="--target=x86_64-linux-musl -L${pkgs.pkgsMusl.stdenv.cc.libc}/lib -fuse-ld=lld" + + echo "Environment configured:" + echo " Compiler: $CC" + echo " libc: musl (cross-compilation)" + '' + + (commonHookTail "Clang + musl"); + }; +in { + inherit devShell clangDevShell muslDevShell clangMuslDevShell gdbConfig flameGraphScript pgbenchScript; +} diff --git a/src/backend/port/sysv_shmem.c b/src/backend/port/sysv_shmem.c index 2e3886cf9fe49..2877f21308e55 100644 --- a/src/backend/port/sysv_shmem.c +++ b/src/backend/port/sysv_shmem.c @@ -27,6 +27,10 @@ #include #include +#ifdef USE_LIBNUMA +#include +#endif + #include "miscadmin.h" #include "port/pg_bitutils.h" #include "portability/mem.h" @@ -604,6 +608,27 @@ CreateAnonymousSegment(Size *size) int mmap_errno = 0; int mmap_flags = MAP_SHARED | MAP_ANONYMOUS | MAP_HASSEMAPHORE; + /* + * If requested, populate the shared memory by MAP_POPULATE. The flag is + * Linux-specific, so platforms without it simply fault the pages in on + * first touch, which is the pre-existing behaviour. + */ +#ifdef MAP_POPULATE + if (shmem_populate) + mmap_flags |= MAP_POPULATE; +#endif + +#ifdef USE_LIBNUMA + /* + * If requested, interleave the shared memory by setting a memory policy + * before the mmap() call. This really matters only with MAP_POPULATE, + * because without page faults the memory does not actually get placed + * to the nodes. But without MAP_POPULATE it's virtually free. + */ + if (shmem_interleave) + numa_set_interleave_mask(numa_all_nodes_ptr); +#endif + #ifndef MAP_HUGETLB /* PGSharedMemoryCreate should have dealt with this case */ Assert(huge_pages != HUGE_PAGES_ON); @@ -665,6 +690,30 @@ CreateAnonymousSegment(Size *size) allocsize) : 0)); } +#ifdef USE_LIBNUMA + /* + * If set the policy to interleaving by numa_set_membind(), undo it now by + * setting the policy to localalloc. With MAP_POPULATE, all the pages were + * faulted and are now interleaved on the available nodes. + * + * To handle the case without MAP_POPULATE, apply the interleaving policy to + * the shared memory segment allocated by mmap() before touching it in any + * way, so that it gets placed on the correct node on first access. + * + * This matters especially with huge pages, where it's possible to run out + * of huge pages on some nodes and then crash. By explicitly interleaving + * the whole segment, that's much less likely. + */ + if (shmem_interleave) + { + /* undo the policy set by numa_set_membind() earlier */ + numa_set_localalloc(); + + /* set interleaving policy for not yet faulted memory */ + numa_interleave_memory(ptr, allocsize, numa_all_nodes_ptr); + } +#endif + *size = allocsize; return ptr; } diff --git a/src/backend/storage/buffer/buf_init.c b/src/backend/storage/buffer/buf_init.c index 9c5f2449cf399..8d49eb7b1bf27 100644 --- a/src/backend/storage/buffer/buf_init.c +++ b/src/backend/storage/buffer/buf_init.c @@ -14,20 +14,30 @@ */ #include "postgres.h" +#ifdef USE_LIBNUMA +#include +#include +#endif + +#include "port/pg_numa.h" #include "storage/aio.h" #include "storage/buf_internals.h" #include "storage/bufmgr.h" #include "storage/proclist.h" #include "storage/shmem.h" #include "storage/subsystems.h" +#include "utils/guc_hooks.h" +#include "utils/varlena.h" BufferDescPadded *BufferDescriptors; char *BufferBlocks; ConditionVariableMinimallyPadded *BufferIOCVArray; CkptSortItem *CkptBufferIds; +BufferPartitions *BufferPartitionsRegistry; static void BufferManagerShmemRequest(void *arg); static void BufferManagerShmemInit(void *arg); +static void BufferPartitionsInit(void); const ShmemCallbacks BufferManagerShmemCallbacks = { .request_fn = BufferManagerShmemRequest, @@ -66,6 +76,12 @@ const ShmemCallbacks BufferManagerShmemCallbacks = { * multiple times. Check the PrivateRefCount infrastructure in bufmgr.c. */ +/* + * Minimum number of buffer partitions, no matter the number of NUMA nodes. + */ +#define MIN_BUFFER_PARTITIONS 4 + +bool shared_buffers_numa = false; /* * Register shared memory area for the buffer pool. @@ -73,6 +89,10 @@ const ShmemCallbacks BufferManagerShmemCallbacks = { static void BufferManagerShmemRequest(void *arg) { + int nparts; + + BufferPartitionsCalculate(NULL, &nparts, NULL); + ShmemRequestStruct(.name = "Buffer Descriptors", .size = NBuffers * sizeof(BufferDescPadded), /* Align descriptors to a cacheline boundary. */ @@ -94,6 +114,13 @@ BufferManagerShmemRequest(void *arg) .ptr = (void **) &BufferIOCVArray, ); + ShmemRequestStruct(.name = "Buffer Partition Registry", + .size = nparts * sizeof(BufferPartition), + /* Align descriptors to a cacheline boundary. */ + .alignment = PG_CACHE_LINE_SIZE, + .ptr = (void **) &BufferPartitionsRegistry, + ); + /* * The array used to sort to-be-checkpointed buffer ids is located in * shared memory, to avoid having to allocate significant amounts of @@ -116,6 +143,16 @@ BufferManagerShmemRequest(void *arg) static void BufferManagerShmemInit(void *arg) { + /* + * Initialize the buffer partition registry first, before other parts + * have a chance to touch the memory. + * + * Also moves memory to different NUMA nodes (if enabled by a GUC). + * Do this before the loop that initializes buffer headers etc. which + * may fault some of the memory pages etc. + */ + BufferPartitionsInit(); + /* * Initialize all the buffer headers. */ @@ -136,3 +173,317 @@ BufferManagerShmemInit(void *arg) ConditionVariableInit(BufferDescriptorGetIOCV(buf)); } } + +/* + * Sanity checks of buffers partitions - there must be no gaps, it must cover + * the whole range of buffers, etc. + */ +static void +AssertCheckBufferPartitions(void) +{ +#ifdef USE_ASSERT_CHECKING + int num_buffers = 0; + + Assert(BufferPartitionsRegistry->npartitions > 0); + + for (int i = 0; i < BufferPartitionsRegistry->npartitions; i++) + { + BufferPartition *part = &BufferPartitionsRegistry->partitions[i]; + + /* + * We can get a single-buffer partition, if the sizing forces the last + * partition to be just one buffer. But it's unlikely (and + * undesirable). + */ + Assert(part->first_buffer <= part->last_buffer); + Assert((part->last_buffer - part->first_buffer + 1) == part->num_buffers); + + num_buffers += part->num_buffers; + + /* + * The first partition needs to start on buffer 0. Later partitions + * need to be contiguous, without skipping any buffers. + */ + if (i == 0) + { + Assert(part->first_buffer == 0); + } + else + { + BufferPartition *prev = &BufferPartitionsRegistry->partitions[i - 1]; + + Assert((part->first_buffer - 1) == prev->last_buffer); + } + + /* the last partition needs to end on buffer (NBuffers - 1) */ + if (i == (BufferPartitionsRegistry->npartitions - 1)) + { + Assert(part->last_buffer == (NBuffers - 1)); + } + } + + Assert(num_buffers == NBuffers); +#endif +} + +/* + * BufferPartitionsInit + * Initialize registry of buffer partitions. + */ +static void +BufferPartitionsInit(void) +{ + int buffer = 0; + + int nnodes, + npartitions, + npartitions_per_node; + + int buffers_per_partition, + buffers_remaining; + + /* calculate partitioning parameters */ + BufferPartitionsCalculate(&nnodes, &npartitions, &npartitions_per_node); + + /* paranoia */ + Assert(nnodes > 0); + Assert(npartitions >= MIN_BUFFER_PARTITIONS); + Assert((npartitions % nnodes) == 0); + Assert((npartitions_per_node * nnodes) == npartitions); + + BufferPartitionsRegistry->nnodes = nnodes; + BufferPartitionsRegistry->npartitions = npartitions; + BufferPartitionsRegistry->npartitions_per_node = npartitions_per_node; + + /* regular partition size, the first couple get an extra buffer */ + buffers_per_partition = (NBuffers / npartitions); + buffers_remaining = (NBuffers % buffers_per_partition); + + /* should have all the buffers */ + Assert((buffers_per_partition * npartitions + buffers_remaining) == NBuffers); + + /* + * Now walk the partitions, and set the buffer range. Optionally, place + * the partitions on a given node (for all partitions at once). + */ + for (int n = 0; n < nnodes; n++) + { + for (int p = 0; p < npartitions_per_node; p++) + { + int idx = (n * npartitions_per_node) + p; + BufferPartition *part = &BufferPartitionsRegistry->partitions[idx]; + + /* + * Assign to the NUMA node, but only with shared_buffers_numa=on. + * + * XXX we should get an actual node ID from the mask, in case the + * task is restricted to only some nodes. + */ + part->numa_node = (shared_buffers_numa) ? n : -1; + + /* The first couple partitions may get an extra buffer. */ + part->num_buffers = buffers_per_partition; + if (idx < buffers_remaining) + part->num_buffers += 1; + + /* remember the buffer range */ + part->first_buffer = buffer; + part->last_buffer = buffer + (part->num_buffers - 1); + + /* remember start of the next partition */ + buffer += part->num_buffers; + } + +#ifdef USE_LIBNUMA + /* + * Now try to locate buffers and buffer descriptors to the node (all + * partitions for the node at once). + */ + if (shared_buffers_numa) + { + Size numa_page_size = pg_numa_page_size(); + + int part_first, + part_last, + buff_first, + buff_last; + + char *startptr, + *endptr; + + /* first/last partition for this node */ + part_first = (n * npartitions_per_node); + part_last = part_first + (npartitions_per_node - 1); + + /* buffers (blocks) */ + + /* first/last buffer */ + buff_first = BufferPartitionsRegistry->partitions[part_first].first_buffer; + buff_last = BufferPartitionsRegistry->partitions[part_last].last_buffer; + + /* beginning of the first block, end of last block */ + startptr = BufferBlocks + ((Size) buff_first * BLCKSZ); + endptr = BufferBlocks + ((Size) (buff_last + 1) * BLCKSZ); + + /* print some warnings when the partitions are not aligned */ + if ((startptr != (char *) TYPEALIGN(numa_page_size, startptr)) || + (endptr != (char *) TYPEALIGN(numa_page_size, endptr))) + { + elog(WARNING, "buffers for node %d not well aligned [%p,%p]", + n, startptr, endptr); + } + + /* best effort: align the pointers, so that the mbind() works */ + startptr = (char *) TYPEALIGN_DOWN(numa_page_size, startptr); + + /* the last partition aligns to the end of the buffer */ + if (n == (nnodes - 1)) + endptr = (char *) TYPEALIGN(numa_page_size, endptr); + else + endptr = (char *) TYPEALIGN_DOWN(numa_page_size, endptr); + + /* XXX or should we use pg_numa_move_to_node? */ + if (pg_numa_bind_to_node(startptr, endptr, n) != 0) + elog(WARNING, "failed to bind shared buffers partition to node %d", n); + + /* buffer descriptors */ + + /* beginning of the first descriptor, end of last descriptor */ + startptr = (char *) &BufferDescriptors[buff_first]; + endptr = (char *) &BufferDescriptors[buff_last] + 1; + + /* print some warnings when the partitions are not aligned */ + if ((startptr != (char *) TYPEALIGN(numa_page_size, startptr)) || + (endptr != (char *) TYPEALIGN(numa_page_size, endptr))) + { + elog(WARNING, "buffers descriptors for node %d not well aligned [%p,%p]", + n, startptr, endptr); + } + + /* best effort: align the pointers, so that the mbind() works */ + startptr = (char *) TYPEALIGN_DOWN(numa_page_size, startptr); + + if (n == (nnodes - 1)) + endptr = (char *) TYPEALIGN(numa_page_size, endptr); + else + endptr = (char *) TYPEALIGN_DOWN(numa_page_size, endptr); + + /* XXX or should we use pg_numa_move_to_node? */ + if (pg_numa_bind_to_node(startptr, endptr, n) != 0) + elog(WARNING, "failed to bind shared buffer descriptors partition to node %d", n); + } +#endif + } + + AssertCheckBufferPartitions(); +} + +/* + * BufferPartitionsCalculate + * Pick number of buffer partitions for the number of nodes and + * MIN_BUFFER_PARTITIONS. + * + * Picks the smallest number of partitions higher thah MIN_BUFFER_PARTITIONS, + * such that all nodes have the same number of partitions. + * + * This is best-effort with respect to size of the partitions. It's possible + * the partitions are not a perfect multiple of page size, in which case + * we set location only for the part where that is possible. The buffers on + * the "boundary" may get located up on arbitrary nodes. + * + * The extra complexity of figuring out the right "partition size" is not + * worth it, and it can lead to some partitions being much smaller. This way + * we end up with partitions of almost exactly the same size (one BLCKSZ is + * the largest difference). + * + * We expect shared buffers to be much larger than page size (at least on + * system where NUMA is a relevant feature), so the number of "not located" + * buffers should be a negligible fraction. This only affects pages between + * partitions for different nodes, so (nodes-1) pages. This is certainly + * fine with 2MB huge pages, but even with 1GB pages it should be OK (as + * such systems should have humongous amounts of memory). + * + * It also means we don't need to worry about memory page size before knowing + * if huge pages got used (which we only learn during allocation). + */ +void +BufferPartitionsCalculate(int *num_nodes, int *num_partitions, + int *num_partitions_per_node) +{ + int nnodes, + nparts, + nparts_per_node; + +#if USE_LIBNUMA + nnodes = numa_num_configured_nodes(); + nparts_per_node = 1; /* at least one partition per node */ + + while ((nparts_per_node * nnodes) < MIN_BUFFER_PARTITIONS) + nparts_per_node++; + + nparts = (nnodes * nparts_per_node); +#else + /* without NUMA, assume there's just one node */ + nnodes = 1; + nparts = MIN_BUFFER_PARTITIONS; + nparts_per_node = MIN_BUFFER_PARTITIONS; +#endif + + if (num_nodes) + *num_nodes = nnodes; + + if (num_partitions) + *num_partitions = nparts; + + if (num_partitions_per_node) + *num_partitions_per_node = nparts_per_node; +} + +/* + * BufferPartitionCount + * Returns the number of partitions created. + */ +int +BufferPartitionCount(void) +{ + return BufferPartitionsRegistry->npartitions; +} + +/* + * BufferPartitionGet + * Returns information about a partition at the provided index. + * + * The returned information is first/last buffer, number of buffers. + */ +void +BufferPartitionGet(int idx, int *node, int *num_buffers, + int *first_buffer, int *last_buffer) +{ + if ((idx >= 0) && (idx < BufferPartitionsRegistry->npartitions)) + { + BufferPartition *part = &BufferPartitionsRegistry->partitions[idx]; + + *node = part->numa_node; + *num_buffers = part->num_buffers; + *first_buffer = part->first_buffer; + *last_buffer = part->last_buffer; + + return; + } + + elog(ERROR, "invalid partition index"); +} + +void +BufferPartitionsParams(int *num_nodes, int *num_partitions, + int *num_partitions_per_node) +{ + if (num_nodes) + *num_nodes = BufferPartitionsRegistry->nnodes; + + if (num_partitions) + *num_partitions = BufferPartitionsRegistry->npartitions; + + if (num_partitions_per_node) + *num_partitions_per_node = BufferPartitionsRegistry->npartitions_per_node; +} diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index f81c7732e6899..30bbc6a2af38e 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -636,7 +636,7 @@ static void PinBuffer_Locked(BufferDesc *buf); static void UnpinBuffer(BufferDesc *buf); static void UnpinBufferNoOwner(BufferDesc *buf); static void BufferSync(int flags); -static int SyncOneBuffer(int buf_id, bool skip_recently_used, +static int SyncOneBuffer(int buf_id, bool skip_recently_used, bool spend_chance, WritebackContext *wb_context); static void WaitIO(BufferDesc *buf); static void AbortBufferIO(Buffer buffer); @@ -2335,7 +2335,11 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * checkpoints, except for their "init" forks, which need to be treated * just like permanent relations. */ - set_bits |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + /* + * Admit the page on probation: no second chance yet, so the next pass of + * the hand may reclaim it. A further access promotes it (see PinBuffer). + */ + set_bits |= BM_TAG_VALID; if (relpersistence == RELPERSISTENCE_PERMANENT || forkNum == INIT_FORKNUM) set_bits |= BM_PERMANENT; @@ -2445,7 +2449,7 @@ InvalidateBuffer(BufferDesc *buf) UnlockBufHdrExt(buf, buf_state, 0, - BUF_FLAG_MASK | BUF_USAGECOUNT_MASK, + BUF_FLAG_MASK, 0); /* @@ -2528,7 +2532,7 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) ClearBufferTag(&buf_hdr->tag); UnlockBufHdrExt(buf_hdr, buf_state, 0, - BUF_FLAG_MASK | BUF_USAGECOUNT_MASK, + BUF_FLAG_MASK, 0); Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); @@ -3004,7 +3008,8 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, victim_buf_hdr->tag = tag; - set_bits |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + /* Admit on probation; see the comment at the other admission site. */ + set_bits |= BM_TAG_VALID; if (bmr.relpersistence == RELPERSISTENCE_PERMANENT || fork == INIT_FORKNUM) set_bits |= BM_PERMANENT; @@ -3336,18 +3341,21 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy, if (strategy == NULL) { - /* Default case: increase usagecount unless already max. */ - if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT) - buf_state += BUF_USAGECOUNT_ONE; + /* Default case: grant the buffer its second chance. */ + buf_state |= BM_SECOND_CHANCE; } else { /* - * Ring buffers shouldn't evict others from pool. Thus we - * don't make usagecount more than 1. + * Ring buffers: a page is pinned once per cycle through the + * ring by the scan that owns it, and those pins must not keep + * it resident -- otherwise a sequential scan would protect its + * own pages and evict the working set. With a single bit + * there is nothing to cap: leave the bit alone and let the + * ring recycle the page, while an access from outside the ring + * (which takes the default path above) grants the chance and + * so removes the page from the ring's reach. */ - if (BUF_STATE_GET_USAGECOUNT(buf_state) == 0) - buf_state += BUF_USAGECOUNT_ONE; } if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state, @@ -3787,7 +3795,7 @@ BufferSync(int flags) */ if (pg_atomic_read_u64(&bufHdr->state) & BM_CHECKPOINT_NEEDED) { - if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) + if (SyncOneBuffer(buf_id, false, false, &wb_context) & BUF_WRITTEN) { TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id); PendingCheckpointerStats.buffers_written++; @@ -3842,33 +3850,34 @@ BufferSync(int flags) } /* - * BgBufferSync -- Write out some dirty buffers in the pool. + * Information saved between calls so we can determine the strategy point's + * advance rate and avoid scanning already-cleaned buffers. * - * This is called periodically by the background writer process. - * - * Returns true if it's appropriate for the bgwriter process to go into - * low-power hibernation mode. (This happens if the strategy clock-sweep - * has been "lapped" and no buffer allocations have occurred recently, - * or if the bgwriter has been effectively disabled by setting - * bgwriter_lru_maxpages to 0.) + * XXX Does it actually make sense to split all of this information per + * partition? For example, does per-partition advance rate mean anything? + * Maybe we should have a global advance rate? Although, if we want to + * keep enough clean buffers in each partition, maybe having per-partition + * rates makes sense. */ -bool -BgBufferSync(WritebackContext *wb_context) +typedef struct BufferSyncPartition +{ + int prev_strategy_buf_id; + uint32 prev_strategy_passes; + int next_to_clean; + uint32 next_passes; +} BufferSyncPartition; + +static BufferSyncPartition *saved_info = NULL; +static bool saved_info_valid = false; + +static bool +BgBufferSyncPartition(WritebackContext *wb_context, int num_partitions, + int partition, int recent_alloc_partition, + BufferSyncPartition *saved) { /* info obtained from freelist.c */ int strategy_buf_id; uint32 strategy_passes; - uint32 recent_alloc; - - /* - * Information saved between calls so we can determine the strategy - * point's advance rate and avoid scanning already-cleaned buffers. - */ - static bool saved_info_valid = false; - static int prev_strategy_buf_id; - static uint32 prev_strategy_passes; - static int next_to_clean; - static uint32 next_passes; /* Moving averages of allocation rate and clean-buffer density */ static float smoothed_alloc = 0; @@ -3891,30 +3900,30 @@ BgBufferSync(WritebackContext *wb_context) int num_to_scan; int num_written; int reusable_buffers; + bool spend_chance; + uint64 sweep_claims; + + /* + * Claim count as of our previous cycle, to get a rate rather than a total. + * One entry per partition: each has its own clock hand and can starve + * independently, so the decision is made for each one separately. + */ + static uint64 *prev_sweep_claims = NULL; /* Variables for final smoothed_density update */ long new_strategy_delta; uint32 new_recent_alloc; + /* buffer range for the clocksweep partition */ + int first_buffer; + int num_buffers; + /* * Find out where the clock-sweep currently is, and how many buffer * allocations have happened since our last call. */ - strategy_buf_id = StrategySyncStart(&strategy_passes, &recent_alloc); - - /* Report buffer alloc counts to pgstat */ - PendingBgWriterStats.buf_alloc += recent_alloc; - - /* - * If we're not running the LRU scan, just stop after doing the stats - * stuff. We mark the saved state invalid so that we can recover sanely - * if LRU scan is turned back on later. - */ - if (bgwriter_lru_maxpages <= 0) - { - saved_info_valid = false; - return true; - } + strategy_buf_id = StrategySyncStart(partition, &strategy_passes, + &first_buffer, &num_buffers); /* * Compute strategy_delta = how many buffers have been scanned by the @@ -3926,17 +3935,17 @@ BgBufferSync(WritebackContext *wb_context) */ if (saved_info_valid) { - int32 passes_delta = strategy_passes - prev_strategy_passes; + int32 passes_delta = strategy_passes - saved->prev_strategy_passes; - strategy_delta = strategy_buf_id - prev_strategy_buf_id; - strategy_delta += (long) passes_delta * NBuffers; + strategy_delta = strategy_buf_id - saved->prev_strategy_buf_id; + strategy_delta += (long) passes_delta * num_buffers; Assert(strategy_delta >= 0); - if ((int32) (next_passes - strategy_passes) > 0) + if ((int32) (saved->next_passes - strategy_passes) > 0) { /* we're one pass ahead of the strategy point */ - bufs_to_lap = strategy_buf_id - next_to_clean; + bufs_to_lap = strategy_buf_id - saved->next_to_clean; #ifdef BGW_DEBUG elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d", next_passes, next_to_clean, @@ -3944,11 +3953,11 @@ BgBufferSync(WritebackContext *wb_context) strategy_delta, bufs_to_lap); #endif } - else if (next_passes == strategy_passes && - next_to_clean >= strategy_buf_id) + else if (saved->next_passes == strategy_passes && + saved->next_to_clean >= strategy_buf_id) { /* on same pass, but ahead or at least not behind */ - bufs_to_lap = NBuffers - (next_to_clean - strategy_buf_id); + bufs_to_lap = num_buffers - (saved->next_to_clean - strategy_buf_id); #ifdef BGW_DEBUG elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d", next_passes, next_to_clean, @@ -3968,9 +3977,9 @@ BgBufferSync(WritebackContext *wb_context) strategy_passes, strategy_buf_id, strategy_delta); #endif - next_to_clean = strategy_buf_id; - next_passes = strategy_passes; - bufs_to_lap = NBuffers; + saved->next_to_clean = strategy_buf_id; + saved->next_passes = strategy_passes; + bufs_to_lap = num_buffers; } } else @@ -3984,15 +3993,16 @@ BgBufferSync(WritebackContext *wb_context) strategy_passes, strategy_buf_id); #endif strategy_delta = 0; - next_to_clean = strategy_buf_id; - next_passes = strategy_passes; - bufs_to_lap = NBuffers; + saved->next_to_clean = strategy_buf_id; + saved->next_passes = strategy_passes; + bufs_to_lap = num_buffers; } /* Update saved info for next time */ - prev_strategy_buf_id = strategy_buf_id; - prev_strategy_passes = strategy_passes; - saved_info_valid = true; + saved->prev_strategy_buf_id = strategy_buf_id; + saved->prev_strategy_passes = strategy_passes; + /* XXX this needs to happen only after all partitions */ + /* saved_info_valid = true; */ /* * Compute how many buffers had to be scanned for each new allocation, ie, @@ -4000,9 +4010,9 @@ BgBufferSync(WritebackContext *wb_context) * * If the strategy point didn't move, we don't update the density estimate */ - if (strategy_delta > 0 && recent_alloc > 0) + if (strategy_delta > 0 && recent_alloc_partition > 0) { - scans_per_alloc = (float) strategy_delta / (float) recent_alloc; + scans_per_alloc = (float) strategy_delta / (float) recent_alloc_partition; smoothed_density += (scans_per_alloc - smoothed_density) / smoothing_samples; } @@ -4012,7 +4022,7 @@ BgBufferSync(WritebackContext *wb_context) * strategy point and where we've scanned ahead to, based on the smoothed * density estimate. */ - bufs_ahead = NBuffers - bufs_to_lap; + bufs_ahead = num_buffers - bufs_to_lap; reusable_buffers_est = (float) bufs_ahead / smoothed_density; /* @@ -4020,10 +4030,10 @@ BgBufferSync(WritebackContext *wb_context) * a true average we want a fast-attack, slow-decline behavior: we * immediately follow any increase. */ - if (smoothed_alloc <= (float) recent_alloc) - smoothed_alloc = recent_alloc; + if (smoothed_alloc <= (float) recent_alloc_partition) + smoothed_alloc = recent_alloc_partition; else - smoothed_alloc += ((float) recent_alloc - smoothed_alloc) / + smoothed_alloc += ((float) recent_alloc_partition - smoothed_alloc) / smoothing_samples; /* Scale the estimate by a GUC to allow more aggressive tuning. */ @@ -4050,7 +4060,7 @@ BgBufferSync(WritebackContext *wb_context) * the BGW will be called during the scan_whole_pool time; slice the * buffer pool into that many sections. */ - min_scan_buffers = (int) (NBuffers / (scan_whole_pool_milliseconds / BgWriterDelay)); + min_scan_buffers = (int) (num_buffers / (scan_whole_pool_milliseconds / BgWriterDelay)); if (upcoming_alloc_est < (min_scan_buffers + reusable_buffers_est)) { @@ -4072,23 +4082,45 @@ BgBufferSync(WritebackContext *wb_context) num_written = 0; reusable_buffers = reusable_buffers_est; + /* + * Decide whether to spend second chances as we scan. StrategySweepClaims() + * counts allocations that gave up on the second-chance rule because the + * sweep could not find a candidate; while that count is rising the sweep is + * starving and the cheapest help we can give is to make candidates ahead of + * the hand. While it is flat we leave the rule alone, so this costs + * nothing on a workload that does not need it. + */ + if (prev_sweep_claims == NULL) + { + /* Same reason as saved_info below: a variable-length static. */ + prev_sweep_claims = calloc(num_partitions, sizeof(uint64)); + if (prev_sweep_claims == NULL) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"))); + } + + sweep_claims = StrategySweepClaims(partition); + spend_chance = (sweep_claims > prev_sweep_claims[partition]); + prev_sweep_claims[partition] = sweep_claims; + /* Execute the LRU scan */ while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est) { - int sync_state = SyncOneBuffer(next_to_clean, true, - wb_context); + int sync_state = SyncOneBuffer(saved->next_to_clean, true, + spend_chance, wb_context); - if (++next_to_clean >= NBuffers) + if (++saved->next_to_clean >= (first_buffer + num_buffers)) { - next_to_clean = 0; - next_passes++; + saved->next_to_clean = first_buffer; + saved->next_passes++; } num_to_scan--; if (sync_state & BUF_WRITTEN) { reusable_buffers++; - if (++num_written >= bgwriter_lru_maxpages) + if (++num_written >= (bgwriter_lru_maxpages / num_partitions)) { PendingBgWriterStats.maxwritten_clean++; break; @@ -4102,7 +4134,7 @@ BgBufferSync(WritebackContext *wb_context) #ifdef BGW_DEBUG elog(DEBUG1, "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d upcoming_est=%d scanned=%d wrote=%d reusable=%d", - recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead, + recent_alloc_partition, smoothed_alloc, strategy_delta, bufs_ahead, smoothed_density, reusable_buffers_est, upcoming_alloc_est, bufs_to_lap - num_to_scan, num_written, @@ -4132,13 +4164,96 @@ BgBufferSync(WritebackContext *wb_context) #endif } + /* can this partition hibernate */ + return (bufs_to_lap == 0 && recent_alloc_partition == 0); +} + +/* + * BgBufferSync -- Write out some dirty buffers in the pool. + * + * This is called periodically by the background writer process. + * + * Returns true if it's appropriate for the bgwriter process to go into + * low-power hibernation mode. (This happens if the strategy clock-sweep + * has been "lapped" and no buffer allocations have occurred recently, + * or if the bgwriter has been effectively disabled by setting + * bgwriter_lru_maxpages to 0.) + */ +bool +BgBufferSync(WritebackContext *wb_context) +{ + /* info obtained from freelist.c */ + uint32 recent_alloc; + uint32 recent_alloc_partition; + int num_partitions; + + /* assume we can hibernate, any partition can set to false */ + bool hibernate = true; + + /* trigger partition rebalancing first */ + StrategySyncBalance(); + + /* get the number of clocksweep partitions, and total alloc count */ + StrategySyncPrepare(&num_partitions, &recent_alloc); + + /* allocate space for per-partition information between calls */ + if (saved_info == NULL) + { + /* + * XXX Not great it's using malloc(), but how else to allocate a + * variable-length array? + */ + saved_info = malloc(sizeof(BufferSyncPartition) * num_partitions); + } + + /* Report buffer alloc counts to pgstat */ + PendingBgWriterStats.buf_alloc += recent_alloc; + + /* average alloc buffers per partition */ + recent_alloc_partition = (recent_alloc / num_partitions); + + /* + * If we're not running the LRU scan, just stop after doing the stats + * stuff. We mark the saved state invalid so that we can recover sanely + * if LRU scan is turned back on later. + */ + if (bgwriter_lru_maxpages <= 0) + { + saved_info_valid = false; + return true; + } + + /* + * now process the clocksweep partitions, one by one, using the same + * cleanup that we used for all buffers + * + * XXX Maybe we should randomize the order of partitions a bit, so that we + * don't start from partition 0 all the time? Perhaps not entirely, but at + * least pick a random starting point? + */ + for (int partition = 0; partition < num_partitions; partition++) + { + /* hibernate if all partitions can hibernate */ + hibernate &= BgBufferSyncPartition(wb_context, num_partitions, + partition, recent_alloc_partition, + &saved_info[partition]); + } + + /* now that we've scanned all partitions, mark the cached info as valid */ + saved_info_valid = true; + /* Return true if OK to hibernate */ - return (bufs_to_lap == 0 && recent_alloc == 0); + return hibernate; } /* * SyncOneBuffer -- process a single buffer during syncing. * + * If spend_chance is true, an unpinned buffer that still has its second + * chance has it spent here, which makes it reclaimable by the next pass of + * the clock hand. The caller sets this only while the sweep is starving for + * candidates; see BgBufferSync(). + * * If skip_recently_used is true, we don't write currently-pinned buffers, nor * buffers marked recently used, as these are not replacement candidates. * @@ -4151,7 +4266,8 @@ BgBufferSync(WritebackContext *wb_context) * after locking it, but we don't care all that much.) */ static int -SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) +SyncOneBuffer(int buf_id, bool skip_recently_used, bool spend_chance, + WritebackContext *wb_context) { BufferDesc *bufHdr = GetBufferDescriptor(buf_id); int result = 0; @@ -4174,10 +4290,24 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) buf_state = LockBufHdr(bufHdr); if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 && - BUF_STATE_GET_USAGECOUNT(buf_state) == 0) + !BUF_STATE_HAS_SECOND_CHANCE(buf_state)) { result |= BUF_REUSABLE; } + else if (spend_chance && + BUF_STATE_GET_REFCOUNT(buf_state) == 0 && + BUF_STATE_HAS_SECOND_CHANCE(buf_state)) + { + /* + * Unpinned, but it still has its chance. Spend it here so the next + * pass of the hand can reclaim the buffer, and count it toward our + * target: we have made a candidate, which is what the caller is short + * of. The buffer survives this cycle either way, exactly as it would + * have if the hand had passed it. + */ + UnlockBufHdrExt(bufHdr, buf_state, 0, BM_SECOND_CHANCE, 0); + return result | BUF_REUSABLE; + } else if (skip_recently_used) { /* Caller told us not to write recently-used buffers */ diff --git a/src/backend/storage/buffer/freelist.c b/src/backend/storage/buffer/freelist.c index fdb5bad7910a2..780ad9f9a2381 100644 --- a/src/backend/storage/buffer/freelist.c +++ b/src/backend/storage/buffer/freelist.c @@ -15,6 +15,15 @@ */ #include "postgres.h" +#ifdef USE_LIBNUMA +#include +#endif + +#ifdef USE_LIBNUMA +#include +#include +#endif + #include "pgstat.h" #include "port/atomics.h" #include "storage/buf_internals.h" @@ -27,17 +36,72 @@ /* - * The shared freelist control information. + * XXX We need to make ClockSweep fixed-size, so that we can have an array + * in shared memory. The easiest way is to pick a sufficiently high value + * that no system will actually need. 32 seems high enough. + * + * XXX We should enforce this in bufmgr.c, when initializing the partitions. + */ +#define MAX_BUFFER_PARTITIONS 32 + +/* + * Coefficient used to combine the old and new balance coefficients, using + * weighted average, so that we don't flap too much. The higher the value, the + * more the old value affects the result. + * + * XXX Doesn't this obscure the interpretation of weights as probabilities to + * allocate from a given partition? Does it still sum to 100%? I don't think + * so, it's just a fraction of allocations to go from a given partition. + */ +#define CLOCKSWEEP_HISTORY_COEFF 0.5 + +/* How often backend should re-fetch the CPU/node on which it is running on? */ +#define CLOCKSWEEP_CPU_NODE_REFRESH 128 + +/* + * GUCs controlling the NUMA-aware clock-sweep behavior. + * + * clocksweep_balance - when enabled, allocations may get redirected between + * clock-sweep partitions to keep them balanced (see StrategySyncBalance). + * + * clocksweep_balance_recalc - when enabled, the balance weights are + * periodically recalculated (see StrategySyncBalance). Disabling this keeps + * the current weights, e.g. ones configured manually using + * pg_buffercache_set_partition(), while still using them to balance + * allocations (if clocksweep_balance is enabled). + * + * clocksweep_scan_all_partitions - when enabled, looking for a free buffer + * scans all clock-sweep partitions (in a round-robin way), not just the + * backend's "home" partition. + * + */ +bool clocksweep_balance = true; +bool clocksweep_balance_recalc = true; +bool clocksweep_scan_all_partitions = true; + +/* + * Information about one partition of the ClockSweep (on a subset of buffers). + * + * XXX Should be careful to align this to cachelines, etc. */ typedef struct { /* Spinlock: protects the values below */ - slock_t buffer_strategy_lock; + slock_t clock_sweep_lock; + + /* range for this clock sweep partition */ + int32 node; + int32 firstBuffer; + int32 numBuffers; /* * clock-sweep hand: index of next buffer to consider grabbing. Note that * this isn't a concrete buffer - we only ever increase the value. So, to * get an actual buffer, it needs to be used modulo NBuffers. + * + * XXX This is relative to firstBuffer, so needs to be offset properly. + * + * XXX firstBuffer + (nextVictimBuffer % numBuffers) */ pg_atomic_uint32 nextVictimBuffer; @@ -48,11 +112,63 @@ typedef struct uint32 completePasses; /* Complete cycles of the clock-sweep */ pg_atomic_uint32 numBufferAllocs; /* Buffers allocated since last reset */ + /* + * Buffers that should have been allocated in this partition (but might + * have been redirected to keep allocations balanced). + */ + pg_atomic_uint32 numRequestedAllocs; + + /* running total of allocs */ + pg_atomic_uint64 numTotalAllocs; + pg_atomic_uint64 numTotalRequestedAllocs; + + /* + * Allocations in this partition that gave up on the second-chance rule + * and took a buffer that still had its chance (see StrategyGetBuffer). + * Per-partition, because each partition has its own clock hand: one can + * starve for candidates while another sits idle. + */ + pg_atomic_uint64 numSweepClaims; + + /* + * Weights to balance buffer allocations for all the partitions. Each + * partition gets a vector of weights 0-100, determining what fraction + * of buffers to allocate from that partition. So [75, 15, 5, 5] would + * mean 75% allocations should go from partition 0, 15% from partition + * 1, and 5% from partitions 2&3. Each partition gets a different vector + * of weights. + * + * Backends use the budget from it's "home" partition, so that a busy + * partitions (with a lot of processes on that NUMA node etc.) spread + * the allocations evenly. + * + * XXX Allocate a fixed-length array, to simplify working with array of + * the structs, etc. + */ + uint8 balance[MAX_BUFFER_PARTITIONS]; +} ClockSweep; + +/* + * The shared freelist control information. + */ +typedef struct +{ + /* Spinlock: protects the values below */ + slock_t buffer_strategy_lock; + /* * Bgworker process to be notified upon activity or -1 if none. See * StrategyNotifyBgWriter. */ int bgwprocno; + + /* cached info about freelist partitioning */ + int num_nodes; + int num_partitions; + int num_partitions_per_node; + + /* clocksweep partitions */ + ClockSweep sweeps[FLEXIBLE_ARRAY_MEMBER]; } BufferStrategyControl; /* Pointers to shared state */ @@ -99,6 +215,69 @@ static BufferDesc *GetBufferFromRing(BufferAccessStrategy strategy, uint64 *buf_state); static void AddBufferToRing(BufferAccessStrategy strategy, BufferDesc *buf); +static ClockSweep *ChooseClockSweep(bool balance); +static BufferDesc *StrategyGetBufferPartition(ClockSweep *sweep, + BufferAccessStrategy strategy, + uint64 *buf_state); + +/* + * clocksweep allocation balancing + * + * To balance allocations from clocksweep partitions, each partition gets a + * budget for allocating buffers from other partitions. A process that + * "exhausts" a budget in it's home partition gets redirected to the other + * partitions, driven by the budgets. + * + * For example, a partition may have budget [25, 25, 25, 25], which means + * each of the 4 partitions should get 1/4 of allocations. Or the buget + * can be [50, 50, 0, 0], which means all allocations will go to the first + * two partitions (one of them being the "home" one); + * + * We could do that based on a random number generator, but for now we + * simply treat the values as a budget, i.e. a number of allocations to + * serve from other partitions, and move in round-robin way. + * + * This is very simple/cheap, and over many allocations it has the same + * effect. For periods of low activity it may diverge, but that does not + * matter much (we care about high-activity periods much more). + * + * We intentionally keep the "budget" fairly low, with the sum for a given + * partition 100. That means we get to the same partition after only 100 + * allocations, keeping it more balanced. We can make the budgets higher + * (say, to match the expected number of allocations, i.e. bout the average + * number of allocations from the past interval). Or maybe configurable. + * + * XXX We should always start allocating from the "home" partition, i.e. + * from from it, and only then redirect to other partitions. + * + * XXX It probably is not great all the processes from that "home" + * partition are coordinated, and move to between partitions at about the + * same time. Not sure what to do about this. + * + * XXX We should also prefer other partitions from the same NUMA node (if + * there are some). Probably by setting the budgets. + * + * FIXME Explain at which point are the budgets recalculated, by which + * process, and how that affects other processes allocating buffers. + */ + +/* + * The "optimal" clock-sweep partition. After a backend gets moved to a + * different NUMA node, we restart the balancing so that it uses the + * correct "budget" from the new home partition. + */ +static int clocksweep_partition_home = -1; + +/* + * The partition the backend is currently allocating from (either the + * home one, or one of the redirected ones). + */ +static int clocksweep_partition_current = -1; + +/* + * The number of buffers to allocate from the current partition. + */ +static int clocksweep_partition_budget = 0; /* * ClockSweepTick - Helper routine for StrategyGetBuffer() @@ -107,7 +286,7 @@ static void AddBufferToRing(BufferAccessStrategy strategy, * id of the buffer now under the hand. */ static inline uint32 -ClockSweepTick(void) +ClockSweepTick(ClockSweep *sweep) { uint32 victim; @@ -117,14 +296,14 @@ ClockSweepTick(void) * apparent order. */ victim = - pg_atomic_fetch_add_u32(&StrategyControl->nextVictimBuffer, 1); + pg_atomic_fetch_add_u32(&sweep->nextVictimBuffer, 1); - if (victim >= NBuffers) + if (victim >= sweep->numBuffers) { uint32 originalVictim = victim; /* always wrap what we look up in BufferDescriptors */ - victim = victim % NBuffers; + victim = victim % sweep->numBuffers; /* * If we're the one that just caused a wraparound, force @@ -150,19 +329,194 @@ ClockSweepTick(void) * could lead to an overflow of nextVictimBuffers, but that's * highly unlikely and wouldn't be particularly harmful. */ - SpinLockAcquire(&StrategyControl->buffer_strategy_lock); + SpinLockAcquire(&sweep->clock_sweep_lock); - wrapped = expected % NBuffers; + wrapped = expected % sweep->numBuffers; - success = pg_atomic_compare_exchange_u32(&StrategyControl->nextVictimBuffer, + success = pg_atomic_compare_exchange_u32(&sweep->nextVictimBuffer, &expected, wrapped); if (success) - StrategyControl->completePasses++; - SpinLockRelease(&StrategyControl->buffer_strategy_lock); + sweep->completePasses++; + SpinLockRelease(&sweep->clock_sweep_lock); } } } - return victim; + + /* + * Make sure we've calculated a buffer in the range of the partition. Buffer + * IDs are 1-based, we're calculating 0-based indexes. + */ + Assert(victim < sweep->numBuffers); + Assert(BufferIsValid(1 + sweep->firstBuffer + victim)); + + return sweep->firstBuffer + victim; +} + +/* + * ClockSweepPartitionIndex + * pick the clock-sweep partition to use based on PID and NUMA node + * + * With libnuma, use the NUMA node and PID to pick the partition. Otherwise + * use just PID (as if there's a single NUMA node). + * + * XXX This should also check if buffers are NUMA-partitioned, not just if + * compiled with libnuma. + */ +static int +ClockSweepPartitionIndex(void) +{ + int node = 0, + index; + pid_t pid = MyProcPid;; + + Assert(StrategyControl->num_partitions == + (StrategyControl->num_nodes * StrategyControl->num_partitions_per_node)); + + /* + * If buffers are NUMA-partitioned, determine the partition using the NUMA + * node and PID. Without NUMA assume everything is a single NUMA node 0, and + * we pick the partition based on PID. + */ +#ifdef USE_LIBNUMA + if (shared_buffers_numa) + { + /* + * Cache the CPU/NUMA node, refreshing only every CLOCKSWEEP_CPU_NODE_REFRESH + * allocations. It appears that sched_getcpu()/numa_node_of_cpu() are not free. + * On some platforms it take price of full system call, or the rest (x86_64?) + * is can be use VDSO optimization. The backend rarely migrates between NUMA + * nodes, and the balance logic only needs to notice migration after some time, + * so an occasional refresh is good enough. + */ + static int cached_node = -1; + static uint32 refresh_counter = 0; + + if (cached_node < 0 || (refresh_counter++ % CLOCKSWEEP_CPU_NODE_REFRESH) == 0) + { + int cpu; + + /* XXX do we need to check sched_getcpu is available, somehow? */ + if ((cpu = sched_getcpu()) < 0) + elog(ERROR, "sched_getcpu failed: %m"); + + /* XXX/JW: use libnuma wrapper for this */ + cached_node = numa_node_of_cpu(cpu); + } + node = cached_node; + } +#endif + + /* + * We should't get unexpected NUMA nodes, not considered when setting up the + * buffer partitions. It could happen if the allowed NUMA nodes get adjusted + * at runtime, but at this point we just create partitions for all existing + * nodes. We could plan for allowed partitions, but then what if those get + * disabled, and the user allows some other partitions? + */ + if ((node < 0) || (node > StrategyControl->num_nodes)) + elog(ERROR, "node out of range: %d > %u", node, StrategyControl->num_nodes); + + /* + * Calculate the partition index. Nodes have the same number of partitions, + * and we use the PID to pick one of those (for a given node). If there's + * only a single partition per node, we can ignore PID and use node directly. + */ + if (StrategyControl->num_partitions_per_node == 1) + { + /* fast-path */ + index = node; + } + else + { + /* use PID to pick one of node's partitions */ + index = (node * StrategyControl->num_partitions_per_node) + + (pid % StrategyControl->num_partitions_per_node); + } + + /* should have a valid partition index */ + Assert((index >= 0) && (index < StrategyControl->num_partitions)); + + return index; +} + +/* + * ChooseClockSweep + * pick a clocksweep partition based on NUMA node and PID + * + * Pick a partition mapped to the NUMA node the backend is currently running + * on, and use PID if there are multiple partitions per node. Without NUMA + * supported/enabled, use just PID. + * + * XXX Maybe we should do both the total and "per group" counts a power of + * two? That'd allow using shifts instead of divisions in the calculation, + * and that's cheaper. But how would that deal with odd number of nodes? + */ +static ClockSweep * +ChooseClockSweep(bool balance) +{ + /* What's the "optimal" partition for this backend? */ + int index = ClockSweepPartitionIndex(); + ClockSweep *sweep = &StrategyControl->sweeps[index]; + + /* + * Was the process migrated to a different NUMA node? If the home partition + * changed, we need to reset the budget and start over, so that we correctly + * prefer "nearby" partitions etc. + * + * XXX Could this be a problem when processes move all the time? I don't + * think so - if a process moves between many partitions, that alone will + * spread the allocations over partitions. Similarly, if there are many + * processes, that should make it even more even. + */ + if (clocksweep_partition_home != index) + { + clocksweep_partition_home = index; + clocksweep_partition_current = index; + clocksweep_partition_budget = sweep->balance[index]; + } + + /* we should have a valid partition */ + Assert(clocksweep_partition_home != -1); + Assert(clocksweep_partition_current != -1); + Assert(clocksweep_partition_budget >= 0); + + /* + * When balancing allocations, redirect the allocations to other partitions + * according to the budgets. We move through partitions in a round-robin way, + * after allocating the "budget" of allocations from the current one. + * + * Balancing can be disabled at runtime using the clocksweep_balance GUC, in + * which case allocations always stay in the backend's "home" partition. + */ + if (balance && clocksweep_balance) + { + /* + * Ran out of budget from the current partition? Move to the next one + * with non-zero budget. + */ + while (clocksweep_partition_budget == 0) + { + /* wrap around at the end */ + clocksweep_partition_current++; + if (clocksweep_partition_current >= StrategyControl->num_partitions) + clocksweep_partition_current = 0; + + clocksweep_partition_budget + = sweep->balance[clocksweep_partition_current]; + } + + /* account for the current allocation */ + --clocksweep_partition_budget; + + /* + * Account for the allocation in the "home" partition, so that the next + * round of rebalancing (recalculating the budgets) knows about the + * allocation traffic in various partitions. + */ + pg_atomic_fetch_add_u32(&sweep->numRequestedAllocs, 1); + } + + return &StrategyControl->sweeps[clocksweep_partition_current]; } /* @@ -185,7 +539,8 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r { BufferDesc *buf; int bgwprocno; - int trycounter; + ClockSweep *sweep, + *sweep_start; /* starting clock-sweep partition */ *from_ring = false; @@ -233,17 +588,81 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r * We count buffer allocation requests so that the bgwriter can estimate * the rate of buffer consumption. Note that buffers recycled by a * strategy object are intentionally not counted here. + * + * XXX It's not quite right we call ChooseClockSweep twice - now, and then + * a couple lines later (through ClockSweepTick). If the process moves + * between CPUs / NUMA nodes in between, these call may pick different + * partitions, confusing the logic a bit. */ - pg_atomic_fetch_add_u32(&StrategyControl->numBufferAllocs, 1); + pg_atomic_fetch_add_u32(&ChooseClockSweep(false)->numBufferAllocs, 1); + + /* + * Use the "clock sweep" algorithm to find a free buffer + * + * Start with the "preferred" partition, and then proceed in a round-robin + * manner. If we cycle back to the starting partition, it means none of the + * partitions has unpinned buffers. + * + * Scanning of the other partitions can be disabled at runtime using the + * clocksweep_scan_all_partitions GUC. In that case we only scan the + * backend's "home" partition, and fail if it has no unpinned buffers. + */ + sweep = ChooseClockSweep(true); + sweep_start = sweep; + for (;;) + { + buf = StrategyGetBufferPartition(sweep, strategy, buf_state); + + /* found a buffer in the "sweep" partition, we're done */ + if (buf != NULL) + return buf; + + /* don't look at other partitions unless allowed to */ + if (!clocksweep_scan_all_partitions) + break; + + /* + * Try advancing to the next partition, round-robin (if last partition, + * wrap around to the beginning). + * + * XXX This is a bit ugly, there must be a better way to advance to the + * next partition. + */ + if (sweep == &StrategyControl->sweeps[StrategyControl->num_partitions - 1]) + sweep = StrategyControl->sweeps; + else + sweep++; + + /* we've scanned all partitions */ + if (sweep == sweep_start) + break; + } + + /* we shouldn't get here if there are unpinned buffers */ + elog(ERROR, "no unpinned buffers available"); +} - /* Use the "clock sweep" algorithm to find a free buffer */ - trycounter = NBuffers; +/* + * StrategyGetBufferPartition + * get a free buffer from a single clock-sweep partition + * + * Returns NULL if there are no free (unpinned) buffers in the partition. +*/ +static BufferDesc * +StrategyGetBufferPartition(ClockSweep *sweep, BufferAccessStrategy strategy, + uint64 *buf_state) +{ + BufferDesc *buf; + int trycounter; + int spent = 0; /* chances spent by this call */ + + trycounter = sweep->numBuffers; for (;;) { uint64 old_buf_state; uint64 local_buf_state; - buf = GetBufferDescriptor(ClockSweepTick()); + buf = GetBufferDescriptor(ClockSweepTick(sweep)); /* * Check whether the buffer can be used and pin it if so. Do this @@ -255,11 +674,8 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r local_buf_state = old_buf_state; /* - * If the buffer is pinned or has a nonzero usage_count, we cannot - * use it; decrement the usage_count (unless pinned) and keep - * scanning. + * If the buffer is pinned we cannot use it; keep scanning. */ - if (BUF_STATE_GET_REFCOUNT(local_buf_state) != 0) { if (--trycounter == 0) @@ -271,7 +687,7 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r * one eventually, but it's probably better to fail than * to risk getting stuck in an infinite loop. */ - elog(ERROR, "no unpinned buffers available"); + return NULL; } break; } @@ -283,15 +699,65 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r continue; } - if (BUF_STATE_GET_USAGECOUNT(local_buf_state) != 0) + if (BUF_STATE_HAS_SECOND_CHANCE(local_buf_state)) { - local_buf_state -= BUF_USAGECOUNT_ONE; + /* + * The buffer has been used since the hand last passed it. + * Normally we spend its second chance and keep scanning: it + * becomes a candidate only if it is still untouched when the + * hand comes back around, which gives it a full revolution of + * grace. Spending a chance is progress toward a victim, so it + * resets trycounter. + * + * That is the right default, but it is not free. A chance is + * granted again by any access, and the hand needs a full + * revolution to come back; on a large pool re-touched faster + * than that, chances are replenished faster than the hand + * spends them. Since each one resets trycounter, this loop + * can then spend chances indefinitely without finding a + * victim: unbounded work for one allocation. + * + * So give up the rule once the evidence says it is + * unaffordable. Having spent BUF_SWEEP_CLAIM_THRESHOLD + * chances without finding a single buffer that had none left, + * take the next unpinned buffer instead. Under no pressure + * the counter never reaches the threshold and behaviour is + * unchanged. + */ + if (spent < BUF_SWEEP_CLAIM_THRESHOLD) + { + local_buf_state &= ~BM_SECOND_CHANCE; + + if (pg_atomic_compare_exchange_u64(&buf->state, + &old_buf_state, + local_buf_state)) + { + spent++; + trycounter = sweep->numBuffers; + break; + } + continue; + } + + /* + * Take this buffer even though it still had a chance. The bit + * is cleared when the victim is reused, so there is no need to + * spend the chance first. + */ + pg_atomic_fetch_add_u64(&sweep->numSweepClaims, 1); + local_buf_state += BUF_REFCOUNT_ONE; if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state, local_buf_state)) { - trycounter = NBuffers; - break; + /* Found a usable buffer */ + if (strategy != NULL) + AddBufferToRing(strategy, buf); + *buf_state = local_buf_state; + + TrackNewBufferPin(BufferDescriptorGetBuffer(buf)); + + return buf; } } else @@ -316,6 +782,317 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r } } +/* + * StrategySyncBalance + * update partition budgets, to balance the buffer allocations + * + * We want to give preference to allocating buffers on the same NUMA node, + * but that might lead to imbalance - a single process would only use a + * fraction of shared buffers. We don't want that, we want to utilize the + * whole shared buffers. The number of allocations in each partition may + * also change over time, so we need to adapt to that. + * + * To allow this "adaptive balancing", each partition has a set of weights, + * determining what fraction of allocations to direct to other partitions. + * For simplicity the coefficients are integers 0-100, expressing the + * percentage of allocations redirected to that partition. + * + * Consider for example weights [50, 25, 25, 0] for one of 4 partitions. + * This means 50% of allocations will be redirected to partition 0, 25% + * to partitions 1 and 2, and no allocations will go to partition 3. + * + * This means an allocation may be requested in partition A (i.e. the + * home partition of the process requesting it), but end up allocating + * the buffer in partition B. We have a counter for both - the number of + * allocations requested in a partition, and the number of allocations + * actually handled by that partition. The former is used for calculating + * weights, the latter is used only for monitoring. + * + * The balancing happens in intervals - it adjusts future allocations + * based on stats about recent allocations, namely: + * + * - numBufferAllocs - number of allocations served by a partition + * + * - numRequestedAllocs - number of allocatios requested in a partition + * + * We're trying to smooth numBufferAllocs in the next interval, based on + * numRequestedAllocs measured in the last interval. + * + * The balancing algorithm works like this: + * + * - the target (average number of allocations per partition) is calculated + * from total number of allocations requested in the last intervaal + * + * - partitions get divided into two groups - those with more allocation + * requests than the target, and those with fewer requests + * + * - we "distribute" the delta (which is the same between the groups) + * between the groups (one has more, the other fewer) + * + * Partitions with (nallocs > avg_nallocs) redirect the extra allocations, + * with each target allocation getting a proportional part (with respect + * to the total delta). + * + * XXX Currently this does not give preference to other partitions on the + * same NUMA node (redirect to it first), but it could. + */ +void +StrategySyncBalance(void) +{ + /* snapshot of allocation requests for partitions */ + uint32 allocs[MAX_BUFFER_PARTITIONS]; + + uint32 total_allocs = 0, /* total number of allocations */ + avg_allocs, /* average allocations (per partition) */ + delta_allocs = 0, /* sum of allocs above average */ + redirect_cutoff; /* redirect only above this many allocs */ + + if (!clocksweep_balance || !clocksweep_balance_recalc) + return; + + /* + * Collect the number of allocations requested in the past interval. + * While at it, reset the counter to start the new interval. + * + * XXX We lock the partitions one by one, so this is not a perfectly + * consistent snapshot of the counts, and the resets happen before we + * update the weights too. But we're only looking for heuristics, so + * this should be good enough. + * + * XXX A similar issue applies to the counter reset later - we haven't + * updated the weights yet, so some of the requests counted for the next + * interval will be redirected per current weights. Should be fine, it's + * just an approximate heuristics, and there should be very few requests in + * between. Alternatively, we could reset the request counters when setting + * the new weights, and just ignore the couple requests in between. + * + * XXX Does this need to worry about the completePasses too? + */ + for (int i = 0; i < StrategyControl->num_partitions; i++) + { + ClockSweep *sweep = &StrategyControl->sweeps[i]; + + /* no need for a spinlock */ + allocs[i] = pg_atomic_exchange_u32(&sweep->numRequestedAllocs, 0); + + /* add the allocs to running total */ + pg_atomic_fetch_add_u64(&sweep->numTotalRequestedAllocs, allocs[i]); + + total_allocs += allocs[i]; + } + + /* Calculate the "fair share" of allocations per partition. */ + avg_allocs = (total_allocs / StrategyControl->num_partitions); + + /* + * Calculate the "delta" from balanced state for each partition, i.e. how + * many more/fewer allocations it handled relative to the average. + */ + for (int i = 0; i < StrategyControl->num_partitions; i++) + { + if (allocs[i] > avg_allocs) + delta_allocs += (allocs[i] - avg_allocs); + } + + /* + * Skip rebalancing when there's not enough activity, and just keep the + * current weights. + * + * XXX The threshold of 100 allocation is pretty arbitrary. + * + * XXX Maybe a better strategy would be to slowly return to the default + * weights, with each partition allocation only from itself? + * + * XXX Maybe we shouldn't even reset the counters in this case? But it + * should not matter, if the activity is low. + */ + if (avg_allocs < 100) + { + elog(DEBUG1, "rebalance skipped: not enough allocations (allocs: %u)", + avg_allocs); + return; + } + + /* + * Likewise, skip rebalancing if the misbalance is not significant. We + * consider it acceptable if the amount of allocations we'd need to + * redistribute is less than 10% of the average. + * + * XXX Again, these threshold are rather arbitrary. And maybe we should + * do the rabalancing in this case anyway, it's likely cheap and on a big + * system 10% can be quite a lot. + */ + if (delta_allocs < (avg_allocs * 0.1)) + { + elog(DEBUG1, "rebalance skipped: delta within limit (delta: %u, threshold: %u)", + delta_allocs, (uint32) (avg_allocs * 0.1)); + return; + } + + /* + * A partition only redirects allocations to other partitions when it + * exceeds the average by more than some threshold percent. + * Below this cutoff we keep allocations local, to preserve NUMA locality. + * + * TODO: maybe better value is possible. On 4s with 25 I've got good results, + * but with value of 50 I've got slight degradation. Maybe it should + * be equal to 100/numa_nodes ? + * + */ +#define CLOCKSWEEP_CUTOFF_THRESHOLD 25 + redirect_cutoff = avg_allocs + + (uint32) ((uint64) avg_allocs * CLOCKSWEEP_CUTOFF_THRESHOLD / 100); + + /* + * The actual rebalancing + * + * Partition with fewer than average allocations, should not redirect any + * allocations to other partitions. So just use weights with a single + * non-zero weight for the partition itself. + * + * Partition with more than average allocations, should not receive any + * redirected allocations, and instead it should redirect excess allocations + * to other partitions. + * + * The redistribution is "proportional" - if the excess allocations of a + * partition represent 10% of the "delta", then each partition that + * needs more allocations will get 10% of the gap from it. + * + * XXX We should add hysteresis, so that it does not oscillate or something + * like that. Maybe CLOCKSWEEP_HISTORY_COEFF already does that? + * + * XXX Ideally, the alternative partitions to use first would be the other + * partitions for the same node (if any). + */ + for (int i = 0; i < StrategyControl->num_partitions; i++) + { + ClockSweep *sweep = &StrategyControl->sweeps[i]; + uint8 balance[MAX_BUFFER_PARTITIONS]; + + /* lock, we're going to modify the balance weights */ + SpinLockAcquire(&sweep->clock_sweep_lock); + + /* reset the weights to start from scratch */ + memset(balance, 0, sizeof(uint8) * MAX_BUFFER_PARTITIONS); + + /* + * Does this partition exceed its fair share by more than the + * threshold? If not, keep all allocations local - redirecting them + * would push memory onto remote NUMA nodes for no real benefit when + * the load is already close to balanced. + */ + if (allocs[i] <= redirect_cutoff) + { + /* near fair share (or below) - keep allocations local */ + balance[i] = 100; + } + else + { + /* + * more - redistribute the excess allocations + * + * Each "target" partition (with less than avg_allocs) should get + * a fraction proportional to (excess/delta) from this one. + */ + + /* fraction of the "total" delta */ + double delta_frac = (allocs[i] - avg_allocs) * 1.0 / delta_allocs; + + /* how much we keep local; we hand out the rest below */ + int kept = 100; + + /* redirect the extra allocations */ + for (int j = 0; j < StrategyControl->num_partitions; j++) + { + /* How many allocations to receive from i-th partition? */ + uint32 receive_allocs = delta_frac * (avg_allocs - allocs[j]); + int w; + + /* do not redirect to ourselves */ + if (j == i) + continue; + + /* ignore partitions that don't need additional allocations */ + if (allocs[j] > avg_allocs) + continue; + + /* + * Only use other partitions that actually have demand of + * their own (avoid idle). If we fail, there's always the + * scan-all-partitions fallback. + * + * TODO:: just guessing,heuristics + */ + if (allocs[j] < (avg_allocs / 2)) + continue; + + /* fraction to redirect */ + w = (int) ((100.0 * receive_allocs / allocs[i]) + 0.5); + balance[j] = w; + kept -= w; + } + + /* avoid negative balances */ + if (kept > 0) + balance[i] = kept; + else + balance[i] = 1; + } + + /* combine the old and new weights (hysteresis) */ + for (int j = 0; j < MAX_BUFFER_PARTITIONS; j++) + { + sweep->balance[j] + = CLOCKSWEEP_HISTORY_COEFF * sweep->balance[j] + + (1.0 - CLOCKSWEEP_HISTORY_COEFF) * balance[j]; + } + + SpinLockRelease(&sweep->clock_sweep_lock); + } +} + +/* + * StrategySyncPrepare -- prepare for sync of all partitions + * + * Determine the number of clocksweep partitions, and calculate the recent + * buffers allocs (as a sum of all the partitions). This allows BgBufferSync + * to calculate average number of allocations per partition for the next + * sync cycle. + * + * In addition it returns the count of recent buffer allocs, which is a total + * summed from all partitions. The alloc counts are reset after being read, + * as the partitions are walked. + */ +void +StrategySyncPrepare(int *num_parts, uint32 *num_buf_alloc) +{ + *num_buf_alloc = 0; + *num_parts = StrategyControl->num_partitions; + + /* + * We lock the partitions one by one, so not exacly in sync, but that + * should be fine. We're only looking for heuristics anyway. + */ + for (int i = 0; i < StrategyControl->num_partitions; i++) + { + ClockSweep *sweep = &StrategyControl->sweeps[i]; + + /* XXX Do we need the lock, if we're only accessing atomics? Surely not. */ + /* XXX Are we ever calling this without num_buf_alloc? */ + SpinLockAcquire(&sweep->clock_sweep_lock); + if (num_buf_alloc) + { + uint32 allocs = pg_atomic_exchange_u32(&sweep->numBufferAllocs, 0); + + /* include the count in the running total */ + pg_atomic_fetch_add_u64(&sweep->numTotalAllocs, allocs); + + *num_buf_alloc += allocs; + } + SpinLockRelease(&sweep->clock_sweep_lock); + } +} + /* * StrategySyncStart -- tell BgBufferSync where to start syncing * @@ -323,37 +1100,61 @@ StrategyGetBuffer(BufferAccessStrategy strategy, uint64 *buf_state, bool *from_r * BgBufferSync() will proceed circularly around the buffer array from there. * * In addition, we return the completed-pass count (which is effectively - * the higher-order bits of nextVictimBuffer) and the count of recent buffer - * allocs if non-NULL pointers are passed. The alloc count is reset after - * being read. + * the higher-order bits of nextVictimBuffer). + * + * This only considers a single clocksweep partition, as BgBufferSync looks + * at them one by one. */ int -StrategySyncStart(uint32 *complete_passes, uint32 *num_buf_alloc) +StrategySyncStart(int partition, uint32 *complete_passes, + int *first_buffer, int *num_buffers) { uint32 nextVictimBuffer; int result; + ClockSweep *sweep = &StrategyControl->sweeps[partition]; - SpinLockAcquire(&StrategyControl->buffer_strategy_lock); - nextVictimBuffer = pg_atomic_read_u32(&StrategyControl->nextVictimBuffer); - result = nextVictimBuffer % NBuffers; + Assert((partition >= 0) && (partition < StrategyControl->num_partitions)); + + SpinLockAcquire(&sweep->clock_sweep_lock); + nextVictimBuffer = pg_atomic_read_u32(&sweep->nextVictimBuffer); + result = nextVictimBuffer % sweep->numBuffers; + + *first_buffer = sweep->firstBuffer; + *num_buffers = sweep->numBuffers; if (complete_passes) { - *complete_passes = StrategyControl->completePasses; + *complete_passes = sweep->completePasses; /* * Additionally add the number of wraparounds that happened before * completePasses could be incremented. C.f. ClockSweepTick(). */ - *complete_passes += nextVictimBuffer / NBuffers; + *complete_passes += nextVictimBuffer / sweep->numBuffers; } + SpinLockRelease(&sweep->clock_sweep_lock); - if (num_buf_alloc) - { - *num_buf_alloc = pg_atomic_exchange_u32(&StrategyControl->numBufferAllocs, 0); - } - SpinLockRelease(&StrategyControl->buffer_strategy_lock); - return result; + /* XXX buffer IDs start at 1, we're calculating 0-based indexes */ + Assert(BufferIsValid(1 + sweep->firstBuffer + result)); + + return sweep->firstBuffer + result; +} + +/* + * StrategySweepClaims -- allocations that took a buffer with a chance left + * + * Returns a monotonically increasing count of allocations in this partition + * that gave up on the second-chance rule because the sweep could not find a + * candidate. The background writer compares successive readings per + * partition: a rising count means that partition's sweep is starving, which is + * when making candidates ahead of the hand pays for itself. + */ +uint64 +StrategySweepClaims(int partition) +{ + Assert((partition >= 0) && (partition < StrategyControl->num_partitions)); + + return pg_atomic_read_u64(&StrategyControl->sweeps[partition].numSweepClaims); } /* @@ -385,8 +1186,14 @@ StrategyNotifyBgWriter(int bgwprocno) static void StrategyCtlShmemRequest(void *arg) { + int num_partitions; + + /* get the number of buffer partitions */ + BufferPartitionsCalculate(NULL, &num_partitions, NULL); + ShmemRequestStruct(.name = "Buffer Strategy Status", - .size = sizeof(BufferStrategyControl), + .size = offsetof(BufferStrategyControl, sweeps) + + mul_size(num_partitions, sizeof(ClockSweep)), .ptr = (void **) &StrategyControl ); } @@ -399,12 +1206,57 @@ StrategyCtlShmemInit(void *arg) { SpinLockInit(&StrategyControl->buffer_strategy_lock); - /* Initialize the clock-sweep pointer */ - pg_atomic_init_u32(&StrategyControl->nextVictimBuffer, 0); + /* Remember the number of partitions */ + BufferPartitionsParams(&StrategyControl->num_nodes, + &StrategyControl->num_partitions, + &StrategyControl->num_partitions_per_node); + + /* Initialize the clock sweep pointers (for all partitions) */ + for (int i = 0; i < StrategyControl->num_partitions; i++) + { + int node, + num_buffers, + first_buffer, + last_buffer; + + SpinLockInit(&StrategyControl->sweeps[i].clock_sweep_lock); + + pg_atomic_init_u32(&StrategyControl->sweeps[i].nextVictimBuffer, 0); + + /* get info about the buffer partition */ + BufferPartitionGet(i, &node, &num_buffers, + &first_buffer, &last_buffer); + + /* + * FIXME This may not quite right, because if NBuffers is not a + * perfect multiple of numBuffers, the last partition will have + * numBuffers set too high. buf_init handles this by tracking the + * remaining number of buffers, and not overflowing. + */ + StrategyControl->sweeps[i].node = node; + StrategyControl->sweeps[i].numBuffers = num_buffers; + StrategyControl->sweeps[i].firstBuffer = first_buffer; + + /* Clear statistics */ + StrategyControl->sweeps[i].completePasses = 0; + pg_atomic_init_u32(&StrategyControl->sweeps[i].numBufferAllocs, 0); + pg_atomic_init_u32(&StrategyControl->sweeps[i].numRequestedAllocs, 0); + pg_atomic_init_u64(&StrategyControl->sweeps[i].numTotalAllocs, 0); + pg_atomic_init_u64(&StrategyControl->sweeps[i].numSweepClaims, 0); + pg_atomic_init_u64(&StrategyControl->sweeps[i].numTotalRequestedAllocs, 0); - /* Clear statistics */ - StrategyControl->completePasses = 0; - pg_atomic_init_u32(&StrategyControl->numBufferAllocs, 0); + /* + * Initialize the weights - start by allocating 100% buffers from + * the current node / partition. + */ + for (int j = 0; j < MAX_BUFFER_PARTITIONS; j++) + { + if (i == j) + StrategyControl->sweeps[i].balance[i] = 100; + else + StrategyControl->sweeps[i].balance[j] = 0; + } + } /* No pending notification */ StrategyControl->bgwprocno = -1; @@ -655,14 +1507,19 @@ GetBufferFromRing(BufferAccessStrategy strategy, uint64 *buf_state) /* * If the buffer is pinned we cannot use it under any circumstances. * - * If usage_count is 0 or 1 then the buffer is fair game (we expect 1, - * since our own previous usage of the ring element would have left it - * there, but it might've been decremented by clock-sweep since then). - * A higher usage_count indicates someone else has touched the buffer, - * so we shouldn't re-use it. + * The buffer is fair game only if nothing has touched it since we + * last cycled past this ring slot. If it still has its second + * chance, an access from outside the ring has claimed it for the + * working set and we must not recycle it. + * + * This replaces the stock "usage_count > 1 means somebody else + * touched it" test. That needed a count only because a pin taken + * from inside the ring was indistinguishable from one taken outside + * it; now the ring does not grant the bit at all, so its own pins + * leave no trace and a single bit is enough. */ if (BUF_STATE_GET_REFCOUNT(local_buf_state) != 0 - || BUF_STATE_GET_USAGECOUNT(local_buf_state) > 1) + || BUF_STATE_HAS_SECOND_CHANCE(local_buf_state)) break; /* See equivalent code in PinBuffer() */ @@ -768,3 +1625,85 @@ StrategyRejectBuffer(BufferAccessStrategy strategy, BufferDesc *buf, bool from_r return true; } + +void +ClockSweepPartitionGetInfo(int idx, + uint32 *complete_passes, uint32 *next_victim_buffer, + uint64 *buffer_total_allocs, uint32 *buffer_allocs, + uint64 *buffer_total_req_allocs, uint32 *buffer_req_allocs, + int **weights) +{ + ClockSweep *sweep = &StrategyControl->sweeps[idx]; + + Assert((idx >= 0) && (idx < StrategyControl->num_partitions)); + + /* get the clocksweep stats */ + *complete_passes = sweep->completePasses; + + /* calculate the actual buffer ID */ + *next_victim_buffer = pg_atomic_read_u32(&sweep->nextVictimBuffer); + *next_victim_buffer = sweep->firstBuffer + (*next_victim_buffer % sweep->numBuffers); + + *buffer_total_allocs = pg_atomic_read_u64(&sweep->numTotalAllocs); + *buffer_allocs = pg_atomic_read_u32(&sweep->numBufferAllocs); + + *buffer_total_req_allocs = pg_atomic_read_u64(&sweep->numTotalRequestedAllocs); + *buffer_req_allocs = pg_atomic_read_u32(&sweep->numRequestedAllocs); + + /* return the weights in a newly allocated array */ + *weights = palloc_array(int, StrategyControl->num_partitions); + for (int i = 0; i < StrategyControl->num_partitions; i++) + { + (*weights)[i] = (int) sweep->balance[i]; + } +} + +/* + * ClockSweepSetWeights override the clock-sweep balance weights of a single + * partition. + */ +void +ClockSweepSetWeights(int partition, int *weights, int nweights) +{ + ClockSweep *sweep; + + /* + * Disallow manual weights while the automatic recalculation is enabled, as + * StrategySyncBalance would just recompute (and overwrite) them. + */ + if (clocksweep_balance_recalc) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot set clock-sweep weights while debug_clocksweep_balance_recalc is enabled"), + errhint("Set debug_clocksweep_balance_recalc to off before setting the weights manually."))); + + if ((partition < 0) || (partition >= StrategyControl->num_partitions)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("clock-sweep partition %d out of range", partition), + errhint("There are %d clock-sweep partitions, numbered from 0.", + StrategyControl->num_partitions))); + + if (nweights != StrategyControl->num_partitions) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("number of weights (%d) does not match number of clock-sweep partitions (%d)", + nweights, StrategyControl->num_partitions))); + + for (int i = 0; i < nweights; i++) + { + if ((weights[i] < 0) || (weights[i] > 100)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("clock-sweep weight %d out of range", + weights[i]), + errhint("Each weight must be between 0 and 100."))); + } + + sweep = &StrategyControl->sweeps[partition]; + + SpinLockAcquire(&sweep->clock_sweep_lock); + for (int j = 0; j < nweights; j++) + sweep->balance[j] = (uint8) weights[j]; + SpinLockRelease(&sweep->clock_sweep_lock); +} diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c index 4870c8e13d010..3daf63e8e0862 100644 --- a/src/backend/storage/buffer/localbuf.c +++ b/src/backend/storage/buffer/localbuf.c @@ -166,8 +166,8 @@ LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, bufHdr->tag = newTag; buf_state = pg_atomic_read_u64(&bufHdr->state); - buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK); - buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + buf_state &= ~BUF_FLAG_MASK; + buf_state |= BM_TAG_VALID; /* admit on probation */ pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state); *foundPtr = false; @@ -248,9 +248,9 @@ GetLocalVictimBuffer(void) { uint64 buf_state = pg_atomic_read_u64(&bufHdr->state); - if (BUF_STATE_GET_USAGECOUNT(buf_state) > 0) + if (BUF_STATE_HAS_SECOND_CHANCE(buf_state)) { - buf_state -= BUF_USAGECOUNT_ONE; + buf_state &= ~BM_SECOND_CHANCE; pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state); trycounter = NLocBuffer; } @@ -454,7 +454,7 @@ ExtendBufferedRelLocal(BufferManagerRelation bmr, victim_buf_hdr->tag = tag; - buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + buf_state |= BM_TAG_VALID; /* admit on probation */ pg_atomic_unlocked_write_u64(&victim_buf_hdr->state, buf_state); @@ -670,7 +670,6 @@ InvalidateLocalBuffer(BufferDesc *bufHdr, bool check_unreferenced) /* Mark buffer invalid */ ClearBufferTag(&bufHdr->tag); buf_state &= ~BUF_FLAG_MASK; - buf_state &= ~BUF_USAGECOUNT_MASK; pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state); } @@ -838,11 +837,8 @@ PinLocalBuffer(BufferDesc *buf_hdr, bool adjust_usagecount) { NLocalPinnedBuffers++; buf_state += BUF_REFCOUNT_ONE; - if (adjust_usagecount && - BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT) - { - buf_state += BUF_USAGECOUNT_ONE; - } + if (adjust_usagecount) + buf_state |= BM_SECOND_CHANCE; pg_atomic_unlocked_write_u64(&buf_hdr->state, buf_state); /* diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index b6bdfe213feec..0ef21d9065501 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -87,6 +87,10 @@ #include "utils/timestamp.h" #include "utils/varlena.h" +#ifdef USE_LIBNUMA +#include +#endif + /* ---------------- * global variables * ---------------- @@ -110,6 +114,9 @@ int client_connection_check_interval = 0; /* flags for non-system relation kinds to restrict use */ int restrict_nonsystem_relation_kind; +/* NUMA node to pin the backend to at query start; -1 disables pinning */ +int debug_numa_node = -1; + /* * Include signal sender PID/UID in the server log when available * (SA_SIGINFO). The caller must supply the already-captured pid and uid @@ -1021,6 +1028,53 @@ pg_plan_queries(List *querytrees, const char *query_string, int cursorOptions, } +/* + * process_debug_numa_node + * + * If the debug_numa_node GUC is set (>= 0), pin this backend to run on the + * CPUs of the requested NUMA node. -1 disables it (default) + */ +static void +process_debug_numa_node(void) +{ +#ifdef USE_LIBNUMA + static int applied_numa_node = -1; + + if (debug_numa_node == applied_numa_node) + return; + + /* Nothing we can do if the kernel/library has no NUMA support. */ + if (numa_available() < 0) + { + applied_numa_node = debug_numa_node; + return; + } + + if (debug_numa_node < 0) + { + /* Pinning disabled: allow running on all nodes again. */ + numa_run_on_node_mask(numa_all_nodes_ptr); + } + else if (debug_numa_node > numa_max_node()) + { + ereport(WARNING, + (errmsg("debug_numa_node %d exceeds the highest available NUMA node %d, ignoring", + debug_numa_node, numa_max_node()))); + } + else if (numa_run_on_node(debug_numa_node) != 0) + { + ereport(WARNING, + (errmsg("could not pin backend to NUMA node %d: %m", + debug_numa_node))); + } + else + elog(DEBUG1, "pinned backend to NUMA node %d", debug_numa_node); + + applied_numa_node = debug_numa_node; +#endif +} + + /* * exec_simple_query * @@ -1045,6 +1099,9 @@ exec_simple_query(const char *query_string) pgstat_report_activity(STATE_RUNNING, query_string); + /* Pin the backend to the configured NUMA node, if requested. */ + process_debug_numa_node(); + TRACE_POSTGRESQL_QUERY_START(query_string); /* @@ -2214,6 +2271,9 @@ exec_execute_message(const char *portal_name, long max_rows) pgstat_report_activity(STATE_RUNNING, sourceText); + /* Pin the backend to the configured NUMA node, if requested. */ + process_debug_numa_node(); + foreach(lc, portal->stmts) { PlannedStmt *stmt = lfirst_node(PlannedStmt, lc); diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index 380679a76c096..c2eac5f535025 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -632,6 +632,27 @@ boot_val => 'DEFAULT_ASSERT_ENABLED', }, +{ name => 'debug_clocksweep_balance', type => 'bool', context => 'PGC_USERSET', group => 'DEVELOPER_OPTIONS', + short_desc => 'Enables balancing of buffer allocations between clock-sweep partitions.', + flags => 'GUC_NOT_IN_SAMPLE', + variable => 'clocksweep_balance', + boot_val => 'true' +}, + +{ name => 'debug_clocksweep_balance_recalc', type => 'bool', context => 'PGC_USERSET', group => 'DEVELOPER_OPTIONS', + short_desc => 'Enables periodic recalculation of clock-sweep partition balance weights.', + flags => 'GUC_NOT_IN_SAMPLE', + variable => 'clocksweep_balance_recalc', + boot_val => 'true' +}, + +{ name => 'debug_clocksweep_scan_all_partitions', type => 'bool', context => 'PGC_USERSET', group => 'DEVELOPER_OPTIONS', + short_desc => 'Enables scanning all clock-sweep partitions when looking for a free buffer.', + flags => 'GUC_NOT_IN_SAMPLE', + variable => 'clocksweep_scan_all_partitions', + boot_val => 'true' +}, + { name => 'debug_copy_parse_plan_trees', type => 'bool', context => 'PGC_SUSET', group => 'DEVELOPER_OPTIONS', short_desc => 'Set this to force all parse and plan trees to be passed through copyObject(), to facilitate catching errors and omissions in copyObject().', flags => 'GUC_NOT_IN_SAMPLE', @@ -684,6 +705,16 @@ options => 'debug_logical_replication_streaming_options', }, +{ name => 'debug_numa_node', type => 'int', context => 'PGC_USERSET', group => 'DEVELOPER_OPTIONS', + short_desc => 'Pins the backend to the given NUMA node at query start.', + long_desc => '-1 (the default) disables pinning.', + flags => 'GUC_NOT_IN_SAMPLE', + variable => 'debug_numa_node', + boot_val => '-1', + min => '-1', + max => 'INT_MAX', +}, + { name => 'debug_parallel_query', type => 'enum', context => 'PGC_USERSET', group => 'DEVELOPER_OPTIONS', short_desc => 'Forces the planner\'s use parallel query nodes.', long_desc => 'This can be useful for testing the parallel query infrastructure by forcing the planner to generate plans that contain nodes that perform tuple communication between workers and the main process.', @@ -731,6 +762,20 @@ ifdef => 'DEBUG_NODE_TESTS_ENABLED', }, +{ name => 'debug_shmem_interleave', type => 'bool', context => 'PGC_SUSET', group => 'DEVELOPER_OPTIONS', + short_desc => 'Forces interleaving for the whole shared memory segment.', + flags => 'GUC_NOT_IN_SAMPLE', + variable => 'shmem_interleave', + boot_val => 'false' +}, + +{ name => 'debug_shmem_populate', type => 'bool', context => 'PGC_SUSET', group => 'DEVELOPER_OPTIONS', + short_desc => 'Populates (faults) the whole shared memory segment using MAP_POPULATE.', + flags => 'GUC_NOT_IN_SAMPLE', + variable => 'shmem_populate', + boot_val => 'false' +}, + { name => 'debug_write_read_parse_plan_trees', type => 'bool', context => 'PGC_SUSET', group => 'DEVELOPER_OPTIONS', short_desc => 'Set this to force all parse and plan trees to be passed through outfuncs.c/readfuncs.c, to facilitate catching errors and omissions in those modules.', flags => 'GUC_NOT_IN_SAMPLE', @@ -2725,6 +2770,12 @@ max => 'INT_MAX / 2', }, +{ name => 'shared_buffers_numa', type => 'bool', context => 'PGC_POSTMASTER', group => 'RESOURCES_MEM', + short_desc => 'Locate partitions of shared buffers (and descriptors) to NUMA nodes.', + variable => 'shared_buffers_numa', + boot_val => 'false', +}, + { name => 'shared_memory_size', type => 'int', context => 'PGC_INTERNAL', group => 'PRESET_OPTIONS', short_desc => 'Shows the size of the server\'s main shared memory area (rounded up to the nearest MB).', flags => 'GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE | GUC_UNIT_MB | GUC_RUNTIME_COMPUTED', diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 342aaeef59afd..44cd16fdac562 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -533,6 +533,14 @@ extern const struct config_enum_entry dynamic_shared_memory_options[]; * GUC option variables that are exported from this module */ bool AllowAlterSystem = true; + +/* + * Defined here rather than in sysv_shmem.c because that file is only linked + * on platforms with System V shared memory; Windows links win32_shmem.c + * instead, and the GUC machinery below needs these on every platform. + */ +bool shmem_populate = false; +bool shmem_interleave = false; bool log_duration = false; bool Debug_print_plan = false; bool Debug_print_parse = false; diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 01f98de8a9ffc..670d7e900b5a4 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -142,6 +142,7 @@ #temp_buffers = 8MB # min 800kB #max_prepared_transactions = 0 # zero disables the feature # (change requires restart) +#shared_buffers_numa = off # NUMA-aware partitioning # Caution: it is not advisable to set max_prepared_transactions nonzero unless # you actively intend to use prepared transactions. #work_mem = 4MB # min 64kB diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 8d6aacc4d5ac2..bd520356231db 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -213,6 +213,11 @@ extern PGDLLIMPORT Oid MyDatabaseTableSpace; extern PGDLLIMPORT bool MyDatabaseHasLoginEventTriggers; +extern PGDLLIMPORT bool shmem_populate; +extern PGDLLIMPORT bool shmem_interleave; +extern PGDLLIMPORT int debug_numa_node; + + /* * Date/Time Configuration * diff --git a/src/include/port/pg_numa.h b/src/include/port/pg_numa.h index 1b668fe1d9112..8fe4d4ab7e349 100644 --- a/src/include/port/pg_numa.h +++ b/src/include/port/pg_numa.h @@ -17,6 +17,13 @@ extern PGDLLIMPORT int pg_numa_init(void); extern PGDLLIMPORT int pg_numa_query_pages(int pid, unsigned long count, void **pages, int *status); extern PGDLLIMPORT int pg_numa_get_max_node(void); +extern PGDLLIMPORT Size pg_numa_page_size(void); +extern PGDLLIMPORT void pg_numa_move_to_node(char *startptr, char *endptr, int node); +extern PGDLLIMPORT int pg_numa_bind_to_node(char *startptr, char *endptr, int node); + +extern PGDLLIMPORT int numa_flags; + +#define NUMA_BUFFERS 0x01 #ifdef USE_LIBNUMA diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index 726ab75dfe992..b4254eae16d92 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -30,12 +30,33 @@ #include "utils/relcache.h" #include "utils/resowner.h" +/* + * How many second chances a single StrategyGetBuffer() call will spend before + * it stops looking for a buffer that has none left and takes the next unpinned + * buffer it sees. + * + * Spending a chance is progress -- the buffer becomes reclaimable on the next + * pass -- so the sweep treats it as a reason to keep scanning. When the pool + * is re-touched faster than the hand can traverse it, chances are granted + * again before the hand returns, and a single allocation can spend them + * indefinitely without ever finding a victim. Giving up after a fixed number + * makes the work one allocation can do a constant instead. + * + * The value trades that bound against replacement quality, and both sides have + * been measured. The worst case an allocation reaches is almost exactly this + * threshold plus two, so the bound is a direct function of it. The cost is + * taking a buffer that still had a chance left: on a workload whose hot set + * most queries re-read, a threshold of 4 leaves 4,519 hot pages non-resident + * and costs 1.8 points of hot-set hit ratio, 32 leaves 11 pages and costs 0.03 + * points, and at 128 nothing hot is displaced and the cost is not measurable. + */ +#define BUF_SWEEP_CLAIM_THRESHOLD 128 + /* * Buffer state is a single 64-bit variable where following data is combined. * * State of the buffer itself (in order): * - 18 bits refcount - * - 4 bits usage count * - 12 bits of flags * - 18 bits share-lock count * - 1 bit share-exclusive locked @@ -47,11 +68,10 @@ * The definition of buffer state components is below. */ #define BUF_REFCOUNT_BITS 18 -#define BUF_USAGECOUNT_BITS 4 #define BUF_FLAG_BITS 12 #define BUF_LOCK_BITS (18+2) -StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS + BUF_LOCK_BITS <= 64, +StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_FLAG_BITS + BUF_LOCK_BITS <= 64, "parts of buffer state space need to be <= 64"); /* refcount related definitions */ @@ -59,17 +79,9 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS + BUF_L #define BUF_REFCOUNT_MASK \ ((UINT64CONST(1) << BUF_REFCOUNT_BITS) - 1) -/* usage count related definitions */ -#define BUF_USAGECOUNT_SHIFT \ - BUF_REFCOUNT_BITS -#define BUF_USAGECOUNT_MASK \ - (((UINT64CONST(1) << BUF_USAGECOUNT_BITS) - 1) << (BUF_USAGECOUNT_SHIFT)) -#define BUF_USAGECOUNT_ONE \ - (UINT64CONST(1) << BUF_REFCOUNT_BITS) - /* flags related definitions */ #define BUF_FLAG_SHIFT \ - (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS) + BUF_REFCOUNT_BITS #define BUF_FLAG_MASK \ (((UINT64CONST(1) << BUF_FLAG_BITS) - 1) << BUF_FLAG_SHIFT) @@ -86,11 +98,19 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS + BUF_L ((((uint64) MAX_BACKENDS) << BM_LOCK_SHIFT) | BM_LOCK_VAL_SHARE_EXCLUSIVE | BM_LOCK_VAL_EXCLUSIVE) -/* Get refcount and usagecount from buffer state */ +/* Get refcount from buffer state */ #define BUF_STATE_GET_REFCOUNT(state) \ ((uint32)((state) & BUF_REFCOUNT_MASK)) -#define BUF_STATE_GET_USAGECOUNT(state) \ - ((uint32)(((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT)) + +/* + * Does this buffer still have its second chance? + * + * The clock sweep reclaims a buffer only on a visit that finds the bit + * already clear, so a buffer that is accessed between two visits is always + * passed over once. See BM_SECOND_CHANCE. + */ +#define BUF_STATE_HAS_SECOND_CHANCE(state) \ + (((state) & BM_SECOND_CHANCE) != 0) /* * Flags for buffer descriptors @@ -114,7 +134,19 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS + BUF_L #define BM_IO_IN_PROGRESS BUF_DEFINE_FLAG( 4) /* previous I/O failed */ #define BM_IO_ERROR BUF_DEFINE_FLAG( 5) -/* flag bit 6 is not used anymore */ +/* + * Set when a buffer is pinned, cleared by the clock sweep as the hand passes. + * The sweep reclaims a buffer only if it finds this bit already clear, so any + * buffer touched since the hand last passed it survives one more revolution: + * the classic second chance. + * + * This occupies the bit position of the 0..5 usage counter it replaces. The + * counter let a buffer accumulate up to five chances, which cost a full + * revolution each to spend; one chance is enough to protect a working set and + * makes the work the sweep must do to find a victim independent of how hot the + * pool has been. + */ +#define BM_SECOND_CHANCE BUF_DEFINE_FLAG( 6) /* have waiter for sole pin */ #define BM_PIN_COUNT_WAITER BUF_DEFINE_FLAG( 7) /* must write for checkpoint */ @@ -133,18 +165,6 @@ StaticAssertDecl(MAX_BACKENDS_BITS <= (BUF_LOCK_BITS - 2), "MAX_BACKENDS_BITS needs to be <= BUF_LOCK_BITS - 2"); -/* - * The maximum allowed value of usage_count represents a tradeoff between - * accuracy and speed of the clock-sweep buffer management algorithm. A - * large value (comparable to NBuffers) would approximate LRU semantics. - * But it can take as many as BM_MAX_USAGE_COUNT+1 complete cycles of the - * clock-sweep hand to find a free buffer, so in practice we don't want the - * value to be very large. - */ -#define BM_MAX_USAGE_COUNT 5 - -StaticAssertDecl(BM_MAX_USAGE_COUNT < (UINT64CONST(1) << BUF_USAGECOUNT_BITS), - "BM_MAX_USAGE_COUNT doesn't fit in BUF_USAGECOUNT_BITS bits"); /* * Buffer tag identifies which disk block the buffer contains. @@ -365,10 +385,10 @@ typedef struct BufferDesc * line sized. * * XXX: As this is primarily matters in highly concurrent workloads which - * probably all are 64bit these days, and the space wastage would be a bit - * more noticeable on 32bit systems, we don't force the stride to be cache - * line sized on those. If somebody does actual performance testing, we can - * reevaluate. + * probably all are 64bit these days. We force the stride to be cache line + * sized even on 32bit systems, where the space wastage is be a bit more + * noticeable, to allow partitioning of shared buffers (which requires the + * memory page be a multiple of buffer descriptor). * * Note that local buffer descriptors aren't forced to be aligned - as there's * no concurrent access to those it's unlikely to be beneficial. @@ -378,7 +398,7 @@ typedef struct BufferDesc * platform with either 32 or 128 byte line sizes, it's good to align to * boundaries and avoid false sharing. */ -#define BUFFERDESC_PAD_TO_SIZE (SIZEOF_VOID_P == 8 ? 64 : 1) +#define BUFFERDESC_PAD_TO_SIZE 64 typedef union BufferDescPadded { @@ -411,8 +431,17 @@ typedef struct WritebackContext /* in buf_init.c */ extern PGDLLIMPORT BufferDescPadded *BufferDescriptors; +extern PGDLLIMPORT BufferPartitions *BufferPartitionsRegistry; extern PGDLLIMPORT ConditionVariableMinimallyPadded *BufferIOCVArray; +extern int BufferPartitionCount(void); +extern void BufferPartitionGet(int idx, int *node, int *num_buffers, + int *first_buffer, int *last_buffer); +extern void BufferPartitionsCalculate(int *num_nodes, int *num_partitions, + int *num_partitions_per_node); +extern void BufferPartitionsParams(int *num_nodes, int *num_partitions, + int *num_partitions_per_node); + /* in localbuf.c */ extern PGDLLIMPORT BufferDesc *LocalBufferDescriptors; @@ -468,9 +497,6 @@ UnlockBufHdr(BufferDesc *desc) * unset_bits, adding the flags in set_bits, and changing the refcount by * refcount_change. If a flag is both cleared and added, it will end up being * set. - * - * Note that this approach would not trivially work for usagecount, since we - * need to cap the usagecount at BM_MAX_USAGE_COUNT. */ static inline uint64 UnlockBufHdrExt(BufferDesc *desc, uint64 old_buf_state, @@ -590,7 +616,11 @@ extern BufferDesc *StrategyGetBuffer(BufferAccessStrategy strategy, extern bool StrategyRejectBuffer(BufferAccessStrategy strategy, BufferDesc *buf, bool from_ring); -extern int StrategySyncStart(uint32 *complete_passes, uint32 *num_buf_alloc); +extern void StrategySyncBalance(void); +extern void StrategySyncPrepare(int *num_parts, uint32 *num_buf_alloc); +extern int StrategySyncStart(int partition, uint32 *complete_passes, + int *first_buffer, int *num_buffers); +extern uint64 StrategySweepClaims(int partition); extern void StrategyNotifyBgWriter(int bgwprocno); /* buf_table.c */ diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 6837b35fc6d0b..b5c3873acae68 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -155,6 +155,29 @@ struct ReadBuffersOperation typedef struct ReadBuffersOperation ReadBuffersOperation; +/* + * information about one partition of shared buffers + * + * numa_nod specifies node for this partition (-1 means allocated on any node) + * first/last buffer - the values are inclusive + */ +typedef struct BufferPartition +{ + int numa_node; /* NUMA node (-1 no node) */ + int num_buffers; /* number of buffers */ + int first_buffer; /* first buffer of partition */ + int last_buffer; /* last buffer of partition */ +} BufferPartition; + +/* an array of information about all partitions */ +typedef struct BufferPartitions +{ + int nnodes; /* number of NUMA nodes */ + int npartitions; /* number of partitions */ + int npartitions_per_node; /* for convenience */ + BufferPartition partitions[FLEXIBLE_ARRAY_MEMBER]; +} BufferPartitions; + /* to avoid having to expose buf_internals.h here */ typedef struct WritebackContext WritebackContext; @@ -167,6 +190,11 @@ extern PGDLLIMPORT int bgwriter_lru_maxpages; extern PGDLLIMPORT double bgwriter_lru_multiplier; extern PGDLLIMPORT bool track_io_timing; +/* in freelist.c */ +extern PGDLLIMPORT bool clocksweep_balance; +extern PGDLLIMPORT bool clocksweep_balance_recalc; +extern PGDLLIMPORT bool clocksweep_scan_all_partitions; + #define DEFAULT_EFFECTIVE_IO_CONCURRENCY 16 #define DEFAULT_MAINTENANCE_IO_CONCURRENCY 16 extern PGDLLIMPORT int effective_io_concurrency; @@ -187,6 +215,7 @@ extern PGDLLIMPORT const PgAioHandleCallbacks aio_local_buffer_readv_cb; /* in buf_init.c */ extern PGDLLIMPORT char *BufferBlocks; +extern PGDLLIMPORT bool shared_buffers_numa; /* in localbuf.c */ extern PGDLLIMPORT int NLocBuffer; @@ -371,6 +400,9 @@ extern void MarkDirtyAllUnpinnedBuffers(int32 *buffers_dirtied, int32 *buffers_already_dirty, int32 *buffers_skipped); +/* in buf_init.c */ +extern int BufferGetNode(Buffer buffer); + /* in localbuf.c */ extern void AtProcExit_LocalBuffers(void); @@ -383,7 +415,15 @@ extern int GetAccessStrategyBufferCount(BufferAccessStrategy strategy); extern int GetAccessStrategyPinLimit(BufferAccessStrategy strategy); extern void FreeAccessStrategy(BufferAccessStrategy strategy); - +extern void ClockSweepPartitionGetInfo(int idx, + uint32 *complete_passes, + uint32 *next_victim_buffer, + uint64 *buffer_total_allocs, + uint32 *buffer_allocs, + uint64 *buffer_total_req_allocs, + uint32 *buffer_req_allocs, + int **weights); +extern void ClockSweepSetWeights(int partition, int *weights, int nweights); /* inline functions */ diff --git a/src/port/pg_numa.c b/src/port/pg_numa.c index 8954669273ae3..e62f858e4a743 100644 --- a/src/port/pg_numa.c +++ b/src/port/pg_numa.c @@ -18,6 +18,9 @@ #include "miscadmin.h" #include "port/pg_numa.h" +#include "storage/pg_shmem.h" + +int numa_flags; /* * At this point we provide support only for Linux thanks to libnuma, but in @@ -118,6 +121,94 @@ pg_numa_get_max_node(void) return numa_max_node(); } +/* + * pg_numa_move_to_node + * move memory to different NUMA nodes in larger chunks + * + * startptr - start of the region (should be aligned to page size) + * endptr - end of the region (doesn't need to be aligned) + * node - node to move the memory to + * + * The "startptr" is expected to be a multiple of system memory page size, as + * determined by pg_numa_page_size. + * + * XXX We only expect to do this during startup, when the shared memory is + * still being setup. + */ +void +pg_numa_move_to_node(char *startptr, char *endptr, int node) +{ + Size sz = (endptr - startptr); + + Assert((int64) startptr % pg_numa_page_size() == 0); + + /* + * numa_tonode_memory does not actually cause a page fault, and thus does + * not locate the memory on the node. So it's fast, at least compared to + * pg_numa_query_pages, and does not make startup longer. But it also + * means the expensive part happen later, on the first access. + */ + numa_tonode_memory(startptr, sz, node); +} + +int +pg_numa_bind_to_node(char *startptr, char *endptr, int node) +{ + int ret; + struct bitmask *nodemask; + + if (node < 0) + { + errno = EINVAL; + return -1; + } + + nodemask = numa_allocate_nodemask(); + if (nodemask == NULL) + { + errno = ENOMEM; + return -1; + } + + numa_bitmask_setbit(nodemask, node); + + /* + * MPOL_BIND places the pages strictly on the node, and MPOL_MF_MOVE migrates + * pages already faulted in to that node. If mbind() fails, leave the default + * placement in effect, and report the failure. + */ + ret = mbind(startptr, (endptr - startptr), + MPOL_BIND, nodemask->maskp, nodemask->size, MPOL_MF_MOVE); + + numa_free_nodemask(nodemask); + + return ret; +} + +Size +pg_numa_page_size(void) +{ + Size os_page_size; + Size huge_page_size; + +#ifdef WIN32 + SYSTEM_INFO sysinfo; + + GetSystemInfo(&sysinfo); + os_page_size = sysinfo.dwPageSize; +#else + os_page_size = sysconf(_SC_PAGESIZE); +#endif + + /* assume huge pages get used, unless HUGE_PAGES_OFF */ + if (huge_pages_status != HUGE_PAGES_OFF) + GetHugePageSize(&huge_page_size, NULL); + else + huge_page_size = 0; + + return Max(os_page_size, huge_page_size); +} + #else /* Empty wrappers */ @@ -140,4 +231,27 @@ pg_numa_get_max_node(void) return 0; } +void +pg_numa_move_to_node(char *startptr, char *endptr, int node) +{ + /* we don't expect to ever get here in builds without libnuma */ + Assert(false); +} + +int +pg_numa_bind_to_node(char *startptr, char *endptr, int node) +{ + /* we don't expect to ever get here in builds without libnuma */ + Assert(false); + return -1; +} + +Size +pg_numa_page_size(void) +{ + /* we don't expect to ever get here in builds without libnuma */ + Assert(false); + return 0; +} + #endif diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index 13944701bc7b0..278cbfb735ce0 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -1243,7 +1243,7 @@ spawn_process(const char *cmdline) char *cmdline2; cmdline2 = psprintf("exec %s", cmdline); - execl(shellprog, shellprog, "-c", cmdline2, (char *) NULL); + execlp(shellprog, shellprog, "-c", cmdline2, (char *) NULL); /* Not using the normal bail() here as we want _exit */ bail_noatexit("could not exec \"%s\": %m", shellprog); } diff --git a/src/tools/pgindent/pgindent b/src/tools/pgindent/pgindent index fa2041dc34b93..92526f6248581 100755 --- a/src/tools/pgindent/pgindent +++ b/src/tools/pgindent/pgindent @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # Copyright (c) 2021-2026, PostgreSQL Global Development Group diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index a7a8dbdd23f79..6559cdaeaf1e7 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -361,6 +361,8 @@ BufferHeapTupleTableSlot BufferLockMode BufferLookupEnt BufferManagerRelation +BufferPartition +BufferPartitions BufferStrategyControl BufferTag BufferUsage @@ -445,6 +447,7 @@ ClientCertName ClientConnectionInfo ClientData ClientSocket +ClockSweep ClonePtrType ClosePortalStmt ClosePtrType