From 1b0acfc61a950aea9f970e377cf106ef5eb954de Mon Sep 17 00:00:00 2001 From: Alexey Sharov Date: Sat, 13 Jun 2026 10:39:55 +0500 Subject: [PATCH 1/2] db/datastruct/btindex: use off-heap mmap for EliasFano during BTree index build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NewEliasFano allocates a multi-GB contiguous bit array on the Go heap for large snapshots (mainnet: ~2B keys → ~3.5 GB). NewEliasFanoOffHeap backs the same arrays with a mmap'd temp file so the OS pages it, avoiding the heap spike. --- db/datastruct/btindex/btree_index.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db/datastruct/btindex/btree_index.go b/db/datastruct/btindex/btree_index.go index 0cd8062f28d..2f85e44bfb6 100644 --- a/db/datastruct/btindex/btree_index.go +++ b/db/datastruct/btindex/btree_index.go @@ -271,7 +271,12 @@ func (btw *BtIndexWriter) Build() error { log.Log(btw.args.Lvl, "[index] calculating", "file", btw.indexFileName) if btw.keysWritten > 0 { - btw.ef = eliasfano32.NewEliasFano(btw.keysWritten, btw.maxOffset) + efBuilder, err := eliasfano32.NewEliasFanoOffHeap(btw.keysWritten, btw.maxOffset, btw.args.TmpDir) + if err != nil { + return fmt.Errorf("[index] create offheap ef: %w", err) + } + defer efBuilder.Close() + btw.ef = efBuilder.EliasFano nodes := make([]Node, 0, btw.keysWritten/btw.args.M) var ki uint64 From 3af43f9a5b0f477de897e2d936dfb84fe159f245 Mon Sep 17 00:00:00 2001 From: Alexey Sharov Date: Sat, 13 Jun 2026 10:56:03 +0500 Subject: [PATCH 2/2] save --- db/datastruct/btindex/btree_index_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/db/datastruct/btindex/btree_index_test.go b/db/datastruct/btindex/btree_index_test.go index cddf9c1ad46..210bcb20022 100644 --- a/db/datastruct/btindex/btree_index_test.go +++ b/db/datastruct/btindex/btree_index_test.go @@ -400,6 +400,7 @@ func TestNewBtIndex(t *testing.T) { require.NotNil(t, bt) bplus := bt.bplus require.GreaterOrEqual(t, len(bplus.mx), keyCount/int(DefaultBtreeM)) + require.LessOrEqual(t, len(bplus.mx), keyCount/int(DefaultBtreeM)+2) for i := 1; i < len(bt.bplus.mx); i++ { require.NotZero(t, bt.bplus.mx[i].di)