From d81578e5e42d6ebf3d62d8903b60806b131cdde4 Mon Sep 17 00:00:00 2001 From: Arun Sharma Date: Fri, 25 Sep 2026 12:02:52 -0700 Subject: [PATCH] fix(vector): use index metric and efc for post-build HNSW inserts OnDiskHNSWIndex::HNSWInsertState built its search state with a default-constructed HNSWIndexConfig (metric = cosine) and default QueryHNSWConfig (efs = 200), so rows inserted after CREATE_VECTOR_INDEX (CREATE/MERGE via initInsertState, COPY FROM via finalize, embedding updates via initUpdateState) picked HNSW neighbours by cosine distance regardless of the index metric, silently lowering recall on l2/l2sq/ dotproduct indexes over non-normalized vectors. Thread the index config through and map efc onto the insert search ef. Related to LadybugDB/ladybug#1023 --- vector/src/include/index/hnsw_index.h | 2 +- vector/src/index/hnsw_index.cpp | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/vector/src/include/index/hnsw_index.h b/vector/src/include/index/hnsw_index.h index fdcc3705..55f0e4d8 100644 --- a/vector/src/include/index/hnsw_index.h +++ b/vector/src/include/index/hnsw_index.h @@ -348,7 +348,7 @@ class OnDiskHNSWIndex final : public HNSWIndex { HNSWInsertState(main::ClientContext* context, catalog::TableCatalogEntry* nodeTableEntry, catalog::TableCatalogEntry* upperRelTableEntry, catalog::TableCatalogEntry* lowerRelTableEntry, storage::NodeTable& nodeTable, - common::column_id_t columnID, uint64_t degree); + common::column_id_t columnID, uint64_t degree, const HNSWIndexConfig& indexConfig); }; struct HNSWUpdateState final : UpdateState { diff --git a/vector/src/index/hnsw_index.cpp b/vector/src/index/hnsw_index.cpp index 8b1e07cb..4f951a1f 100644 --- a/vector/src/index/hnsw_index.cpp +++ b/vector/src/index/hnsw_index.cpp @@ -482,13 +482,19 @@ HNSWSearchState::HNSWSearchState(main::ClientContext* context, upperGraph = std::make_unique(context, std::move(upperGraphEntry)); } +static QueryHNSWConfig getInsertQueryConfig(const HNSWIndexConfig& indexConfig) { + QueryHNSWConfig queryConfig; + queryConfig.efs = indexConfig.efc; + return queryConfig; +} + OnDiskHNSWIndex::HNSWInsertState::HNSWInsertState(main::ClientContext* context, catalog::TableCatalogEntry* nodeTableEntry, catalog::TableCatalogEntry* upperRelTableEntry, catalog::TableCatalogEntry* lowerRelTableEntry, NodeTable& nodeTable, - common::column_id_t columnID, uint64_t degree) + common::column_id_t columnID, uint64_t degree, const HNSWIndexConfig& indexConfig) : searchState{context, nodeTableEntry, upperRelTableEntry, lowerRelTableEntry, nodeTable, columnID, nodeTable.getNumTotalRows(Transaction::Get(*context)), degree, - QueryHNSWConfig{}, HNSWIndexConfig{}, false} { + getInsertQueryConfig(indexConfig), indexConfig, false} { std::vector insertTypes; insertTypes.push_back(common::LogicalType::INTERNAL_ID()); insertTypes.push_back(common::LogicalType::INTERNAL_ID()); @@ -722,7 +728,7 @@ std::unique_ptr OnDiskHNSWIndex::initInsertState(main::Clien auto [nodeTableEntry, upperRelTableEntry, lowerRelTableEntry] = getIndexTableCatalogEntries(catalog::Catalog::Get(*context), transaction, indexInfo); return std::make_unique(context, nodeTableEntry, upperRelTableEntry, - lowerRelTableEntry, nodeTable, indexInfo.columnIDs[0], config.ml); + lowerRelTableEntry, nodeTable, indexInfo.columnIDs[0], config.ml, config); } class CommitInsertEmbeddingScanState final : public GetEmbeddingsScanState { @@ -823,8 +829,9 @@ void OnDiskHNSWIndex::finalize(main::ClientContext* context) { const auto embeddingDim = typeInfo.constPtrCast()->getNumElements(); const auto scanState = std::make_unique(transaction, mm, nodeTable, indexInfo.columnIDs[0], embeddingDim); - const auto insertState = std::make_unique(context, nodeTableEntry, - upperRelTableEntry, lowerRelTableEntry, nodeTable, indexInfo.columnIDs[0], config.ml); + const auto insertState = + std::make_unique(context, nodeTableEntry, upperRelTableEntry, + lowerRelTableEntry, nodeTable, indexInfo.columnIDs[0], config.ml, config); std::unique_ptr quantizedEmbeddings; if (config.quantization != QuantizationType::NONE) { DASSERT(quantizedEmbeddingsTable != nullptr);