Skip to content

fix(fts): snapshot FTS aux info in bind data (Fixes ladybugdb/ladybug#1105) #94

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adsharma merged 2 commits into from
Oct 7, 2026
Merged

fix(fts): snapshot FTS aux info in bind data (Fixes ladybugdb/ladybug#1105) #94

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
772519e
fix(fts): snapshot FTS aux info in bind data (Fixes ladybugdb/ladybug…
adsharma Oct 4, 2026
fc0fa8c
fix(fts): invalidate cached FTS plan on index drop/recreate
adsharma Oct 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 44 additions & 3 deletions fts/src/function/query_fts_bind_data.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@
#include "catalog/fts_index_catalog_entry.h"
#include "common/exception/binder.h"
#include "common/string_utils.h"
#include "index/fts_index.h"
#include "libstemmer.h"
#include "main/client_context.h"
#include "re2.h"
#include "storage/storage_manager.h"
#include "storage/table/node_table.h"
#include "utils/fts_utils.h"
#include <format>

namespace lbug {
namespace fts_extension {
Expand Down Expand Up @@ -46,7 +49,7 @@ void QueryFTSOptionalParams::evaluateParams(main::ClientContext* context) {
std::vector<std::string> QueryFTSBindData::getQueryTerms(main::ClientContext& context) const {
auto queryInStr =
ExpressionUtil::evaluateLiteral<std::string>(&context, query, LogicalType::STRING());
auto config = entry.getAuxInfo().cast<FTSIndexAuxInfo>().config;
auto config = auxInfo.config;
FTSUtils::normalizeQuery(queryInStr, config.ignorePatternQuery,
true /* protectWildcardChars */);
auto terms = FTSUtils::tokenizeString(queryInStr, config);
Expand All @@ -57,11 +60,49 @@ std::vector<std::string> QueryFTSBindData::getQueryTerms(main::ClientContext& co
config.stopWordsTableName)
->getTableID())
->ptrCast<NodeTable>();
return FTSUtils::stemTerms(terms, entry.getAuxInfo().cast<FTSIndexAuxInfo>().config,
MemoryManager::Get(context), stopWordsTable, transaction::Transaction::Get(context),
return FTSUtils::stemTerms(terms, auxInfo.config, MemoryManager::Get(context), stopWordsTable,
transaction::Transaction::Get(context),
optionalParams->constCast<QueryFTSOptionalParams>().conjunctive.getParamVal(),
true /* isQuery */);
}

void QueryFTSBindData::refreshFromCatalog(main::ClientContext* context) {
std::lock_guard guard{refreshMutex};
context->setUseInternalCatalogEntry(true /* useInternalCatalogEntry */);
try {
auto catalog = catalog::Catalog::Get(*context);
auto transaction = transaction::Transaction::Get(*context);
auto tableEntry = catalog->getTableCatalogEntry(transaction, tableName);
if (!catalog->containsIndex(transaction, tableEntry->getTableID(), indexName)) {
throw common::BinderException{std::format(
"Table {} doesn't have an index with name {}.", tableEntry->getName(), indexName)};
}
auto ftsIndexEntry = catalog->getIndex(transaction, tableEntry->getTableID(), indexName);
auto termsEntry = catalog->getTableCatalogEntry(transaction,
FTSUtils::getTermsTableName(tableEntry->getTableID(), indexName));
auto docsEntry = catalog->getTableCatalogEntry(transaction,
FTSUtils::getDocsTableName(tableEntry->getTableID(), indexName));
auto appearsInEntry = catalog->getTableCatalogEntry(transaction,
FTSUtils::getAppearsInTableName(tableEntry->getTableID(), indexName));
graphEntry = graph::NativeGraphEntry({termsEntry, docsEntry}, {appearsInEntry});
auxInfo.config = ftsIndexEntry->getAuxInfo().cast<FTSIndexAuxInfo>().config;
auto nodeTable = StorageManager::Get(*context)
->getTable(ftsIndexEntry->getTableID())
->ptrCast<NodeTable>();
auto index = nodeTable->getIndex(indexName);
if (!index.has_value()) {
throw common::BinderException{std::format(
"Table {} doesn't have an index with name {}.", tableEntry->getName(), indexName)};
}
auto [numDocs_, avgDocLen_] = index.value()->cast<FTSIndex>().getStats(transaction);
numDocs = numDocs_;
avgDocLen = avgDocLen_;
} catch (...) {
context->setUseInternalCatalogEntry(false /* useInternalCatalogEntry */);
throw;
}
context->setUseInternalCatalogEntry(false /* useInternalCatalogEntry */);
}

} // namespace fts_extension
} // namespace lbug
55 changes: 42 additions & 13 deletions fts/src/function/query_fts_index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,7 @@ struct QFTSEdgeCompute final : EdgeCompute {
: scores{scores}, dfs{dfs}, scoresMutex{std::make_shared<std::mutex>()} {}

QFTSEdgeCompute(node_id_map_t<ScoreInfo>& scores,
const std::unordered_map<offset_t, uint64_t>& dfs,
std::shared_ptr<std::mutex> scoresMutex)
const std::unordered_map<offset_t, uint64_t>& dfs, std::shared_ptr<std::mutex> scoresMutex)
: scores{scores}, dfs{dfs}, scoresMutex{std::move(scoresMutex)} {}

std::vector<nodeID_t> edgeCompute(nodeID_t boundNodeID, graph::NbrScanState::Chunk& resultChunk,
Expand Down Expand Up @@ -212,7 +211,7 @@ void QFTSOutputWriter::write(processor::FactorizedTable& scoreFT, nodeID_t docNo
scoreInfo.scoreData.size() != numUniqueTerms) {
return;
}
auto auxInfo = bindData.entry.getAuxInfo().cast<FTSIndexAuxInfo>();
auto auxInfo = bindData.auxInfo;
for (auto& scoreData : scoreInfo.scoreData) {
auto numDocs = bindData.numDocs;
auto avgDocLen = bindData.avgDocLen;
Expand Down Expand Up @@ -385,11 +384,39 @@ static void initFrontier(FrontierPair& frontierPair, table_id_t termsTableID,

static offset_t tableFunc(const TableFuncInput& input, TableFuncOutput&) {
auto& clientContext = *input.context->clientContext;
auto transaction = transaction::Transaction::Get(clientContext);
auto& qFTSBindData = input.bindData->cast<QueryFTSBindData>();
// The prepared-plan cache reuses this bind data across executions, so re-resolve the
// index on every execution. This throws when the index was dropped (matching the
// literal path) and picks up the new backing tables after a recreate.
qFTSBindData.refreshFromCatalog(&clientContext);
auto sharedState = input.sharedState->ptrCast<QFTSSharedState>();
// The cached physical plan also reuses the shared state (and its graph) across
// executions, so rebuild the graph when the backing tables changed.
{
auto cachedEntry = sharedState->graph->getGraphEntry();
bool stale = cachedEntry->nodeInfos.size() != qFTSBindData.graphEntry.nodeInfos.size() ||
cachedEntry->relInfos.size() != qFTSBindData.graphEntry.relInfos.size();
for (size_t i = 0; !stale && i < cachedEntry->nodeInfos.size(); ++i) {
if (cachedEntry->nodeInfos[i].entry->getTableID() !=
qFTSBindData.graphEntry.nodeInfos[i].entry->getTableID()) {
stale = true;
}
}
for (size_t i = 0; !stale && i < cachedEntry->relInfos.size(); ++i) {
if (cachedEntry->relInfos[i].entry->getTableID() !=
qFTSBindData.graphEntry.relInfos[i].entry->getTableID()) {
stale = true;
}
}
if (stale) {
sharedState->graph = std::make_unique<graph::OnDiskGraph>(&clientContext,
qFTSBindData.graphEntry.copy());
sharedState->outputTableID = qFTSBindData.outputTableID;
}
}
auto transaction = transaction::Transaction::Get(clientContext);
auto graph = sharedState->graph.get();
auto graphEntry = graph->getGraphEntry();
auto& qFTSBindData = input.bindData->cast<QueryFTSBindData>();
auto qFTSOptionalParams = qFTSBindData.optionalParams->constCast<QueryFTSOptionalParams>();
if (qFTSOptionalParams.topK.isSet()) {
sharedState->ptrCast<QFTSTopKSharedState>()->setTopK(qFTSOptionalParams.topK.getParamVal());
Expand Down Expand Up @@ -492,7 +519,8 @@ static std::unique_ptr<TableFuncBindData> bindFunc(main::ClientContext* context,
auto& ftsIndex = index.value()->cast<FTSIndex>();
auto [numDocs, avgDocLen] = ftsIndex.getStats(transaction);
auto bindData = std::make_unique<QueryFTSBindData>(std::move(columns), std::move(graphEntry),
nodeOutput, std::move(query), *ftsIndexEntry,
nodeOutput, std::move(query), ftsIndexEntry->getAuxInfo().cast<FTSIndexAuxInfo>(),
inputTableName, indexName,
std::make_unique<QueryFTSOptionalParams>(input->optionalParamsLegacy), numDocs, avgDocLen);
context->setUseInternalCatalogEntry(false /* useInternalCatalogEntry */);
return bindData;
Expand Down Expand Up @@ -521,16 +549,17 @@ static void getLogicalPlan(Planner* planner, const BoundReadingClause& readingCl
}

std::shared_ptr<TableFuncSharedState> initSharedState(const TableFuncInitSharedStateInput& input) {
auto bindData = input.bindData->constPtrCast<QueryFTSBindData>();
auto& bindData = input.bindData->cast<QueryFTSBindData>();
bindData.refreshFromCatalog(input.context->clientContext);
auto graph = std::make_unique<graph::OnDiskGraph>(input.context->clientContext,
bindData->graphEntry.copy());
if (!bindData->optionalParams->constCast<QueryFTSOptionalParams>().topK.isSet()) {
bindData.graphEntry.copy());
if (!bindData.optionalParams->constCast<QueryFTSOptionalParams>().topK.isSet()) {
// The user does not give a topK parameter, skip topK optimization.
return std::make_shared<QFTSSharedState>(bindData->getResultTable(), std::move(graph),
bindData->outputTableID);
return std::make_shared<QFTSSharedState>(bindData.getResultTable(), std::move(graph),
bindData.outputTableID);
} else {
return std::make_shared<QFTSTopKSharedState>(bindData->getResultTable(), std::move(graph),
bindData->outputTableID);
return std::make_shared<QFTSTopKSharedState>(bindData.getResultTable(), std::move(graph),
bindData.outputTableID);
}
}

Expand Down
34 changes: 29 additions & 5 deletions fts/src/include/function/query_fts_bind_data.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
#pragma once

#include <mutex>
#include <string>

#include "binder/expression/node_expression.h"
#include "catalog/catalog_entry/index_catalog_entry.h"
#include "catalog/fts_index_catalog_entry.h"
#include "function/fts_config.h"
#include "function/gds/gds.h"

Expand Down Expand Up @@ -31,18 +34,31 @@ struct QueryFTSOptionalParams : public function::OptionalParams {

struct QueryFTSBindData final : public function::GDSBindData {
std::shared_ptr<binder::Expression> query;
const catalog::IndexCatalogEntry& entry;
// Owned copy of the FTS aux info. This must NOT be a reference/pointer into the
// catalog: bind data outlives the bind transaction (prepared-plan cache reuses it
// across executions on the same connection), so a catalog reference dangles on the
// second execution and segfaults (see issue #1082).
FTSIndexAuxInfo auxInfo;
// Identity of the queried index. The prepared-plan cache reuses this bind data
// across executions, so every execution must re-resolve these against the catalog
// (see refreshFromCatalog): after DROP_FTS_INDEX the cached snapshot would
// otherwise keep serving stale results, and after a recreate it would reference
// dropped backing tables.
std::string tableName;
std::string indexName;
common::table_id_t outputTableID;
common::idx_t numDocs;
double avgDocLen;
mutable std::mutex refreshMutex;

QueryFTSBindData(binder::expression_vector columns, graph::NativeGraphEntry graphEntry,
std::shared_ptr<binder::Expression> docs, std::shared_ptr<binder::Expression> query,
const catalog::IndexCatalogEntry& entry,
const FTSIndexAuxInfo& auxInfo, std::string tableName, std::string indexName,
std::unique_ptr<QueryFTSOptionalParams> optionalParams, common::idx_t numDocs,
double avgDocLen)
: GDSBindData{std::move(columns), std::move(graphEntry), binder::expression_vector{docs}},
query{std::move(query)}, entry{entry},
query{std::move(query)}, auxInfo{auxInfo}, tableName{std::move(tableName)},
indexName{std::move(indexName)},
outputTableID{output[0]->constCast<binder::NodeExpression>().getTableIDs()[0]},
numDocs{numDocs}, avgDocLen{avgDocLen} {
auto& nodeExpr = output[0]->constCast<binder::NodeExpression>();
Expand All @@ -51,9 +67,17 @@ struct QueryFTSBindData final : public function::GDSBindData {
this->optionalParams = std::move(optionalParams);
}
QueryFTSBindData(const QueryFTSBindData& other)
: GDSBindData{other}, query{other.query}, entry{other.entry},
: GDSBindData{other}, query{other.query}, auxInfo{other.auxInfo},
tableName{other.tableName}, indexName{other.indexName},
outputTableID{other.outputTableID}, numDocs{other.numDocs}, avgDocLen{other.avgDocLen} {}

// Re-resolve the index against the current catalog and refresh the cached snapshot
// (aux info, backing-table graph entry, index stats). Throws BinderException when the
// index no longer exists so the parameterized path matches the literal path.
// Must be called at execution time before the snapshot is used, because the
// prepared-plan cache reuses this bind data across executions on the same connection.
void refreshFromCatalog(main::ClientContext* context);

std::vector<std::string> getQueryTerms(main::ClientContext& context) const;

std::unique_ptr<TableFuncBindData> copy() const override {
Expand Down
Loading