From 1f99d59b1f8a405cec5d516b007a2917a66731ec Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Sun, 24 May 2026 17:38:49 +0100 Subject: [PATCH 01/14] Add SchemaIdentifier tag to scan expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce SchemaIdentifier value type (null = current schema) and attach it as a metadata field to FullUnorderedScanExpression and LogicalTypeFilterExpression. Both expressions gain a getSchemaId() accessor and a second constructor/factory overload accepting a SchemaIdentifier. The field does not participate in equalsWithoutChildren or computeHashCodeWithoutChildren to keep planning behavior unchanged — ImplementNestedLoopJoinRule will read it directly in a later step. --- .../query/plan/cascades/SchemaIdentifier.java | 93 +++++++++++++++++++ .../FullUnorderedScanExpression.java | 16 +++- .../LogicalTypeFilterExpression.java | 35 ++++++- 3 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/SchemaIdentifier.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/SchemaIdentifier.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/SchemaIdentifier.java new file mode 100644 index 0000000000..4059998a92 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/SchemaIdentifier.java @@ -0,0 +1,93 @@ +/* + * SchemaIdentifier.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + +import com.apple.foundationdb.annotation.API; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Objects; + +/** + * Identifies a schema (record store) within a query plan. A {@code null} schema name means "the + * schema of the current connection" — the default for single-schema queries. + * + *

Used to tag scan expressions ({@link com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression}, + * {@link com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalTypeFilterExpression}) + * so that {@code ImplementNestedLoopJoinRule} can detect when both sides of a join belong to + * different schemas and wrap the inner side in a {@code RecordQueryStoreBindingPlan}. + */ +@API(API.Status.EXPERIMENTAL) +public final class SchemaIdentifier { + + private static final SchemaIdentifier CURRENT = new SchemaIdentifier(null); + + @Nullable + private final String schemaName; + + private SchemaIdentifier(@Nullable final String schemaName) { + this.schemaName = schemaName; + } + + /** Returns the singleton representing the current connection's schema. */ + @Nonnull + public static SchemaIdentifier current() { + return CURRENT; + } + + /** Returns a {@code SchemaIdentifier} for the named schema. */ + @Nonnull + public static SchemaIdentifier of(@Nonnull final String schemaName) { + return new SchemaIdentifier(schemaName); + } + + /** Returns {@code true} if this identifier represents the current connection schema. */ + public boolean isCurrentSchema() { + return schemaName == null; + } + + /** Returns the schema name, or {@code null} if this is the current connection schema. */ + @Nullable + public String getSchemaName() { + return schemaName; + } + + @Override + public boolean equals(final Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + return Objects.equals(schemaName, ((SchemaIdentifier) other).schemaName); + } + + @Override + public int hashCode() { + return Objects.hashCode(schemaName); + } + + @Override + public String toString() { + return schemaName == null ? "" : schemaName; + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/FullUnorderedScanExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/FullUnorderedScanExpression.java index 6203da6fcb..31a31fc0aa 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/FullUnorderedScanExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/FullUnorderedScanExpression.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.query.plan.cascades.AccessHints; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange; import com.apple.foundationdb.record.query.plan.cascades.Compensation; @@ -74,10 +75,18 @@ public class FullUnorderedScanExpression extends AbstractRelationalExpressionWit @Nonnull final AccessHints accessHints; + @Nonnull + private final SchemaIdentifier schemaId; + public FullUnorderedScanExpression(@Nonnull final Set recordTypes, @Nonnull final Type flowedType, @Nonnull final AccessHints accessHints) { + this(recordTypes, flowedType, accessHints, SchemaIdentifier.current()); + } + + public FullUnorderedScanExpression(@Nonnull final Set recordTypes, @Nonnull final Type flowedType, @Nonnull final AccessHints accessHints, @Nonnull final SchemaIdentifier schemaId) { this.recordTypes = ImmutableSet.copyOf(recordTypes); this.flowedType = flowedType; this.accessHints = accessHints; + this.schemaId = schemaId; } @Nonnull @@ -90,6 +99,11 @@ public AccessHints getAccessHints() { return accessHints; } + @Nonnull + public SchemaIdentifier getSchemaId() { + return schemaId; + } + @Nonnull @Override public Value getResultValue() { @@ -114,7 +128,7 @@ public FullUnorderedScanExpression translateCorrelations(@Nonnull final Translat final boolean shouldSimplifyValues, @Nonnull final List translatedQuantifiers) { Verify.verify(translatedQuantifiers.isEmpty()); - // this is ok as there are no new quantifiers + // schemaId is not correlation-dependent; return this unchanged return this; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/LogicalTypeFilterExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/LogicalTypeFilterExpression.java index fd4f1e8ab2..0212fbc66b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/LogicalTypeFilterExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/LogicalTypeFilterExpression.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.record.query.plan.cascades.PredicateMultiMap; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.Quantifiers; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; import com.apple.foundationdb.record.query.plan.cascades.ValueEquivalence; import com.apple.foundationdb.record.query.plan.cascades.explain.Attribute; import com.apple.foundationdb.record.query.plan.cascades.explain.NodeInfo; @@ -84,13 +85,17 @@ public class LogicalTypeFilterExpression extends AbstractRelationalExpressionWit private final Quantifier innerQuantifier; @Nonnull private final Type resultType; + @Nonnull + private final SchemaIdentifier schemaId; private LogicalTypeFilterExpression(@Nonnull QueryPredicate recordTypePredicate, @Nonnull Set recordTypes, - @Nonnull Quantifier innerQuantifier, @Nonnull Type resultType) { + @Nonnull Quantifier innerQuantifier, @Nonnull Type resultType, + @Nonnull SchemaIdentifier schemaId) { this.recordTypePredicate = recordTypePredicate; this.recordTypes = ImmutableSet.copyOf(recordTypes); this.innerQuantifier = innerQuantifier; this.resultType = resultType; + this.schemaId = schemaId; } @Nonnull @@ -116,6 +121,11 @@ public QueryPredicate getRecordTypePredicate() { return recordTypePredicate; } + @Nonnull + public SchemaIdentifier getSchemaId() { + return schemaId; + } + @Override public int getRelationalChildCount() { return 1; @@ -139,7 +149,7 @@ public LogicalTypeFilterExpression translateCorrelations(@Nonnull final Translat @Nonnull final List translatedQuantifiers) { final var translatedPredicates = recordTypePredicate.translateCorrelations(translationMap, shouldSimplifyValues); return new LogicalTypeFilterExpression(translatedPredicates, getRecordTypes(), Iterables.getOnlyElement(translatedQuantifiers), - resultType); + resultType, schemaId); } @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") @@ -438,6 +448,15 @@ public static LogicalTypeFilterExpression forMatchCandidate(@Nonnull final Set recordTypes, + @Nonnull final Quantifier innerQuantifier, + @Nonnull final Type resultType, + @Nullable final CorrelationIdentifier recordTypeKeyParameterAlias, + @Nonnull final SchemaIdentifier schemaId) { final var value = new RecordTypeValue(QuantifiedObjectValue.of(innerQuantifier)); final var rangeConstraints = recordTypeNamesToRangeConstraints(recordTypes); @@ -452,7 +471,7 @@ public static LogicalTypeFilterExpression forMatchCandidate(@Nonnull final Set recordTypes, @Nonnull final Quantifier innerQuantifier, @Nonnull final Type resultType) { + return of(recordTypes, innerQuantifier, resultType, SchemaIdentifier.current()); + } + + @Nonnull + public static LogicalTypeFilterExpression of(@Nonnull final Set recordTypes, + @Nonnull final Quantifier innerQuantifier, + @Nonnull final Type resultType, + @Nonnull final SchemaIdentifier schemaId) { final var value = new RecordTypeValue(QuantifiedObjectValue.of(innerQuantifier)); final var rangeConstraints = recordTypeNamesToRangeConstraints(recordTypes); final var recordTypePredicate = PredicateWithValueAndRanges.ofRanges(value, rangeConstraints); - return new LogicalTypeFilterExpression(recordTypePredicate, recordTypes, innerQuantifier, resultType); + return new LogicalTypeFilterExpression(recordTypePredicate, recordTypes, innerQuantifier, resultType, schemaId); } @Nonnull From aaabe6355cbcc9fd3d8a7340c2b9323659e3c96b Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Sun, 24 May 2026 18:48:00 +0100 Subject: [PATCH 02/14] Add EvaluationContext auxiliary stores and RecordQueryStoreBindingPlan EvaluationContext gains an ImmutableMap> auxiliaryStores field (empty by default) plus getAuxiliaryStore(name) and withAuxiliaryStores(map) accessors. withBinding() preserves the map. RecordQueryStoreBindingPlan is a new transparent plan node that intercepts executePlan() and re-dispatches to the secondary store bound for schemaId in EvaluationContext. All planning properties delegate to the child plan. Visitor implementations added for all RecordQueryPlanVisitor and RelationalExpressionVisitor subscribers. The plan node is unreachable from any existing query path until ImplementNestedLoopJoinRule starts emitting it (Step 11). --- .../record/EvaluationContext.java | 57 ++++- .../record/EvaluationContextBuilder.java | 8 +- .../cascades/explain/ExplainPlanVisitor.java | 8 + .../properties/CardinalitiesProperty.java | 7 + .../properties/DerivationsProperty.java | 7 + .../properties/DistinctRecordsProperty.java | 7 + .../cascades/properties/OrderingProperty.java | 7 + .../properties/PrimaryKeyProperty.java | 7 + .../properties/StoredRecordProperty.java | 7 + .../plans/RecordQueryStoreBindingPlan.java | 240 ++++++++++++++++++ 10 files changed, 346 insertions(+), 9 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContext.java index 48d9029da8..77a85db348 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContext.java @@ -21,8 +21,10 @@ package com.apple.foundationdb.record; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; +import com.google.common.collect.ImmutableMap; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -44,7 +46,10 @@ public class EvaluationContext { @Nonnull private final TypeRepository typeRepository; - public static final EvaluationContext EMPTY = new EvaluationContext(Bindings.EMPTY_BINDINGS, TypeRepository.EMPTY_SCHEMA); + @Nonnull + private final ImmutableMap> auxiliaryStores; + + public static final EvaluationContext EMPTY = new EvaluationContext(Bindings.EMPTY_BINDINGS, TypeRepository.EMPTY_SCHEMA, ImmutableMap.of()); /** * Get an empty evaluation context. @@ -55,9 +60,11 @@ public static EvaluationContext empty() { return EMPTY; } - private EvaluationContext(@Nonnull Bindings bindings, @Nonnull TypeRepository typeRepository) { + private EvaluationContext(@Nonnull Bindings bindings, @Nonnull TypeRepository typeRepository, + @Nonnull ImmutableMap> auxiliaryStores) { this.bindings = bindings; this.typeRepository = typeRepository; + this.auxiliaryStores = auxiliaryStores; } /** @@ -68,7 +75,7 @@ private EvaluationContext(@Nonnull Bindings bindings, @Nonnull TypeRepository ty */ @Nonnull public static EvaluationContext forBindings(@Nonnull Bindings bindings) { - return new EvaluationContext(bindings, TypeRepository.EMPTY_SCHEMA); + return new EvaluationContext(bindings, TypeRepository.EMPTY_SCHEMA, ImmutableMap.of()); } /** @@ -80,12 +87,18 @@ public static EvaluationContext forBindings(@Nonnull Bindings bindings) { */ @Nonnull public static EvaluationContext forBindingsAndTypeRepository(@Nonnull Bindings bindings, @Nonnull TypeRepository typeRepository) { - return new EvaluationContext(bindings, typeRepository); + return new EvaluationContext(bindings, typeRepository, ImmutableMap.of()); + } + + @Nonnull + public static EvaluationContext forBindingsAndTypeRepository(@Nonnull Bindings bindings, @Nonnull TypeRepository typeRepository, + @Nonnull ImmutableMap> auxiliaryStores) { + return new EvaluationContext(bindings, typeRepository, auxiliaryStores); } @Nonnull public static EvaluationContext forTypeRepository(@Nonnull TypeRepository typeRepository) { - return new EvaluationContext(Bindings.EMPTY_BINDINGS, typeRepository); + return new EvaluationContext(Bindings.EMPTY_BINDINGS, typeRepository, ImmutableMap.of()); } /** @@ -97,7 +110,7 @@ public static EvaluationContext forTypeRepository(@Nonnull TypeRepository typeRe */ @Nonnull public static EvaluationContext forBinding(@Nonnull String bindingName, @Nullable Object value) { - return new EvaluationContext(Bindings.newBuilder().set(bindingName, value).build(), TypeRepository.EMPTY_SCHEMA); + return new EvaluationContext(Bindings.newBuilder().set(bindingName, value).build(), TypeRepository.EMPTY_SCHEMA, ImmutableMap.of()); } /** @@ -197,6 +210,31 @@ public TypeRepository getTypeRepository() { return typeRepository; } + /** + * Returns the auxiliary store bound to the given schema name, or {@code null} if no such + * store has been injected. Used by {@code RecordQueryStoreBindingPlan} to redirect execution + * to a secondary schema's record store. + * + * @param schemaName the name of the secondary schema + * @return the bound store, or {@code null} + */ + @Nullable + public FDBRecordStoreBase getAuxiliaryStore(@Nonnull final String schemaName) { + return auxiliaryStores.get(schemaName); + } + + /** + * Returns a new {@link EvaluationContext} identical to this one except that the given + * auxiliary stores are injected. Existing bindings and type repository are preserved. + * + * @param stores map from schema name to pre-opened record store + * @return new context with auxiliary stores + */ + @Nonnull + public EvaluationContext withAuxiliaryStores(@Nonnull final ImmutableMap> stores) { + return new EvaluationContext(bindings, typeRepository, stores); + } + /** * Construct a builder from this context. This allows the user to create * a new EvaluationContext that has all of the same data @@ -232,7 +270,10 @@ public static EvaluationContextBuilder newBuilder() { */ @Nonnull public EvaluationContext withBinding(@Nonnull String bindingName, @Nullable Object value) { - return childBuilder().setBinding(bindingName, value).build(typeRepository); + return new EvaluationContext( + bindings.childBuilder().set(bindingName, value).build(), + typeRepository, + auxiliaryStores); } /** @@ -248,6 +289,6 @@ public EvaluationContext withBinding(@Nonnull String bindingName, @Nullable Obje * @return a new EvaluationContext with the new binding */ public EvaluationContext withBinding(final Bindings.Internal type, @Nonnull CorrelationIdentifier alias, @Nullable Object value) { - return childBuilder().setBinding(type.bindingName(alias.getId()), value).build(typeRepository); + return withBinding(type.bindingName(alias.getId()), value); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContextBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContextBuilder.java index 721c0c4ab0..de91e1b0e7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContextBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/EvaluationContextBuilder.java @@ -21,8 +21,10 @@ package com.apple.foundationdb.record; import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; +import com.google.common.collect.ImmutableMap; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -37,12 +39,15 @@ public class EvaluationContextBuilder { @Nonnull protected final Bindings.Builder bindings; + @Nonnull + protected ImmutableMap> auxiliaryStores; /** * Create an empty builder. */ protected EvaluationContextBuilder() { this.bindings = Bindings.newBuilder(); + this.auxiliaryStores = ImmutableMap.of(); } /** @@ -54,6 +59,7 @@ protected EvaluationContextBuilder() { */ protected EvaluationContextBuilder(@Nonnull EvaluationContext original) { this.bindings = original.getBindings().childBuilder(); + this.auxiliaryStores = ImmutableMap.of(); } /** @@ -116,6 +122,6 @@ public EvaluationContextBuilder setConstant(@Nonnull CorrelationIdentifier alias */ @Nonnull public EvaluationContext build(@Nonnull final TypeRepository typeRepository) { - return EvaluationContext.forBindingsAndTypeRepository(bindings.build(), typeRepository); + return EvaluationContext.forBindingsAndTypeRepository(bindings.build(), typeRepository, auxiliaryStores); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/ExplainPlanVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/ExplainPlanVisitor.java index 823321eb9c..c4aafb8159 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/ExplainPlanVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/ExplainPlanVisitor.java @@ -73,6 +73,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTextIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan; @@ -613,6 +614,13 @@ public ExplainTokens visitTypeFilterPlan(@Nonnull final RecordQueryTypeFilterPla .iterator()); } + @Nonnull + @Override + public ExplainTokens visitStoreBindingPlan(@Nonnull final RecordQueryStoreBindingPlan storeBindingPlan) { + visit(storeBindingPlan.getChild()); + return pipe().addKeyword("STORE_BIND").addWhitespace().addToString(storeBindingPlan.getSchemaId().toString()); + } + @Nonnull @Override public ExplainTokens visitRecursiveDfsJoinPlan(@Nonnull final RecordQueryRecursiveDfsJoinPlan recursiveDfsJoinPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java index c9addce386..f73f5ccb37 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java @@ -87,6 +87,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveLevelUnionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; @@ -410,6 +411,12 @@ public Cardinalities visitRecordQueryTypeFilterPlan(@Nonnull final RecordQueryTy return fromChild(typeFilterPlan); } + @Nonnull + @Override + public Cardinalities visitRecordQueryStoreBindingPlan(@Nonnull final RecordQueryStoreBindingPlan storeBindingPlan) { + return fromChild(storeBindingPlan); + } + @Nonnull @Override public Cardinalities visitRecordQueryInUnionOnKeyExpressionPlan(@Nonnull final RecordQueryInUnionOnKeyExpressionPlan inUnionOnKeyExpressionPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java index afa214cbf3..6c51fc35f1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java @@ -84,6 +84,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySetPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTextIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan; @@ -573,6 +574,12 @@ public Derivations visitTypeFilterPlan(@Nonnull final RecordQueryTypeFilterPlan return new Derivations(resultValuesBuilder.build(), childDerivations.getLocalValues()); } + @Nonnull + @Override + public Derivations visitStoreBindingPlan(@Nonnull final RecordQueryStoreBindingPlan storeBindingPlan) { + return visit(storeBindingPlan.getChild()); + } + @Nonnull @Override public Derivations visitInUnionOnKeyExpressionPlan(@Nonnull final RecordQueryInUnionOnKeyExpressionPlan inUnionOnKeyExpressionPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java index c02563d564..fe2c641c5e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java @@ -62,6 +62,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; @@ -331,6 +332,12 @@ public Boolean visitTypeFilterPlan(@Nonnull final RecordQueryTypeFilterPlan type return distinctRecordsFromSingleChild(typeFilterPlan); } + @Nonnull + @Override + public Boolean visitStoreBindingPlan(@Nonnull final RecordQueryStoreBindingPlan storeBindingPlan) { + return visit(storeBindingPlan.getChild()); + } + @Nonnull @Override public Boolean visitInUnionOnKeyExpressionPlan(@Nonnull final RecordQueryInUnionOnKeyExpressionPlan element) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java index 2b1d68a7e4..b150dee282 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java @@ -79,6 +79,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTextIndexPlan; @@ -550,6 +551,12 @@ public Ordering visitTypeFilterPlan(@Nonnull final RecordQueryTypeFilterPlan typ return Ordering.ofOrderingSet(resultBindingMap, orderingSet, childOrdering.isDistinct()); } + @Nonnull + @Override + public Ordering visitStoreBindingPlan(@Nonnull final RecordQueryStoreBindingPlan storeBindingPlan) { + return orderingFromSingleChild(storeBindingPlan); + } + @Nonnull @Override public Ordering visitInUnionOnKeyExpressionPlan(@Nonnull final RecordQueryInUnionOnKeyExpressionPlan inUnionOnKeyExpressionPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java index 3d0c00e2ac..7511b3832e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java @@ -64,6 +64,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTextIndexPlan; @@ -353,6 +354,12 @@ public Optional> visitTypeFilterPlan(@Nonnull final RecordQueryTypeF return primaryKeyFromSingleChild(typeFilterPlan); } + @Nonnull + @Override + public Optional> visitStoreBindingPlan(@Nonnull final RecordQueryStoreBindingPlan storeBindingPlan) { + return visit(storeBindingPlan.getChild()); + } + @Nonnull @Override public Optional> visitInUnionOnKeyExpressionPlan(@Nonnull final RecordQueryInUnionOnKeyExpressionPlan inUnionOnKeyExpressionPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java index ce0511559e..c01bbf2ef6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java @@ -60,6 +60,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryRangePlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryScanPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; @@ -313,6 +314,12 @@ public Boolean visitTypeFilterPlan(@Nonnull final RecordQueryTypeFilterPlan type return storedRecordsFromSingleChild(typeFilterPlan); } + @Nonnull + @Override + public Boolean visitStoreBindingPlan(@Nonnull final RecordQueryStoreBindingPlan storeBindingPlan) { + return visit(storeBindingPlan.getChild()); + } + @Nonnull @Override public Boolean visitInUnionOnKeyExpressionPlan(@Nonnull final RecordQueryInUnionOnKeyExpressionPlan element) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java new file mode 100644 index 0000000000..41e7571158 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java @@ -0,0 +1,240 @@ +/* + * RecordQueryStoreBindingPlan.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.plans; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.planprotos.PRecordQueryPlan; +import com.apple.foundationdb.record.provider.common.StoreTimer; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.Reference; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.explain.Attribute; +import com.apple.foundationdb.record.query.plan.cascades.explain.NodeInfo; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph; +import com.apple.foundationdb.record.query.plan.cascades.expressions.AbstractRelationalExpressionWithChildren; +import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * A plan node that intercepts {@link #executePlan} and substitutes a secondary + * {@link FDBRecordStoreBase} (looked up by schema name from + * {@link EvaluationContext#getAuxiliaryStore}) for its inner subtree. + * + *

This node is emitted by {@code ImplementNestedLoopJoinRule} when the inner side of a + * {@code RecordQueryFlatMapPlan} belongs to a different schema than the outer side. It is + * transparent for all planning purposes — it does not reorder, filter, or transform records.

+ */ +@API(API.Status.EXPERIMENTAL) +public class RecordQueryStoreBindingPlan extends AbstractRelationalExpressionWithChildren implements RecordQueryPlanWithChild { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Record-Query-Store-Binding-Plan"); + + @Nonnull + private final Quantifier.Physical inner; + @Nonnull + private final SchemaIdentifier schemaId; + + public RecordQueryStoreBindingPlan(@Nonnull final Quantifier.Physical inner, + @Nonnull final SchemaIdentifier schemaId) { + this.inner = inner; + this.schemaId = schemaId; + } + + @Nonnull + public SchemaIdentifier getSchemaId() { + return schemaId; + } + + @Nonnull + public RecordQueryPlan getInnerPlan() { + return inner.getRangesOverPlan(); + } + + @Nonnull + @Override + public RecordQueryPlan getChild() { + return getInnerPlan(); + } + + @Nonnull + @Override + public RecordQueryPlanWithChild withChild(@Nonnull final Reference childRef) { + return new RecordQueryStoreBindingPlan( + Quantifier.physical(childRef, inner.getAlias()), + schemaId); + } + + @Nonnull + @Override + @SuppressWarnings("unchecked") + public RecordCursor executePlan(@Nonnull final FDBRecordStoreBase store, + @Nonnull final EvaluationContext context, + @Nullable final byte[] continuation, + @Nonnull final ExecuteProperties executeProperties) { + final String schemaName = Objects.requireNonNull(schemaId.getSchemaName(), + "RecordQueryStoreBindingPlan requires a named schema, not current()"); + final FDBRecordStoreBase secondaryStore = context.getAuxiliaryStore(schemaName); + if (secondaryStore == null) { + throw new RecordCoreException("No auxiliary store bound for schema: " + schemaName); + } + return getInnerPlan().executePlan((FDBRecordStoreBase) secondaryStore, context, continuation, executeProperties); + } + + @Override + public boolean isReverse() { + return getInnerPlan().isReverse(); + } + + @Nonnull + @Override + public List getQuantifiers() { + return ImmutableList.of(inner); + } + + @Nonnull + @Override + public Value getResultValue() { + return inner.getFlowedObjectValue(); + } + + @Nonnull + @Override + public Set computeCorrelatedToWithoutChildren() { + return ImmutableSet.of(); + } + + @Nonnull + @Override + public RecordQueryStoreBindingPlan translateCorrelations(@Nonnull final TranslationMap translationMap, + final boolean shouldSimplifyValues, + @Nonnull final List translatedQuantifiers) { + return new RecordQueryStoreBindingPlan( + Iterables.getOnlyElement(translatedQuantifiers).narrow(Quantifier.Physical.class), + schemaId); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @Override + public boolean equals(final Object other) { + return structuralEquals(other); + } + + @Override + public int hashCode() { + return structuralHashCode(); + } + + @Override + public int computeHashCodeWithoutChildren() { + return Objects.hash(schemaId); + } + + @Override + public boolean equalsWithoutChildren(@Nonnull final RelationalExpression otherExpression, + @Nonnull final AliasMap equivalencesMap) { + if (this == otherExpression) { + return true; + } + if (getClass() != otherExpression.getClass()) { + return false; + } + return schemaId.equals(((RecordQueryStoreBindingPlan) otherExpression).schemaId); + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + switch (mode.getKind()) { + case LEGACY: + return getInnerPlan().planHash(mode); + case FOR_CONTINUATION: + return PlanHashable.objectsPlanHash(mode, BASE_HASH, getInnerPlan(), + schemaId.getSchemaName()); + default: + throw new RecordCoreException("Unknown PlanHashMode: " + mode.getKind()); + } + } + + @Override + public void logPlanStructure(@Nonnull final StoreTimer timer) { + getInnerPlan().logPlanStructure(timer); + } + + @Override + public int getComplexity() { + return 1 + getInnerPlan().getComplexity(); + } + + @Nonnull + @Override + public String toString() { + return "STORE_BIND(" + schemaId + ") | " + getInnerPlan(); + } + + @Nonnull + @Override + public PlannerGraph rewritePlannerGraph(@Nonnull final List childGraphs) { + return PlannerGraph.fromNodeAndChildGraphs( + new PlannerGraph.LogicalOperatorNodeWithInfo(this, + NodeInfo.PREDICATE_FILTER_OPERATOR, + ImmutableList.of("STORE_BIND {{schema}}"), + ImmutableMap.of("schema", Attribute.gml(schemaId.toString()))), + childGraphs); + } + + @Nonnull + @Override + public Message toProto(@Nonnull final PlanSerializationContext serializationContext) { + throw new RecordCoreException("RecordQueryStoreBindingPlan proto serialization not yet implemented"); + } + + @Nonnull + @Override + public PRecordQueryPlan toRecordQueryPlanProto(@Nonnull final PlanSerializationContext serializationContext) { + throw new RecordCoreException("RecordQueryStoreBindingPlan proto serialization not yet implemented"); + } + + @Nonnull + public static RecordQueryStoreBindingPlan of(@Nonnull final RecordQueryPlan innerPlan, + @Nonnull final SchemaIdentifier schemaId) { + return new RecordQueryStoreBindingPlan(Quantifier.physical(Reference.plannedOf(innerPlan)), schemaId); + } +} From 7215685d0a13073012c7dd217f647ecdcea1d95e Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Sun, 24 May 2026 21:24:26 +0100 Subject: [PATCH 03/14] Multi-schema MetaDataPlanContext and cross-schema store binding in NLJ rule --- .../plan/cascades/MetaDataPlanContext.java | 59 ++++++++++++++++++- .../rules/ImplementNestedLoopJoinRule.java | 47 ++++++++++++++- 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index cb3b44cd3d..b30a5c9e4a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -31,6 +31,7 @@ import com.apple.foundationdb.record.query.IndexQueryabilityFilter; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; +import com.apple.foundationdb.record.util.pair.NonnullPair; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; @@ -39,6 +40,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -179,6 +181,61 @@ public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConf return new MetaDataPlanContext(plannerConfiguration, ImmutableSet.of()); } + return new MetaDataPlanContext(plannerConfiguration, + buildMatchCandidates(metaData, recordStoreState, matchCandidateRegistry, + queriedRecordTypeNames, allowedIndexesOptional, indexQueryabilityFilter)); + } + + /** + * Build a plan context pooling match candidates from the primary schema and all additional schemas. + * Used when a query references tables from more than one schema. + * + * @param plannerConfiguration planner configuration + * @param metaData primary schema metadata + * @param recordStoreState primary schema store state + * @param matchCandidateRegistry registry for match candidates + * @param rootReference root reference of the query + * @param allowedIndexesOptional optional set of allowed index names + * @param indexQueryabilityFilter filter for queryable indexes + * @param additionalSchemas map from secondary schema identifier to (metadata, store state) pair + * @return a plan context with match candidates from all schemas + */ + public static PlanContext forRootReferenceWithAdditionalSchemas( + @Nonnull final RecordQueryPlannerConfiguration plannerConfiguration, + @Nonnull final RecordMetaData metaData, + @Nonnull final RecordStoreState recordStoreState, + @Nonnull final IndexMatchCandidateRegistry matchCandidateRegistry, + @Nonnull final Reference rootReference, + @Nonnull final Optional> allowedIndexesOptional, + @Nonnull final IndexQueryabilityFilter indexQueryabilityFilter, + @Nonnull final Map> additionalSchemas) { + final var queriedRecordTypeNames = recordTypes().evaluate(rootReference); + final ImmutableSet.Builder allCandidates = ImmutableSet.builder(); + + if (!queriedRecordTypeNames.isEmpty()) { + allCandidates.addAll(buildMatchCandidates(metaData, recordStoreState, matchCandidateRegistry, + queriedRecordTypeNames, allowedIndexesOptional, indexQueryabilityFilter)); + } + + for (final NonnullPair schemaEntry : additionalSchemas.values()) { + final RecordMetaData secondaryMetaData = schemaEntry.getLeft(); + final RecordStoreState secondaryState = schemaEntry.getRight(); + final Set allTypes = secondaryMetaData.getRecordTypes().keySet(); + allCandidates.addAll(buildMatchCandidates(secondaryMetaData, secondaryState, matchCandidateRegistry, + allTypes, allowedIndexesOptional, indexQueryabilityFilter)); + } + + return new MetaDataPlanContext(plannerConfiguration, allCandidates.build()); + } + + @Nonnull + private static ImmutableSet buildMatchCandidates( + @Nonnull final RecordMetaData metaData, + @Nonnull final RecordStoreState recordStoreState, + @Nonnull final IndexMatchCandidateRegistry matchCandidateRegistry, + @Nonnull final Set queriedRecordTypeNames, + @Nonnull final Optional> allowedIndexesOptional, + @Nonnull final IndexQueryabilityFilter indexQueryabilityFilter) { final var queriedRecordTypes = queriedRecordTypeNames.stream().map(metaData::getRecordType).collect(Collectors.toList()); final var indexList = Lists.newArrayList(); @@ -218,6 +275,6 @@ public static PlanContext forRootReference(@Nonnull final RecordQueryPlannerConf .ifPresent(matchCandidatesBuilder::add); } - return new MetaDataPlanContext(plannerConfiguration, matchCandidatesBuilder.build()); + return matchCandidatesBuilder.build(); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementNestedLoopJoinRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementNestedLoopJoinRule.java index 1abad4ba6a..1bb27a8ecf 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementNestedLoopJoinRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementNestedLoopJoinRule.java @@ -35,7 +35,10 @@ import com.apple.foundationdb.record.query.plan.cascades.Reference; import com.apple.foundationdb.record.query.plan.cascades.RequestedOrdering; import com.apple.foundationdb.record.query.plan.cascades.RequestedOrderingConstraint; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; import com.apple.foundationdb.record.query.plan.cascades.debug.Debugger; +import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalTypeFilterExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.SelectExpression; import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; import com.apple.foundationdb.record.query.plan.cascades.predicates.QueryPredicate; @@ -46,6 +49,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryFirstOrDefaultPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryFlatMapPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryPredicatesFilterPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.google.common.base.Function; import com.google.common.base.Verify; @@ -184,7 +188,7 @@ public void onMatch(@Nonnull final ImplementationCascadesRuleCall call) { for (final PlanPartition innerPlanPartition : rollUpIfSatisfyOrdering(requestedOrdering, innerQuantifier, bindings.getAll(innerPlanPartitionsMatcher), Ordering.empty(), o -> pullUpOrderingFromSelectChild(o, selectExpression, innerAlias))) { final Quantifier.Physical newInnerQuantifier = planPartitionToPhysical(call, innerQuantifier, innerReference, outerInnerPredicates, innerPlanPartition); - call.yieldPlan(new RecordQueryFlatMapPlan(newOuterQuantifier, newInnerQuantifier, selectExpression.getResultValue(), innerQuantifier instanceof Quantifier.Existential)); + call.yieldPlan(new RecordQueryFlatMapPlan(newOuterQuantifier, wrapCrossSchema(call, newInnerQuantifier, innerQuantifier, innerReference), selectExpression.getResultValue(), innerQuantifier instanceof Quantifier.Existential)); } } @@ -198,7 +202,7 @@ public void onMatch(@Nonnull final ImplementationCascadesRuleCall call) { final Quantifier.Physical newOuterQuantifier = planPartitionToPhysical(call, outerQuantifier, outerReference, outerPredicates, outerPlanPartition); for (final PlanPartition innerPlanPartition : PlanPartitions.rollUpTo(bindings.getAll(innerPlanPartitionsMatcher), ImmutableSet.of())) { final Quantifier.Physical newInnerQuantifier = planPartitionToPhysical(call, innerQuantifier, innerReference, outerInnerPredicates, innerPlanPartition); - call.yieldPlan(new RecordQueryFlatMapPlan(newOuterQuantifier, newInnerQuantifier, selectExpression.getResultValue(), innerQuantifier instanceof Quantifier.Existential)); + call.yieldPlan(new RecordQueryFlatMapPlan(newOuterQuantifier, wrapCrossSchema(call, newInnerQuantifier, innerQuantifier, innerReference), selectExpression.getResultValue(), innerQuantifier instanceof Quantifier.Existential)); } } @@ -211,7 +215,7 @@ public void onMatch(@Nonnull final ImplementationCascadesRuleCall call) { for (final PlanPartition innerPlanPartition : rollUpIfSatisfyOrdering(requestedOrdering, innerQuantifier, bindings.getAll(innerPlanPartitionsMatcher), outerOrdering, o -> pullUpOrderingFromSelectChild(o, selectExpression, innerAlias))) { final Quantifier.Physical newInnerQuantifier = planPartitionToPhysical(call, innerQuantifier, innerReference, outerInnerPredicates, innerPlanPartition); - call.yieldPlan(new RecordQueryFlatMapPlan(newOuterQuantifier, newInnerQuantifier, selectExpression.getResultValue(), innerQuantifier instanceof Quantifier.Existential)); + call.yieldPlan(new RecordQueryFlatMapPlan(newOuterQuantifier, wrapCrossSchema(call, newInnerQuantifier, innerQuantifier, innerReference), selectExpression.getResultValue(), innerQuantifier instanceof Quantifier.Existential)); } } } @@ -328,4 +332,41 @@ private Quantifier.Physical planPartitionToPhysical(@Nonnull final Implementatio return Quantifier.physicalBuilder().withAlias(quantifier.getAlias()).build(ref); } + + /** + * If the inner reference belongs to a non-current schema, wraps all plans in the inner quantifier's + * reference with a {@link RecordQueryStoreBindingPlan} so that execution redirects to the correct store. + * Same-schema joins are returned unchanged. + */ + @Nonnull + private Quantifier.Physical wrapCrossSchema(@Nonnull final ImplementationCascadesRuleCall call, + @Nonnull final Quantifier.Physical newInnerQuantifier, + @Nonnull final Quantifier originalInnerQuantifier, + @Nonnull final Reference innerLogicalReference) { + final SchemaIdentifier schemaId = schemaIdFromReference(innerLogicalReference); + if (schemaId.isCurrentSchema()) { + return newInnerQuantifier; + } + final var innerRef = newInnerQuantifier.getRangesOver(); + final var wrappedPlans = innerRef.getFinalExpressions().stream() + .map(e -> (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) e) + .map(p -> (com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan) RecordQueryStoreBindingPlan.of(p, schemaId)) + .collect(ImmutableList.toImmutableList()); + final var wrappedRef = call.memoizePlansBuilder(wrappedPlans).reference(); + return Quantifier.physicalBuilder().withAlias(originalInnerQuantifier.getAlias()).build(wrappedRef); + } + + /** + * Returns the {@link SchemaIdentifier} of the first {@link LogicalTypeFilterExpression} found in the + * exploratory expressions of {@code ref}, or {@link SchemaIdentifier#current()} if none is found. + */ + @Nonnull + private static SchemaIdentifier schemaIdFromReference(@Nonnull final Reference ref) { + for (final RelationalExpression expr : ref.getExploratoryExpressions()) { + if (expr instanceof LogicalTypeFilterExpression) { + return ((LogicalTypeFilterExpression) expr).getSchemaId(); + } + } + return SchemaIdentifier.current(); + } } From d9d824f32d3b3898438267eef7edc8e523c3bccc Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Mon, 25 May 2026 12:19:50 +0100 Subject: [PATCH 04/14] Wire cross-schema table resolution and store injection end-to-end - SemanticAnalyzer: secondary schema lookup (setSecondarySchemaLookup), lazy loading and caching of secondary templates (tryLoadSecondarySchema), cross-schema tableExists/getTable, getSchemaIdFor, getLoadedSecondarySchemas, getAllTableStorageNamesForTemplate - LogicalOperator.generateTableAccess: derive SchemaIdentifier per table, use secondary template for tableNames and isStoreRowVersions; tag LogicalTypeFilterExpression with the schema identifier - PlanContext: secondary schema lookup function + additional schemas map (additionalSchemas); withAdditionalSchemas factory method; Builder adds withSecondarySchemaLookup setter, build passes new fields - EmbeddedRelationalStatement / PreparedStatement: populate secondary schema lookup from backing catalog in createPlanContext - PlanGenerator: create BaseVisitor before logical plan generation so the secondary schema lookup can be set; after generation collect loaded secondary schemas and enrich PlanContext via enrichWithSecondarySchemas - CascadesPlanner: add planGraph overload accepting additional schemas that delegates to MetaDataPlanContext.forRootReferenceWithAdditionalSchemas - QueryPlan.LogicalQueryPlan.optimize: dispatch to the multi-schema planGraph overload when additionalSchemas is non-empty - QueryPlan.PhysicalQueryPlan.executeInternal: collect required secondary schema names from RecordQueryStoreBindingPlan nodes and inject the corresponding FDBRecordStoreBase instances into EvaluationContext via withAuxiliaryStores before executing the plan --- .../query/plan/cascades/CascadesPlanner.java | 34 ++++++++ .../EmbeddedRelationalPreparedStatement.java | 8 ++ .../EmbeddedRelationalStatement.java | 8 ++ .../recordlayer/query/LogicalOperator.java | 20 ++++- .../recordlayer/query/PlanContext.java | 50 +++++++++++- .../recordlayer/query/PlanGenerator.java | 29 ++++++- .../recordlayer/query/QueryPlan.java | 64 +++++++++++++-- .../recordlayer/query/SemanticAnalyzer.java | 79 ++++++++++++++++++- 8 files changed, 272 insertions(+), 20 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java index e281ee4b75..75a00e8896 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/CascadesPlanner.java @@ -61,6 +61,7 @@ import com.apple.foundationdb.record.query.plan.cascades.matching.structure.PlannerBindings; import com.apple.foundationdb.record.query.plan.cascades.matching.structure.ReferenceMatchers; import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; +import com.apple.foundationdb.record.util.pair.NonnullPair; import com.google.common.base.Suppliers; import com.google.common.base.Verify; import com.google.common.collect.Iterables; @@ -71,6 +72,7 @@ import java.util.ArrayDeque; import java.util.Collection; import java.util.Deque; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -400,6 +402,38 @@ public QueryPlanResult planGraph(@Nonnull final Supplier referenceSup } } + public QueryPlanResult planGraph(@Nonnull final Supplier referenceSupplier, + @Nonnull final Optional> allowedIndexesOptional, + @Nonnull final IndexQueryabilityFilter indexQueryabilityFilter, + @Nonnull final EvaluationContext evaluationContext, + @Nonnull final Map> additionalSchemas) { + try { + planPartial(referenceSupplier, + rootReference -> + MetaDataPlanContext.forRootReferenceWithAdditionalSchemas(configuration, + metaData, + recordStoreState, + matchCandidateRegistry, + rootReference, + allowedIndexesOptional, + indexQueryabilityFilter, + additionalSchemas + ), + evaluationContext); + final var plan = resultOrFail(); + final var constraints = QueryPlanConstraint.collectConstraints(plan); + return new QueryPlanResult(plan, + QueryPlanInfo.newBuilder() + .put(QueryPlanInfoKeys.CONSTRAINTS, constraints) + .put(QueryPlanInfoKeys.STATS_MAPS, + PlannerEventStatsCollector.flatMapCollector(PlannerEventStatsCollector::getStatsMaps) + .orElse(null)) + .build()); + } finally { + PlannerEventListeners.dispatchOnDone(); + } + } + private RecordQueryPlan resultOrFail() { final Set finalExpressions = currentRoot.getFinalExpressions(); Verify.verify(finalExpressions.size() <= 1, "more than one variant present"); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/EmbeddedRelationalPreparedStatement.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/EmbeddedRelationalPreparedStatement.java index 2cb478c2c5..8a784796b4 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/EmbeddedRelationalPreparedStatement.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/EmbeddedRelationalPreparedStatement.java @@ -224,6 +224,14 @@ PlanContext createPlanContext(@Nonnull final FDBRecordStoreBase store, @Nonnu .withMetricsCollector(Assert.notNullUnchecked(conn.getMetricCollector())) .withPreparedParameters(PreparedParams.of(parameters, namedParameters)) .withSchemaTemplate(conn.getTransaction().getBoundSchemaTemplateMaybe().orElse(conn.getSchemaTemplate())) + .withSecondarySchemaLookup(schemaName -> { + try { + return java.util.Optional.of( + conn.getBackingCatalog().loadSchema(conn.getTransaction(), conn.getPath(), schemaName).getSchemaTemplate()); + } catch (RelationalException e) { + return java.util.Optional.empty(); + } + }) .build(); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/EmbeddedRelationalStatement.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/EmbeddedRelationalStatement.java index 742bf08f94..d9488bee46 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/EmbeddedRelationalStatement.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/EmbeddedRelationalStatement.java @@ -64,6 +64,14 @@ PlanContext createPlanContext(@Nonnull final FDBRecordStoreBase store, @Nonnu .fromDatabase(conn.getRecordLayerDatabase()) .withMetricsCollector(Assert.notNullUnchecked(conn.getMetricCollector())) .withSchemaTemplate(conn.getTransaction().getBoundSchemaTemplateMaybe().orElse(conn.getSchemaTemplate())) + .withSecondarySchemaLookup(schemaName -> { + try { + return java.util.Optional.of( + conn.getBackingCatalog().loadSchema(conn.getTransaction(), conn.getPath(), schemaName).getSchemaTemplate()); + } catch (RelationalException e) { + return java.util.Optional.empty(); + } + }) .build(); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java index b6476959bd..d73bf12d1f 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java @@ -30,6 +30,7 @@ import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.Reference; import com.apple.foundationdb.record.query.plan.cascades.RequestedOrdering; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; import com.apple.foundationdb.record.query.plan.cascades.expressions.ExplodeExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.FullUnorderedScanExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.GroupByExpression; @@ -53,6 +54,7 @@ import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.api.metadata.Table; +import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.util.Assert; @@ -263,7 +265,17 @@ public static LogicalOperator newOperatorWithPreservedExpressionNames(@Nonnull O public static LogicalOperator generateTableAccess(@Nonnull Identifier tableId, @Nonnull Set indexAccessHints, @Nonnull SemanticAnalyzer semanticAnalyzer) { - final Set tableNames = semanticAnalyzer.getAllTableStorageNames(); + final SchemaIdentifier schemaId = semanticAnalyzer.getSchemaIdFor(tableId); + final SchemaTemplate effectiveTemplate; + final Set tableNames; + if (schemaId.isCurrentSchema()) { + effectiveTemplate = semanticAnalyzer.getMetadataCatalog(); + tableNames = semanticAnalyzer.getAllTableStorageNames(); + } else { + final var secondaryTemplate = semanticAnalyzer.getLoadedSecondarySchemas().get(schemaId.getSchemaName()); + effectiveTemplate = secondaryTemplate; + tableNames = semanticAnalyzer.getAllTableStorageNamesForTemplate(secondaryTemplate); + } semanticAnalyzer.validateIndexes(tableId, indexAccessHints); final var scanExpression = Quantifier.forEach(Reference.initialOf( new FullUnorderedScanExpression(tableNames, @@ -271,7 +283,7 @@ public static LogicalOperator generateTableAccess(@Nonnull Identifier tableId, new AccessHints(indexAccessHints.toArray(new AccessHint[0]))))); final var table = semanticAnalyzer.getTable(tableId); Type.Record type = Assert.castUnchecked(table, RecordLayerTable.class).getType(); - if (semanticAnalyzer.getMetadataCatalog().isStoreRowVersions()) { + if (effectiveTemplate.isStoreRowVersions()) { // Ideally, the RecordLayerTable would have the full type with all the pseudo-fields, // but we need star expansion to skip over these fields. That would be made easier // if we fully supported invisible columns (see: https://github.com/FoundationDB/fdb-record-layer/pull/3787) @@ -279,7 +291,7 @@ public static LogicalOperator generateTableAccess(@Nonnull Identifier tableId, } final String storageName = type.getStorageName(); Assert.thatUnchecked(storageName != null, "storage name for table access must not be null"); - final var typeFilterExpression = LogicalTypeFilterExpression.of(ImmutableSet.of(storageName), scanExpression, type); + final var typeFilterExpression = LogicalTypeFilterExpression.of(ImmutableSet.of(storageName), scanExpression, type, schemaId); final var resultingQuantifier = Quantifier.forEach(Reference.initialOf(typeFilterExpression)); final ImmutableList.Builder attributesBuilder = ImmutableList.builder(); int colCount = 0; @@ -292,7 +304,7 @@ public static LogicalOperator generateTableAccess(@Nonnull Identifier tableId, attributesBuilder.add(new Expression(Optional.of(attributeName), attributeType, attributeExpression)); colCount++; } - if (semanticAnalyzer.getMetadataCatalog().isStoreRowVersions() + if (effectiveTemplate.isStoreRowVersions() && table.getColumns().stream().noneMatch(column -> column.getName().equals(PseudoField.ROW_VERSION.getFieldName()))) { final var pseudoFieldValue = FieldValue.ofFieldName(resultingQuantifier.getFlowedObjectValue(), PseudoField.ROW_VERSION.getFieldName()); final Expression pseudoExpression = new Expression(Optional.of(Identifier.of(PseudoField.ROW_VERSION.getFieldName())), DataType.VersionType.nullable(), pseudoFieldValue); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanContext.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanContext.java index 0e81b916dc..0934625ea6 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanContext.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanContext.java @@ -25,6 +25,7 @@ import com.apple.foundationdb.record.RecordStoreState; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.ddl.DdlQueryFactory; import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; @@ -33,12 +34,16 @@ import com.apple.foundationdb.relational.api.metrics.MetricCollector; import com.apple.foundationdb.relational.recordlayer.AbstractDatabase; import com.apple.foundationdb.relational.util.Assert; +import com.apple.foundationdb.record.util.pair.NonnullPair; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import javax.annotation.Nonnull; import java.net.URI; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; @API(API.Status.EXPERIMENTAL) @@ -66,6 +71,12 @@ public final class PlanContext { private final boolean isCaseSensitive; + @Nonnull + private final Function> secondarySchemaLookup; + + @Nonnull + private final ImmutableMap> additionalSchemas; + /** * Creates a new instance of {@link PlanContext} needed for generating plans. * @@ -90,7 +101,9 @@ private PlanContext(@Nonnull RecordMetaData metaData, @Nonnull URI dbUri, @Nonnull PreparedParams preparedStatementParameters, final int userVersion, - boolean isCaseSensitive) { + boolean isCaseSensitive, + @Nonnull Function> secondarySchemaLookup, + @Nonnull ImmutableMap> additionalSchemas) { this.metaData = metaData; this.metricCollector = metricCollector; this.schemaTemplate = schemaTemplate; @@ -101,6 +114,8 @@ private PlanContext(@Nonnull RecordMetaData metaData, this.preparedStatementParameters = preparedStatementParameters; this.userVersion = userVersion; this.isCaseSensitive = isCaseSensitive; + this.secondarySchemaLookup = secondarySchemaLookup; + this.additionalSchemas = additionalSchemas; } @Nonnull @@ -157,6 +172,24 @@ public int getUserVersion() { return userVersion; } + @Nonnull + public Function> getSecondarySchemaLookup() { + return secondarySchemaLookup; + } + + @Nonnull + public ImmutableMap> getAdditionalSchemas() { + return additionalSchemas; + } + + @Nonnull + public PlanContext withAdditionalSchemas( + @Nonnull final ImmutableMap> newAdditionalSchemas) { + return new PlanContext(metaData, metricCollector, schemaTemplate, plannerConfiguration, + metadataOperationsFactory, ddlQueryFactory, dbUri, preparedStatementParameters, + userVersion, isCaseSensitive, secondarySchemaLookup, newAdditionalSchemas); + } + @Nonnull public static Builder builder() { return new Builder(); @@ -184,6 +217,9 @@ public static final class Builder { private boolean isCaseSensitive; + @Nonnull + private Function> secondarySchemaLookup = s -> Optional.empty(); + private Builder() { } @@ -264,6 +300,12 @@ private static Optional> getReadableIndexes(@Nonnull RecordMetaData } } + @Nonnull + public Builder withSecondarySchemaLookup(@Nonnull final Function> lookup) { + this.secondarySchemaLookup = lookup; + return this; + } + @Nonnull public Builder fromRecordStore(@Nonnull FDBRecordStoreBase recordStore, @Nonnull final Options options) { final var plannerConfig = recordStore.getRecordStoreState().allIndexesReadable() ? @@ -298,7 +340,8 @@ private void verify() throws RelationalException { public PlanContext build() throws RelationalException { verify(); return new PlanContext(metaData, metricCollector, schemaTemplate, plannerConfiguration, metadataOperationsFactory, - ddlQueryFactory, dbUri, preparedStatementParameters, userVersion, isCaseSensitive); + ddlQueryFactory, dbUri, preparedStatementParameters, userVersion, isCaseSensitive, + secondarySchemaLookup, ImmutableMap.of()); } @Nonnull @@ -318,7 +361,8 @@ public static Builder unapply(@Nonnull PlanContext planContext) { .withPlannerConfiguration(planContext.plannerConfiguration) .withUserVersion(planContext.userVersion) .withPreparedParameters(planContext.preparedStatementParameters) - .isCaseSensitive(planContext.isCaseSensitive); + .isCaseSensitive(planContext.isCaseSensitive) + .withSecondarySchemaLookup(planContext.secondarySchemaLookup); } } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java index e8019cecc3..988ea4ea5a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java @@ -30,6 +30,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.provider.foundationdb.IndexMatchCandidateRegistry; import com.apple.foundationdb.record.query.plan.QueryPlanConstraint; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; import com.apple.foundationdb.record.query.plan.cascades.SemanticException; import com.apple.foundationdb.record.query.plan.cascades.StableSelectorCostModel; @@ -49,6 +50,7 @@ import com.apple.foundationdb.relational.continuation.CompiledStatement; import com.apple.foundationdb.relational.continuation.TypedQueryArgument; import com.apple.foundationdb.relational.recordlayer.ContinuationImpl; +import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.query.cache.PhysicalPlanEquivalence; @@ -59,6 +61,7 @@ import com.apple.foundationdb.relational.util.RelationalLoggingUtil; import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.protobuf.InvalidProtocolBufferException; import org.apache.logging.log4j.LogManager; @@ -247,11 +250,13 @@ private Plan generatePhysicalPlanForCompilableStatement(@Nonnull AstNormalize planGenerationContext.setForExplain(ast.getQueryExecutionContext().isForExplain()); final var metadata = Assert.castUnchecked(planContext.getSchemaTemplate(), RecordLayerSchemaTemplate.class); try (var ignored = new PlannerEventStatsCollector.DefaultStatsCollectorController()) { + final var visitor = new BaseVisitor(planGenerationContext, metadata, planContext.getDdlQueryFactory(), + planContext.getConstantActionFactory(), planContext.getDbUri(), caseSensitive); + visitor.getSemanticAnalyzer().setSecondarySchemaLookup(planContext.getSecondarySchemaLookup()); final var maybePlan = planContext.getMetricsCollector().clock(RelationalMetric.RelationalEvent.GENERATE_LOGICAL_PLAN, () -> - new BaseVisitor(planGenerationContext, metadata, planContext.getDdlQueryFactory(), - planContext.getConstantActionFactory(), planContext.getDbUri(), caseSensitive) - .generateLogicalPlan(ast.getParseTree())); - return maybePlan.optimize(planner, planContext, currentPlanHashMode); + visitor.generateLogicalPlan(ast.getParseTree())); + final var enrichedPlanContext = enrichWithSecondarySchemas(planContext, visitor.getSemanticAnalyzer().getLoadedSecondarySchemas()); + return maybePlan.optimize(planner, enrichedPlanContext, currentPlanHashMode); } catch (ProtoUtils.InvalidNameException ine) { throw new RelationalException(ine.getMessage(), ErrorCode.INVALID_NAME, ine).toUncheckedWrappedException(); } catch (MetaDataException mde) { @@ -264,6 +269,22 @@ private Plan generatePhysicalPlanForCompilableStatement(@Nonnull AstNormalize } } + @Nonnull + private static PlanContext enrichWithSecondarySchemas(@Nonnull final PlanContext planContext, + @Nonnull final java.util.Map loadedSecondarySchemas) { + if (loadedSecondarySchemas.isEmpty()) { + return planContext; + } + final ImmutableMap.Builder> builder = ImmutableMap.builder(); + for (final var entry : loadedSecondarySchemas.entrySet()) { + final var schemaId = SchemaIdentifier.of(entry.getKey()); + final var rlTemplate = Assert.castUnchecked(entry.getValue(), RecordLayerSchemaTemplate.class); + final var recMeta = rlTemplate.toRecordMetadata(); + builder.put(schemaId, NonnullPair.of(recMeta, new RecordStoreState(null, null))); + } + return planContext.withAdditionalSchemas(builder.build()); + } + @Nonnull private Plan generatePhysicalPlanForExecuteContinuation(@Nonnull AstNormalizer.NormalizationResult ast, @Nonnull Set validPlanHashModes, diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java index 2f2903a1f2..95530439bb 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java @@ -35,6 +35,8 @@ import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; import com.apple.foundationdb.record.query.plan.cascades.PlannerPhase; import com.apple.foundationdb.record.query.plan.cascades.Reference; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.cascades.events.ExecutingTaskPlannerEvent; import com.apple.foundationdb.record.query.plan.cascades.events.InsertIntoMemoPlannerEvent; import com.apple.foundationdb.record.query.plan.cascades.events.PlannerEvent.Location; @@ -83,6 +85,8 @@ import com.google.common.base.Suppliers; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; @@ -257,7 +261,8 @@ public RelationalResultSet executeInternal(@Nonnull final ExecutionContext execu final String schemaName = conn.getSchema(); try (RecordLayerSchema recordLayerSchema = conn.getRecordLayerDatabase().loadSchema(schemaName)) { final var evaluationContext = queryExecutionContext.getEvaluationContext(); - final var typedEvaluationContext = EvaluationContext.forBindingsAndTypeRepository(evaluationContext.getBindings(), typeRepository); + var typedEvaluationContext = EvaluationContext.forBindingsAndTypeRepository(evaluationContext.getBindings(), typeRepository); + typedEvaluationContext = injectSecondaryStores(typedEvaluationContext, recordQueryPlan, conn); final ContinuationImpl parsedContinuation; try { parsedContinuation = ContinuationImpl.parseContinuation(queryExecutionContext.getContinuation()); @@ -276,6 +281,43 @@ public RelationalResultSet executeInternal(@Nonnull final ExecutionContext execu } } + @Nonnull + private static EvaluationContext injectSecondaryStores(@Nonnull final EvaluationContext base, + @Nonnull final RecordQueryPlan plan, + @Nonnull final EmbeddedRelationalConnection conn) throws RelationalException { + final var schemaNames = collectSecondarySchemaNames(plan); + if (schemaNames.isEmpty()) { + return base; + } + final ImmutableMap.Builder> storeMapBuilder = ImmutableMap.builder(); + for (final var secondarySchemaName : schemaNames) { + try (RecordLayerSchema secondarySchema = conn.getRecordLayerDatabase().loadSchema(secondarySchemaName)) { + storeMapBuilder.put(secondarySchemaName, secondarySchema.loadStore().unwrap(FDBRecordStoreBase.class)); + } + } + return base.withAuxiliaryStores(storeMapBuilder.build()); + } + + @Nonnull + private static Set collectSecondarySchemaNames(@Nonnull final RecordQueryPlan plan) { + final ImmutableSet.Builder builder = ImmutableSet.builder(); + collectSecondarySchemaNames(plan, builder); + return builder.build(); + } + + private static void collectSecondarySchemaNames(@Nonnull final RecordQueryPlan plan, + @Nonnull final ImmutableSet.Builder builder) { + if (plan instanceof RecordQueryStoreBindingPlan) { + final var schemaId = ((RecordQueryStoreBindingPlan) plan).getSchemaId(); + if (!schemaId.isCurrentSchema() && schemaId.getSchemaName() != null) { + builder.add(schemaId.getSchemaName()); + } + } + for (final RecordQueryPlan child : plan.getChildren()) { + collectSecondarySchemaNames(child, builder); + } + } + protected void validatePlanAgainstEnvironment(@Nonnull final ContinuationImpl parsedContinuation, @Nonnull final FDBRecordStoreBase fdbRecordStore, @Nonnull final ExecutionContext executionContext, @@ -643,11 +685,21 @@ public PhysicalQueryPlan optimize(@Nonnull CascadesPlanner planner, @Nonnull Pla final var typedEvaluationContext = EvaluationContext.forBindingsAndTypeRepository(evaluationContext.getBindings(), builder.build()); final QueryPlanResult planResult; try { - planResult = planner.planGraph(() -> - Reference.initialOf(relationalExpression), - planContext.getReadableIndexes().map(s -> s), - IndexQueryabilityFilter.TRUE, - typedEvaluationContext); + final var additionalSchemas = planContext.getAdditionalSchemas(); + if (additionalSchemas.isEmpty()) { + planResult = planner.planGraph(() -> + Reference.initialOf(relationalExpression), + planContext.getReadableIndexes().map(s -> s), + IndexQueryabilityFilter.TRUE, + typedEvaluationContext); + } else { + planResult = planner.planGraph(() -> + Reference.initialOf(relationalExpression), + planContext.getReadableIndexes().map(s -> s), + IndexQueryabilityFilter.TRUE, + typedEvaluationContext, + additionalSchemas); + } } catch (RecordCoreException ex) { throw ExceptionUtil.toRelationalException(ex); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 9c38167100..2962884f77 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.query.plan.cascades.AccessHint; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; import com.apple.foundationdb.record.query.plan.cascades.BuiltInTableFunction; import com.apple.foundationdb.record.query.plan.cascades.CatalogedFunction; @@ -81,6 +82,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URI; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -121,6 +124,12 @@ public class SemanticAnalyzer { private final boolean isCaseSensitive; + @Nonnull + private java.util.function.Function> secondarySchemaLookup; + + @Nonnull + private final LinkedHashMap loadedSecondarySchemas; + public SemanticAnalyzer(@Nonnull SchemaTemplate metadataCatalog, @Nonnull SqlFunctionCatalog functionCatalog, @Nonnull MutablePlanGenerationContext mutablePlanGenerationContext, @@ -129,6 +138,13 @@ public SemanticAnalyzer(@Nonnull SchemaTemplate metadataCatalog, this.functionCatalog = functionCatalog; this.mutablePlanGenerationContext = mutablePlanGenerationContext; this.isCaseSensitive = isCaseSensitive; + this.secondarySchemaLookup = s -> Optional.empty(); + this.loadedSecondarySchemas = new LinkedHashMap<>(); + } + + public void setSecondarySchemaLookup( + @Nonnull final java.util.function.Function> lookup) { + this.secondarySchemaLookup = lookup; } /** @@ -194,7 +210,15 @@ public boolean tableExists(@Nonnull final Identifier tableIdentifier) { if (tableIdentifier.isQualified()) { final var qualifier = tableIdentifier.getQualifier().get(0); if (!metadataCatalog.getName().equals(qualifier)) { - return false; + return tryLoadSecondarySchema(qualifier) + .map(template -> { + try { + return template.findTableByName(tableIdentifier.getName()).isPresent(); + } catch (RelationalException e) { + throw e.toUncheckedWrappedException(); + } + }) + .orElse(false); } } @@ -206,6 +230,17 @@ public boolean tableExists(@Nonnull final Identifier tableIdentifier) { } } + @Nonnull + private Optional tryLoadSecondarySchema(@Nonnull final String schemaName) { + final var cached = loadedSecondarySchemas.get(schemaName); + if (cached != null) { + return Optional.of(cached); + } + final var result = secondarySchemaLookup.apply(schemaName); + result.ifPresent(template -> loadedSecondarySchemas.put(schemaName, template)); + return result; + } + public boolean viewExists(@Nonnull final Identifier viewIdentifier) { if (viewIdentifier.getQualifier().size() > 1) { return false; @@ -238,13 +273,22 @@ public SchemaTemplate getMetadataCatalog() { @Nonnull public Table getTable(@Nonnull Identifier tableIdentifier) { Assert.thatUnchecked(tableIdentifier.getQualifier().size() <= 1, ErrorCode.INTERNAL_ERROR, () -> String.format(Locale.ROOT, "Unknown table %s", tableIdentifier)); + final SchemaTemplate targetCatalog; if (tableIdentifier.isQualified()) { final var qualifier = tableIdentifier.getQualifier().get(0); - Assert.thatUnchecked(metadataCatalog.getName().equals(qualifier), ErrorCode.UNDEFINED_DATABASE, () -> String.format(Locale.ROOT, "Unknown schema template %s", qualifier)); + if (metadataCatalog.getName().equals(qualifier)) { + targetCatalog = metadataCatalog; + } else { + final var secondaryTemplate = tryLoadSecondarySchema(qualifier); + Assert.thatUnchecked(secondaryTemplate.isPresent(), ErrorCode.UNDEFINED_DATABASE, () -> String.format(Locale.ROOT, "Unknown schema template %s", qualifier)); + targetCatalog = secondaryTemplate.get(); + } + } else { + targetCatalog = metadataCatalog; } final var tableName = tableIdentifier.getName(); try { - final var tableMaybe = metadataCatalog.findTableByName(tableName); + final var tableMaybe = targetCatalog.findTableByName(tableName); Assert.thatUnchecked(tableMaybe.isPresent(), ErrorCode.UNDEFINED_TABLE, () -> String.format(Locale.ROOT, "Unknown table %s", tableName)); return tableMaybe.get(); } catch (RelationalException e) { @@ -296,6 +340,35 @@ public Set getAllTableStorageNames() { } } + @Nonnull + public Set getAllTableStorageNamesForTemplate(@Nonnull final SchemaTemplate template) { + try { + return template.getTables().stream() + .map(table -> Assert.castUnchecked(table, RecordLayerTable.class)) + .map(table -> Assert.notNullUnchecked(table.getType().getStorageName())) + .collect(ImmutableSet.toImmutableSet()); + } catch (RelationalException e) { + throw e.toUncheckedWrappedException(); + } + } + + @Nonnull + public SchemaIdentifier getSchemaIdFor(@Nonnull final Identifier tableId) { + if (!tableId.isQualified()) { + return SchemaIdentifier.current(); + } + final var qualifier = tableId.getQualifier().get(0); + if (metadataCatalog.getName().equals(qualifier)) { + return SchemaIdentifier.current(); + } + return SchemaIdentifier.of(qualifier); + } + + @Nonnull + public Map getLoadedSecondarySchemas() { + return Collections.unmodifiableMap(loadedSecondarySchemas); + } + @Nonnull public Expression resolveCorrelatedIdentifier(@Nonnull Identifier identifier, @Nonnull LogicalOperators operators) { From 4a452453a543c42f80a04b2fbe4e1393c2c3a216 Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Mon, 25 May 2026 21:30:31 +0100 Subject: [PATCH 05/14] Fix WHERE predicate drop in cross-schema correlated scan and add join tests When a PredicateWithValueAndRanges has a single range with multiple equality comparisons (e.g. JOIN + WHERE on the same field), asComparisonRange() silently drops all but the first comparison as residuals. The compensation lambda previously returned noCompensationNeeded() whenever the Placeholder alias was bound, causing the constant equality (the WHERE filter) to be silently dropped from the generated plan. Fix: replay the merge sequence inside the compensation lambda to detect residual comparisons. When residuals exist, build a new single-range PVWR from them and apply it as a filter via computeCompensationFunctionForLeaf, ensuring the plan emits SCAN([IS ITEMS, EQUALS q6.ITEM_ID]) | FILTER(output.ID = 2) rather than an unfiltered correlated scan. Also fix schemaIdFromReference to recurse into quantifier child groups so that cross-schema plans nested inside a partitioned SelectExpression are wrapped with RecordQueryStoreBindingPlan. Covers AbstractDataAccessRule schema detection for SelectExpression alternatives created by predicate push-down rules. Add CrossSchemaJoinTest covering: plain cross-schema join, join with WHERE filter on the primary-schema table, and standalone select from a secondary schema. --- .../plan/cascades/MetaDataPlanContext.java | 39 ++++-- .../query/plan/cascades/PlanContext.java | 12 ++ .../PredicateWithValueAndRanges.java | 24 +++- .../rules/AbstractDataAccessRule.java | 65 ++++++++- .../rules/ImplementNestedLoopJoinRule.java | 9 +- .../rules/ImplementTypeFilterRule.java | 20 ++- .../query/CrossSchemaJoinTest.java | 128 ++++++++++++++++++ .../addExplain/add-explain.yamsql | 1 - .../add-result-metadata.yamsql | 1 - 9 files changed, 283 insertions(+), 16 deletions(-) create mode 100644 fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaJoinTest.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java index b30a5c9e4a..060f2090ed 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MetaDataPlanContext.java @@ -61,10 +61,20 @@ public class MetaDataPlanContext implements PlanContext { @Nonnull private final Set matchCandidates; + @Nonnull + private final Map matchCandidateSchemaMap; + private MetaDataPlanContext(@Nonnull final RecordQueryPlannerConfiguration plannerConfiguration, @Nonnull final Set matchCandidates) { + this(plannerConfiguration, matchCandidates, Map.of()); + } + + private MetaDataPlanContext(@Nonnull final RecordQueryPlannerConfiguration plannerConfiguration, + @Nonnull final Set matchCandidates, + @Nonnull final Map matchCandidateSchemaMap) { this.plannerConfiguration = plannerConfiguration; this.matchCandidates = ImmutableSet.copyOf(matchCandidates); + this.matchCandidateSchemaMap = Map.copyOf(matchCandidateSchemaMap); } @Nonnull @@ -73,6 +83,12 @@ public RecordQueryPlannerConfiguration getPlannerConfiguration() { return plannerConfiguration; } + @Nonnull + @Override + public SchemaIdentifier getSchemaIdForMatchCandidate(@Nonnull final MatchCandidate candidate) { + return matchCandidateSchemaMap.getOrDefault(candidate, SchemaIdentifier.current()); + } + @Nullable private static KeyExpression commonPrimaryKey(@Nonnull Iterable recordTypes) { KeyExpression common = null; @@ -211,21 +227,28 @@ public static PlanContext forRootReferenceWithAdditionalSchemas( @Nonnull final Map> additionalSchemas) { final var queriedRecordTypeNames = recordTypes().evaluate(rootReference); final ImmutableSet.Builder allCandidates = ImmutableSet.builder(); + final Map schemaMap = new java.util.LinkedHashMap<>(); - if (!queriedRecordTypeNames.isEmpty()) { + final var primaryTypeNames = queriedRecordTypeNames.stream() + .filter(name -> metaData.getRecordTypes().containsKey(name)) + .collect(ImmutableSet.toImmutableSet()); + if (!primaryTypeNames.isEmpty()) { allCandidates.addAll(buildMatchCandidates(metaData, recordStoreState, matchCandidateRegistry, - queriedRecordTypeNames, allowedIndexesOptional, indexQueryabilityFilter)); + primaryTypeNames, allowedIndexesOptional, indexQueryabilityFilter)); } - for (final NonnullPair schemaEntry : additionalSchemas.values()) { - final RecordMetaData secondaryMetaData = schemaEntry.getLeft(); - final RecordStoreState secondaryState = schemaEntry.getRight(); + for (final Map.Entry> entry : additionalSchemas.entrySet()) { + final SchemaIdentifier schemaId = entry.getKey(); + final RecordMetaData secondaryMetaData = entry.getValue().getLeft(); + final RecordStoreState secondaryState = entry.getValue().getRight(); final Set allTypes = secondaryMetaData.getRecordTypes().keySet(); - allCandidates.addAll(buildMatchCandidates(secondaryMetaData, secondaryState, matchCandidateRegistry, - allTypes, allowedIndexesOptional, indexQueryabilityFilter)); + final ImmutableSet secondaryCandidates = buildMatchCandidates(secondaryMetaData, secondaryState, matchCandidateRegistry, + allTypes, allowedIndexesOptional, indexQueryabilityFilter); + allCandidates.addAll(secondaryCandidates); + secondaryCandidates.forEach(c -> schemaMap.put(c, schemaId)); } - return new MetaDataPlanContext(plannerConfiguration, allCandidates.build()); + return new MetaDataPlanContext(plannerConfiguration, allCandidates.build(), schemaMap); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlanContext.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlanContext.java index bd85ed5e47..1543af9e32 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlanContext.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlanContext.java @@ -53,6 +53,18 @@ public Set getMatchCandidates() { @Nonnull Set getMatchCandidates(); + /** + * Returns the {@link SchemaIdentifier} for a match candidate, or {@link SchemaIdentifier#current()} if the + * candidate belongs to the primary (current) schema. + * + * @param candidate a match candidate + * @return the schema identifier for the candidate + */ + @Nonnull + default SchemaIdentifier getSchemaIdForMatchCandidate(@Nonnull MatchCandidate candidate) { + return SchemaIdentifier.current(); + } + @Nonnull static PlanContext emptyContext() { return EMPTY_CONTEXT; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/PredicateWithValueAndRanges.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/PredicateWithValueAndRanges.java index d374bc5e11..f6f5d02145 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/PredicateWithValueAndRanges.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/PredicateWithValueAndRanges.java @@ -44,6 +44,7 @@ import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence.Precedence; import com.google.auto.service.AutoService; +import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange; import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -345,7 +346,28 @@ public Optional impliesCandidatePredicateMaybe(@Nonnull final PredicateMapping.regularMappingBuilder(originalQueryPredicate, this, candidatePredicate) .setPredicateCompensation((ignore, boundParameterPrefixMap, pullUp) -> { if (boundParameterPrefixMap.containsKey(alias)) { - return PredicateCompensationFunction.noCompensationNeeded(); + // The scan key used one comparison from this range; any remaining + // comparisons (residuals dropped by asComparisonRange()) must still + // be applied as a filter. + final var singleRange = Iterables.getOnlyElement(compensatedQueryPredicate.getRanges()); + var sargRange = ComparisonRange.EMPTY; + final ImmutableList.Builder residualBuilder = ImmutableList.builder(); + for (final var comparison : singleRange.getComparisons()) { + final var mergeResult = sargRange.merge(comparison); + sargRange = mergeResult.getComparisonRange(); + residualBuilder.addAll(mergeResult.getResidualComparisons()); + } + final var residuals = residualBuilder.build(); + if (residuals.isEmpty()) { + return PredicateCompensationFunction.noCompensationNeeded(); + } + final var rangeBuilder = RangeConstraints.newBuilder(); + residuals.forEach(rangeBuilder::addComparisonMaybe); + return rangeBuilder.build() + .map(residualRange -> PredicateWithValueAndRanges.sargable( + compensatedQueryPredicate.getValue(), residualRange) + .computeCompensationFunctionForLeaf(pullUp)) + .orElseGet(() -> computeCompensationFunctionForLeaf(pullUp)); } return computeCompensationFunctionForLeaf(pullUp); }) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java index dab7fa94f4..2041af229d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java @@ -49,9 +49,12 @@ import com.apple.foundationdb.record.query.plan.cascades.RequestedOrderingConstraint; import com.apple.foundationdb.record.query.plan.cascades.ValueIndexScanMatchCandidate; import com.apple.foundationdb.record.query.plan.cascades.events.PlannerEvent.Location; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalDistinctExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalIntersectionExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalTypeFilterExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; import com.apple.foundationdb.record.query.plan.cascades.properties.CardinalitiesProperty.Cardinality; @@ -144,6 +147,40 @@ public void onMatch(@Nonnull final CascadesRuleCall call) { final var expression = bindings.get(getExpressionMatcher()); + final SchemaIdentifier secondarySchemaId; + if (expression instanceof LogicalTypeFilterExpression) { + final SchemaIdentifier sid = ((LogicalTypeFilterExpression) expression).getSchemaId(); + secondarySchemaId = sid.isCurrentSchema() ? null : sid; + } else { + // The expression is not an LTFE. Determine the schema via three fallback sources: + // (a) sibling exploratory expressions in the current group (non-correlated SelectExpression + // added to the same group as the LTFE by exploration rules), + // (b) exploratory expressions in each quantifier's directly-referenced child group + // (correlated SelectExpression placed in a new child group by predicate-pushdown whose + // quantifier still ranges over the LTFE's group), and + // (c) the PlanContext's match candidate schema map (correlated SelectExpression in a deeper + // child group, where the quantifier ranges over the scan group rather than the LTFE group). + SchemaIdentifier sid = secondarySchemaIdFromGroup(call.getRoot()); + if (sid == null) { + sid = expression.getQuantifiers().stream() + .map(Quantifier::getRangesOver) + .map(AbstractDataAccessRule::secondarySchemaIdFromGroup) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + } + if (sid == null) { + final var planContext = call.getContext(); + sid = completeMatches.stream() + .map(PartialMatch::getMatchCandidate) + .map(planContext::getSchemaIdForMatchCandidate) + .filter(s -> !s.isCurrentSchema()) + .findFirst() + .orElse(null); + } + secondarySchemaId = sid; + } + // // return if there is no pre-determined interesting ordering // @@ -220,7 +257,20 @@ public void onMatch(@Nonnull final CascadesRuleCall call) { dataAccessForMatchPartition(call, requestedOrderings, matchPartition); - call.yieldMixedUnknownExpressions(dataAccessExpressions); + if (secondarySchemaId != null) { + final LinkedIdentitySet wrapped = new LinkedIdentitySet<>(); + for (final RelationalExpression expr : dataAccessExpressions) { + if (expr instanceof RecordQueryPlan) { + final Reference innerRef = call.memoizePlan((RecordQueryPlan) expr); + wrapped.add(new RecordQueryStoreBindingPlan(Quantifier.physical(innerRef), secondarySchemaId)); + } else { + wrapped.add(expr); + } + } + call.yieldMixedUnknownExpressions(wrapped); + } else { + call.yieldMixedUnknownExpressions(dataAccessExpressions); + } } } } @@ -1348,6 +1398,19 @@ public static Vectored of(@Nonnull final T element, final int position) { } } + @Nullable + private static SchemaIdentifier secondarySchemaIdFromGroup(@Nonnull final Reference ref) { + for (final RelationalExpression expr : ref.getExploratoryExpressions()) { + if (expr instanceof LogicalTypeFilterExpression) { + final SchemaIdentifier sid = ((LogicalTypeFilterExpression) expr).getSchemaId(); + if (!sid.isCurrentSchema()) { + return sid; + } + } + } + return null; + } + protected static class IntersectionResult { @Nullable private final Ordering.Intersection commonIntersectionOrdering; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementNestedLoopJoinRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementNestedLoopJoinRule.java index 1bb27a8ecf..3ac9ae87b3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementNestedLoopJoinRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementNestedLoopJoinRule.java @@ -358,7 +358,8 @@ private Quantifier.Physical wrapCrossSchema(@Nonnull final ImplementationCascade /** * Returns the {@link SchemaIdentifier} of the first {@link LogicalTypeFilterExpression} found in the - * exploratory expressions of {@code ref}, or {@link SchemaIdentifier#current()} if none is found. + * exploratory expressions of {@code ref} (or recursively in their quantifiers' referenced groups), + * or {@link SchemaIdentifier#current()} if none is found. */ @Nonnull private static SchemaIdentifier schemaIdFromReference(@Nonnull final Reference ref) { @@ -366,6 +367,12 @@ private static SchemaIdentifier schemaIdFromReference(@Nonnull final Reference r if (expr instanceof LogicalTypeFilterExpression) { return ((LogicalTypeFilterExpression) expr).getSchemaId(); } + for (final Quantifier q : expr.getQuantifiers()) { + final SchemaIdentifier sid = schemaIdFromReference(q.getRangesOver()); + if (!sid.isCurrentSchema()) { + return sid; + } + } } return SchemaIdentifier.current(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTypeFilterRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTypeFilterRule.java index 32e9382db1..08a01b245f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTypeFilterRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTypeFilterRule.java @@ -27,12 +27,14 @@ import com.apple.foundationdb.record.query.plan.cascades.PlanPartition; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.Reference; +import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalTypeFilterExpression; import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; import com.apple.foundationdb.record.query.plan.cascades.properties.RecordTypesProperty.RecordTypesVisitor; import com.apple.foundationdb.record.query.plan.cascades.properties.StoredRecordProperty; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.plans.RecordQueryPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Sets; @@ -93,17 +95,29 @@ public void onMatch(@Nonnull final ImplementationCascadesRuleCall call) { } final var unsatisfiedMap = unsatisfiedMapBuilder.build(); + final SchemaIdentifier schemaId = logicalTypeFilterExpression.getSchemaId(); if (!noTypeFilterNeeded.isEmpty()) { - call.yieldPlans(noTypeFilterNeeded); + if (schemaId.isCurrentSchema()) { + call.yieldPlans(noTypeFilterNeeded); + } else { + for (final var plan : noTypeFilterNeeded) { + call.yieldPlan(RecordQueryStoreBindingPlan.of(plan, schemaId)); + } + } } for (Map.Entry, Collection> unsatisfiedEntry : unsatisfiedMap.asMap().entrySet()) { - call.yieldPlan( + final RecordQueryTypeFilterPlan typeFilterPlan = new RecordQueryTypeFilterPlan( Quantifier.physical(call.memoizeMemberPlansFromOther(innerReference, unsatisfiedEntry.getValue())), unsatisfiedEntry.getKey(), - Type.Relation.scalarOf(logicalTypeFilterExpression.getResultType()))); + Type.Relation.scalarOf(logicalTypeFilterExpression.getResultType())); + if (schemaId.isCurrentSchema()) { + call.yieldPlan(typeFilterPlan); + } else { + call.yieldPlan(RecordQueryStoreBindingPlan.of(typeFilterPlan, schemaId)); + } } } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaJoinTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaJoinTest.java new file mode 100644 index 0000000000..643094a9ea --- /dev/null +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaJoinTest.java @@ -0,0 +1,128 @@ +/* + * CrossSchemaJoinTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.query; + +import com.apple.foundationdb.relational.api.Options; +import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalExtension; +import com.apple.foundationdb.relational.recordlayer.RelationalConnectionRule; +import com.apple.foundationdb.relational.recordlayer.RelationalStatementRule; +import com.apple.foundationdb.relational.recordlayer.Utils; +import com.apple.foundationdb.relational.utils.ResultSetAssert; +import com.apple.foundationdb.relational.utils.SchemaRule; +import com.apple.foundationdb.relational.utils.SchemaTemplateRule; +import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.net.URI; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Map; + +public class CrossSchemaJoinTest { + + private static final String PRIMARY_TEMPLATE_DEF = + "CREATE TABLE ITEMS (id BIGINT, name STRING, PRIMARY KEY(id))"; + + private static final String SECONDARY_TEMPLATE_NAME = "CrossSchemaJoinTest_SECONDARY_TEMPLATE"; + private static final String SECONDARY_TEMPLATE_DEF = + "CREATE TABLE TAGS (item_id BIGINT, tag STRING, PRIMARY KEY(item_id))"; + private static final String SECONDARY_SCHEMA_NAME = "SECONDARY_SCHEMA"; + + @RegisterExtension + @Order(0) + public final EmbeddedRelationalExtension relationalExtension = new EmbeddedRelationalExtension(); + + @RegisterExtension + @Order(1) + public final SimpleDatabaseRule db = new SimpleDatabaseRule(CrossSchemaJoinTest.class, PRIMARY_TEMPLATE_DEF); + + @RegisterExtension + @Order(2) + public final SchemaTemplateRule secondaryTemplateRule = new SchemaTemplateRule( + SECONDARY_TEMPLATE_NAME, Options.none(), null, SECONDARY_TEMPLATE_DEF); + + @RegisterExtension + @Order(3) + public final SchemaRule secondarySchemaRule = new SchemaRule( + SECONDARY_SCHEMA_NAME, URI.create("/TEST/CrossSchemaJoinTest"), SECONDARY_TEMPLATE_NAME, Options.none()); + + @RegisterExtension + @Order(4) + public final RelationalConnectionRule connection = new RelationalConnectionRule(db::getConnectionUri) + .withSchema(db.getSchemaName()); + + @RegisterExtension + @Order(5) + public final RelationalStatementRule statement = new RelationalStatementRule(connection); + + public CrossSchemaJoinTest() throws SQLException { + } + + @BeforeEach + void setup() throws Exception { + Utils.enableCascadesDebugger(); + statement.execute("INSERT INTO ITEMS VALUES (1, 'Apple'), (2, 'Banana'), (3, 'Cherry')"); + try (var secondaryConn = DriverManager.getConnection(db.getConnectionUri().toString())) { + secondaryConn.setSchema(SECONDARY_SCHEMA_NAME); + try (var stmt = secondaryConn.createStatement()) { + stmt.execute("INSERT INTO TAGS VALUES (1, 'fruit'), (2, 'yellow'), (3, 'red')"); + } + } + } + + @Test + void innerJoinAcrossSchemas() throws Exception { + try (var rs = statement.executeQuery( + "SELECT a.id, a.name, b.tag FROM ITEMS AS a JOIN " + SECONDARY_SCHEMA_NAME + ".TAGS AS b ON a.id = b.item_id ORDER BY a.id")) { + ResultSetAssert.assertThat(rs) + .hasNextRow().hasColumns(Map.of("id", 1L, "name", "Apple", "tag", "fruit")) + .hasNextRow().hasColumns(Map.of("id", 2L, "name", "Banana", "tag", "yellow")) + .hasNextRow().hasColumns(Map.of("id", 3L, "name", "Cherry", "tag", "red")) + .hasNoNextRow(); + } + } + + @Test + void innerJoinAcrossSchemasWithFilter() throws Exception { + try (var rs = statement.executeQuery( + "SELECT a.id, a.name, b.tag FROM ITEMS AS a JOIN " + SECONDARY_SCHEMA_NAME + ".TAGS AS b ON a.id = b.item_id WHERE a.id = 2")) { + ResultSetAssert.assertThat(rs) + .hasNextRow().hasColumns(Map.of("id", 2L, "name", "Banana", "tag", "yellow")) + .hasNoNextRow(); + } + } + + @Test + void queryFromSecondarySchemaTableAlone() throws Exception { + try (var rs = statement.executeQuery( + "SELECT tag FROM " + SECONDARY_SCHEMA_NAME + ".TAGS ORDER BY item_id")) { + ResultSetAssert.assertThat(rs) + .hasNextRow().hasColumns(Map.of("tag", "fruit")) + .hasNextRow().hasColumns(Map.of("tag", "yellow")) + .hasNextRow().hasColumns(Map.of("tag", "red")) + .hasNoNextRow(); + } + } +} diff --git a/yaml-tests/src/test/resources/check-explain/addExplain/add-explain.yamsql b/yaml-tests/src/test/resources/check-explain/addExplain/add-explain.yamsql index 233262143d..099f213453 100644 --- a/yaml-tests/src/test/resources/check-explain/addExplain/add-explain.yamsql +++ b/yaml-tests/src/test/resources/check-explain/addExplain/add-explain.yamsql @@ -32,6 +32,5 @@ test_block: tests: - - query: SELECT id FROM t1 - - explain: "SCAN([IS T1])" - result: [{!l 1}] ... diff --git a/yaml-tests/src/test/resources/check-result-metadata/addResultMetadata/add-result-metadata.yamsql b/yaml-tests/src/test/resources/check-result-metadata/addResultMetadata/add-result-metadata.yamsql index 6fb6487f97..5052ad78b8 100644 --- a/yaml-tests/src/test/resources/check-result-metadata/addResultMetadata/add-result-metadata.yamsql +++ b/yaml-tests/src/test/resources/check-result-metadata/addResultMetadata/add-result-metadata.yamsql @@ -32,6 +32,5 @@ test_block: tests: - - query: SELECT id FROM t1 - - resultMetadata: [{ID: BIGINT}] - result: [{!l 1}] ... From 008ee0fbf3c87426d37fbac14174cc641e3de2ef Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Sun, 24 May 2026 16:01:03 +0100 Subject: [PATCH 06/14] Introduce PlanCacheSchemaKey and multi-schema QueryCacheKey Replace raw String primary cache key with PlanCacheSchemaKey holding an ImmutableSortedSet of schema names. Replace single schemaTemplateVersion in QueryCacheKey with ImmutableSortedMap schemaVersions, enabling cross-schema plan cache keying. Affects: PlanCacheSchemaKey (new), QueryCacheKey, RelationalPlanCache, AstNormalizer.NormalizationResult, PlanGenerator, and their tests. --- .../recordlayer/query/AstNormalizer.java | 16 +-- .../recordlayer/query/PlanGenerator.java | 2 +- .../query/cache/PlanCacheSchemaKey.java | 84 ++++++++++++++ .../query/cache/QueryCacheKey.java | 85 ++++---------- .../query/cache/RelationalPlanCache.java | 4 +- .../recordlayer/query/AstNormalizerTests.java | 6 +- .../query/cache/RelationalPlanCacheTests.java | 104 +++++++++--------- 7 files changed, 177 insertions(+), 124 deletions(-) create mode 100644 fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/PlanCacheSchemaKey.java diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java index 4a190f9808..f4d89e2b81 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java @@ -37,7 +37,9 @@ import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerInvokedRoutine; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; +import com.apple.foundationdb.relational.recordlayer.query.cache.PlanCacheSchemaKey; import com.apple.foundationdb.relational.recordlayer.query.cache.QueryCacheKey; +import com.google.common.collect.ImmutableSortedMap; import com.apple.foundationdb.relational.recordlayer.util.ExceptionUtil; import com.apple.foundationdb.relational.util.Assert; import com.google.common.annotations.VisibleForTesting; @@ -630,10 +632,10 @@ public static NormalizationResult normalizeAst(@Nonnull final SchemaTemplate sch } } return new NormalizationResult( - recordLayerSchemaTemplate.getName(), + PlanCacheSchemaKey.of(recordLayerSchemaTemplate.getName()), QueryCacheKey.of(astNormalizer.getCanonicalSqlString(), getQuerySpecificPlannerConfig(plannerConfiguration, astNormalizer.getQueryOptions()), recordLayerSchemaTemplate.getTransactionBoundMetadataAsString(), - recordLayerSchemaTemplate.getVersion(), userVersion), + ImmutableSortedMap.of(recordLayerSchemaTemplate.getName(), recordLayerSchemaTemplate.getVersion()), userVersion), astNormalizer.getQueryExecutionParameters(), parseTreeInfo.getRootContext(), astNormalizer.getQueryCachingFlags(), @@ -677,7 +679,7 @@ public enum QueryCachingFlags { } @Nonnull - private final String schemaTemplateName; + private final PlanCacheSchemaKey schemaKey; @Nonnull private final QueryCacheKey queryCacheKey; @@ -697,14 +699,14 @@ public enum QueryCachingFlags { @Nonnull private final String query; - public NormalizationResult(@Nonnull final String schemaTemplateName, + public NormalizationResult(@Nonnull final PlanCacheSchemaKey schemaKey, @Nonnull final QueryCacheKey queryCacheKey, @Nonnull final QueryExecutionContext queryExecutionContext, @Nonnull final ParseTree parseTree, @Nonnull final Set queryCachingFlags, @Nonnull final Options queryOptions, @Nonnull final String query) { - this.schemaTemplateName = schemaTemplateName; + this.schemaKey = schemaKey; this.queryCacheKey = queryCacheKey; this.queryExecutionContext = queryExecutionContext; this.parseTree = parseTree; @@ -714,8 +716,8 @@ public NormalizationResult(@Nonnull final String schemaTemplateName, } @Nonnull - public String getSchemaTemplateName() { - return schemaTemplateName; + public PlanCacheSchemaKey getSchemaKey() { + return schemaKey; } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java index 6066385e16..e8019cecc3 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java @@ -173,7 +173,7 @@ private Plan getPlanInternal(@Nonnull String query, @Nonnull KeyValueLogMessa final var planEquivalence = PhysicalPlanEquivalence.of(astHashResult.getQueryExecutionContext().getEvaluationContext()); return planContext.getMetricsCollector().clock(RelationalMetric.RelationalEvent.CACHE_LOOKUP, () -> cache.get().reduce( - astHashResult.getSchemaTemplateName(), + astHashResult.getSchemaKey(), astHashResult.getQueryCacheKey(), planEquivalence, () -> { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/PlanCacheSchemaKey.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/PlanCacheSchemaKey.java new file mode 100644 index 0000000000..0fcfe6cfd3 --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/PlanCacheSchemaKey.java @@ -0,0 +1,84 @@ +/* + * PlanCacheSchemaKey.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.query.cache; + +import com.apple.foundationdb.annotation.API; +import com.google.common.collect.ImmutableSortedSet; + +import javax.annotation.Nonnull; +import java.util.Collection; + +/** + * Primary key for {@link RelationalPlanCache}. Represents the set of schema template names that a + * cached plan was compiled against. Single-schema queries produce a one-element set; cross-schema + * queries produce a multi-element set. + *

+ * The set is always sorted so that {@code {"a","b"}} and {@code {"b","a"}} map to the same bucket. + */ +@API(API.Status.EXPERIMENTAL) +public final class PlanCacheSchemaKey { + + @Nonnull + private final ImmutableSortedSet schemaNames; + + private final int memoizedHashCode; + + private PlanCacheSchemaKey(@Nonnull final ImmutableSortedSet schemaNames) { + this.schemaNames = schemaNames; + this.memoizedHashCode = schemaNames.hashCode(); + } + + @Nonnull + public ImmutableSortedSet getSchemaNames() { + return schemaNames; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + return schemaNames.equals(((PlanCacheSchemaKey) other).schemaNames); + } + + @Override + public int hashCode() { + return memoizedHashCode; + } + + @Override + public String toString() { + return String.join("|", schemaNames); + } + + @Nonnull + public static PlanCacheSchemaKey of(@Nonnull final String schemaName) { + return new PlanCacheSchemaKey(ImmutableSortedSet.of(schemaName)); + } + + @Nonnull + public static PlanCacheSchemaKey of(@Nonnull final Collection schemaNames) { + return new PlanCacheSchemaKey(ImmutableSortedSet.copyOf(schemaNames)); + } +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/QueryCacheKey.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/QueryCacheKey.java index fb2a8a9112..2f5cf6cbcb 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/QueryCacheKey.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/QueryCacheKey.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.relational.recordlayer.query.AstNormalizer; import com.apple.foundationdb.relational.recordlayer.query.PlannerConfiguration; +import com.google.common.collect.ImmutableSortedMap; import javax.annotation.Nonnull; import java.util.Objects; @@ -32,53 +33,13 @@ * This is used to look up a plan in the primary cache (see {@link MultiStageCache} for more information). * It comprises the following fields: *

    - *
  • The schema template name to which the query is bound. It is necessary to use it so we can segregate - * the otherwise identical plans but coming from different schemas (see Example 1 below)
    • - *
  • The schema template version, user version, and a bit-set of all readable indexes
    • - *
  • The canonical query string where all literals are removed and white spaces are normalised (see example 2)
    • + *
  • The schema template versions to which the query is bound, keyed by schema template name. A plan is + * invalidated when any participating schema bumps its version.
    • + *
  • The user version and a bit-set of all readable indexes.
    • + *
  • The canonical query string where all literals are removed and white spaces are normalised (see example below).
    • *
  • The hash of the query, see {@link AstNormalizer} for more information on how is this generated.
    • *
- * Example1 - *
- * Let us assume we have two schema templates ({@code s1} and {@code s2}) defined as the following: - * 
- * {@code
- * create schema template s1
- * create table t1(id bigint, col1 bigint, primary key(id))
- *
- * create schema template s2
- * create table t1(id bigint, col1 bigint, col3 bigint, primary key(id))
- * }
- *
- * If we run a query like this: - * 
- * {@code
- * create schema /FRL/YOUSSEF/s1s with s1
- * connect: "jdbc:embed:/FRL/YOUSSEF?schema=S1S"
- * select * from t1 where col1 > 42;
- * }
- *
- * we would compile the query and return a result set comprising two columns {@code id, col1}. - * we will also cache this query using a {@link QueryCacheKey} key of something like {@code "s1", "select * from t1 where col1 > ? ", 123456789} - *
- * if we run the same query, however after connecting to a schema that uses a {@code s2} instead: - * 
- * {@code
- * create schema /FRL/YOUSSEF/s2s with s2
- * connect: "jdbc:embed:/FRL/YOUSSEF?schema=S2S"
- * select * from t1 where col1 > 53;
- * }
- *
- * we would correctly compile this query and return a result set comprising three columns {@code id, col1, col2}. - * we will also cache this query using {@link QueryCacheKey} key of something like {@code "s2", "select * from t1 where col1 > ? ", 123456789} - *
- * Note that without having the schema template name as part of the {@link QueryCacheKey} both keys would be identical. - * Therefore, we might incorrectly choose the compiled plan of the first query to execute the second query (because we find - * a match in the cache) causing an error since the result set structure is different because table {@code T1} is defined - * differently in {@code S1} and {@code S2}. - *
- *
- * Example 2 + * Example *
* Although these queries appear different, their canonical representation is the same, and will end up using the same * compiled plan. @@ -108,7 +69,12 @@ public final class QueryCacheKey { @Nonnull private final String auxiliaryMetadata; - private final int schemaTemplateVersion; + /** + * Maps each participating schema template name to its version. A cache miss is triggered when + * any entry in this map changes, which is the correct behaviour for cross-schema queries. + */ + @Nonnull + private final ImmutableSortedMap schemaVersions; private final int userVersion; @@ -117,18 +83,15 @@ public final class QueryCacheKey { private QueryCacheKey(@Nonnull final String canonicalQueryString, @Nonnull final PlannerConfiguration plannerConfiguration, @Nonnull final String auxiliaryMetadata, - int schemaTemplateVersion, + @Nonnull final ImmutableSortedMap schemaVersions, int userVersion) { this.canonicalQueryString = canonicalQueryString; - this.schemaTemplateVersion = schemaTemplateVersion; + this.schemaVersions = schemaVersions; this.userVersion = userVersion; this.auxiliaryMetadata = auxiliaryMetadata; this.plannerConfiguration = plannerConfiguration; - // Memoize the hash code. Because this object is used as a key in a hash map, it is important that - // hashCode() be quick. Note that this includes information about the query (canonicalQueryString is like the query hash), - // the schema template version, and the schema (like the set of readable indexes) - this.memoizedHashCode = Objects.hash(canonicalQueryString, schemaTemplateVersion, plannerConfiguration, userVersion, auxiliaryMetadata); + this.memoizedHashCode = Objects.hash(canonicalQueryString, schemaVersions, plannerConfiguration, userVersion, auxiliaryMetadata); } @Override @@ -140,11 +103,11 @@ public boolean equals(Object other) { return false; } final var that = (QueryCacheKey) other; - return schemaTemplateVersion == that.schemaTemplateVersion && - userVersion == that.userVersion && + return userVersion == that.userVersion && Objects.equals(canonicalQueryString, that.canonicalQueryString) && Objects.equals(auxiliaryMetadata, that.auxiliaryMetadata) && - Objects.equals(plannerConfiguration, that.plannerConfiguration); + Objects.equals(plannerConfiguration, that.plannerConfiguration) && + Objects.equals(schemaVersions, that.schemaVersions); } @Override @@ -157,8 +120,9 @@ public String getCanonicalQueryString() { return canonicalQueryString; } - public int getSchemaTemplateVersion() { - return schemaTemplateVersion; + @Nonnull + public ImmutableSortedMap getSchemaVersions() { + return schemaVersions; } @Nonnull @@ -177,16 +141,15 @@ public String getAuxiliaryMetadata() { @Override public String toString() { - return "(" + schemaTemplateVersion + " || " + auxiliaryMetadata + ")" + "||" + canonicalQueryString + "||" + memoizedHashCode; + return "(" + schemaVersions + " || " + auxiliaryMetadata + ")" + "||" + canonicalQueryString + "||" + memoizedHashCode; } @Nonnull public static QueryCacheKey of(@Nonnull final String query, @Nonnull final PlannerConfiguration plannerConfiguration, @Nonnull final String auxiliaryMetadata, - int schemaTemplateVersion, + @Nonnull final ImmutableSortedMap schemaVersions, int userVersion) { - return new QueryCacheKey(query, plannerConfiguration, auxiliaryMetadata, schemaTemplateVersion, - userVersion); + return new QueryCacheKey(query, plannerConfiguration, auxiliaryMetadata, schemaVersions, userVersion); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCache.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCache.java index 13e8192d8a..d93fc8e43e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCache.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCache.java @@ -34,7 +34,7 @@ * This is just a specialization of {@link MultiStageCache} with concrete types specific to plan caching. */ @API(API.Status.EXPERIMENTAL) -public final class RelationalPlanCache extends MultiStageCache> { +public final class RelationalPlanCache extends MultiStageCache> { @Nonnull private static final TimeUnit DEFAULT_TTL_TIME_UNIT = TimeUnit.MILLISECONDS; @@ -61,7 +61,7 @@ private RelationalPlanCache(int size, super(size, secondarySize, tertiarySize, ttl, ttlTimeUnit, secondaryTtl, secondaryTtlTimeUnit, tertiaryTtl, tertiaryTtlTimeUnit, executor, secondaryExecutor, tertiaryExecutor, ticker); } - public static final class RelationalCacheBuilder extends MultiStageCache.Builder, RelationalCacheBuilder> { + public static final class RelationalCacheBuilder extends MultiStageCache.Builder, RelationalCacheBuilder> { public RelationalCacheBuilder() { size = (Integer) (Options.defaultOptions().get(Options.Name.PLAN_CACHE_PRIMARY_MAX_ENTRIES)); diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizerTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizerTests.java index ba95364cf3..18496cf7b9 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizerTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizerTests.java @@ -39,6 +39,7 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.recordlayer.query.cache.QueryCacheKey; +import com.google.common.collect.ImmutableSortedMap; import com.apple.foundationdb.relational.recordlayer.query.functions.CompiledSqlFunction; import com.apple.foundationdb.relational.recordlayer.util.Hex; import com.apple.foundationdb.relational.util.Assert; @@ -1496,8 +1497,9 @@ void visitFullDescribeStatementThrows() throws ReflectiveOperationException { @Test void queryCacheKeyToString() { - final var key = QueryCacheKey.of("select ? from testTable", plannerConfiguration, "someAuxiliaryMetadata", 3, 7); - final var expected = "(3 || someAuxiliaryMetadata)||select ? from testTable||" + key.hashCode(); + final var key = QueryCacheKey.of("select ? from testTable", plannerConfiguration, "someAuxiliaryMetadata", + ImmutableSortedMap.of("testSchema", 3), 7); + final var expected = "({testSchema=3} || someAuxiliaryMetadata)||select ? from testTable||" + key.hashCode(); Assertions.assertThat(key).hasToString(expected); } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java index 027f3d1dbe..9b8ff69384 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java @@ -53,7 +53,9 @@ import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; import com.apple.foundationdb.relational.utils.TestSchemas; import com.apple.test.BooleanSource; +import com.apple.foundationdb.relational.recordlayer.query.cache.PlanCacheSchemaKey; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSortedMap; import com.google.common.testing.FakeTicker; import org.apache.commons.lang3.tuple.Pair; import org.assertj.core.groups.Tuple; @@ -407,12 +409,12 @@ private static QueryPlanConstraint cons(@Nonnull final QueryPlanConstraint... co private static void shouldBe(@Nonnull final RelationalPlanCache cache, @Nonnull final Map> expectedLayout) { Map> result = new HashMap<>(); - for (String key : cache.getStats().getAllKeys()) { + for (PlanCacheSchemaKey key : cache.getStats().getAllKeys()) { for (QueryCacheKey secondaryKey : cache.getStats().getAllSecondaryKeys(key)) { final var resMap = result.computeIfAbsent( new Tuple(secondaryKey.getCanonicalQueryString(), key, - secondaryKey.getSchemaTemplateVersion(), + secondaryKey.getSchemaVersions(), secondaryKey.getUserVersion(), secondaryKey.getPlannerConfiguration(), secondaryKey.getAuxiliaryMetadata()), @@ -515,13 +517,13 @@ void testCachingDifferentQueries() throws Exception { // adding an entry to the cache to warm it up. planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // let's add an identical query with different boundaries, so we can use a different index (i1980). // however, we end up with in the same primary cache bucket because the primary key is identical. planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1980 AND YEAR < 1985", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1980); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of( ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970, ppe(cons(c1980Cp0(7), c1980Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1980))); @@ -529,11 +531,11 @@ void testCachingDifferentQueries() throws Exception { // let's try another query, now the query itself is different -> must be a different entry in the _primary_ cache. planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1980 OR YEAR < 1985", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), Scan); shouldBe(cache, Map.of( - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of( ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970, ppe(cons(c1980Cp0(7), c1980Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1980), - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? OR \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? OR \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), // tautology is expected since a primary scan accepts everything. Map.of(ppe(tautology, cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), Scan) )); @@ -546,16 +548,16 @@ void testCachingDifferentSchemaTemplateNames() throws Exception { // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // customer 2 issues exactly the same query, the environment is identical to first query, but the schema template is different ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_2", 10, 100, Set.of(i1970, i1980), i1970); shouldBe(cache, Map.of( - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970), // so it is added as a separate entry in the main cache - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_2", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_2"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_2", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); } @@ -566,16 +568,16 @@ void testCachingDifferentSchemaTemplateVersions() throws Exception { // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // customer 2 issues exactly the same query, the environment is identical to first query, but the schema template version is different ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 11, 100, Set.of(i1970, i1980), i1970); shouldBe(cache, Map.of( - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970), // so it is added as a separate entry in the main cache - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 11, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 11), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); } @@ -586,16 +588,16 @@ void testCachingDifferentUserVersion() throws Exception { // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // customer 2 issues exactly the same query, the environment is identical to first query, but the user version is different ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 101, Set.of(i1970, i1980), i1970); shouldBe(cache, Map.of( - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970), // so it is added as a separate entry in the main cache - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 101, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 101, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); } @@ -606,7 +608,7 @@ void testCachingDifferentReadableIndexes() throws Exception { // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // customer 2 issues exactly the same query, the environment is identical to first query, but the readable indexes set is different ... @@ -614,10 +616,10 @@ void testCachingDifferentReadableIndexes() throws Exception { // in that case, we want to make sure that we re-plan the query giving the optimizer a chance to consider the newly created index for producing a better plan. planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), i1970); shouldBe(cache, Map.of( - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970), // so it is added as a separate entry in the main cache - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); } @@ -628,12 +630,12 @@ void testCachingDifferentConstraints() throws Exception { // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // customer 2 issues exactly the same query, the environment is identical to first query, but which predicates that fall outside the ranges of the chosen index of the first query planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1980 AND YEAR < 1983", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1980); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of( ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970, ppe(cons(c1980Cp0(7), c1980Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1980))); @@ -646,7 +648,7 @@ void testEvictionFromPrimaryCache() throws Exception { // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // 10 MS TTL for primary cache, if we pass 11 -> item must be evicted from primary cache. @@ -661,24 +663,24 @@ void testEvictionFromPrimaryCacheWithLru() throws Exception { // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // customer 2 issues a different query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 OR YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), Scan); shouldBe(cache, Map.of( - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970), // ... which is added as a separate entry in the main cache - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? OR \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? OR \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(tautology, cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), Scan))); // customer 3 issues yet a different query, cache size is two, evicts an item ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), Scan); shouldBe(cache, Map.of( - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(tautology, cons(ofTypeIntCp0(7), isNotNullInt(7))), Scan), - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); } @@ -689,7 +691,7 @@ void testEvictionFromSecondaryCacheRemovesPrimaryKeyWhenEmpty() throws Exception // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // 10 MS TTL for primary cache, 5 MS TTL for secondary cache, if we pass 7 -> item must be evicted from secondary cache @@ -710,7 +712,7 @@ void testEvictionFromTertiaryCache() throws Exception { // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // pass some time, so we have some jitter between secondary cache items necessary for producing deterministic test results. @@ -718,7 +720,7 @@ void testEvictionFromTertiaryCache() throws Exception { // customer 2 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1980 AND YEAR < 1985", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1980); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of( ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970, ppe(cons(c1980Cp0(7), c1980Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1980))); @@ -726,7 +728,7 @@ void testEvictionFromTertiaryCache() throws Exception { // 10 MS TTL for primary cache, 5 MS TTL for secondary cache, we already passed 2, now if pass 3 more we'll // evict the first cached item in the secondary cache, but _not_ the more recent one. ticker.advance(Duration.of(3, ChronoUnit.MILLIS)); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1980Cp0(7), c1980Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1980))); } @@ -737,12 +739,12 @@ void testEvictionFromSecondaryCacheWithLru() throws Exception { // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // customer 2 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1980 AND YEAR < 1985", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), i1980); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), ""), Map.of( ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970, ppe(cons(c1980Cp0(7), c1980Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1980))); @@ -750,7 +752,7 @@ void testEvictionFromSecondaryCacheWithLru() throws Exception { // secondary cache has capacity of two, attempting to add a third item causes an eviction (LRU). // customer 3 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1990 AND YEAR < 1992", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), i1990); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), ""), Map.of( ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970, ppe(cons(c1990Cp0(7), c1990Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1990))); @@ -763,7 +765,7 @@ void testEvictionFromTertiaryCacheRemovesSecondaryKeyWhenEmpty() throws Exceptio // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); @@ -778,7 +780,7 @@ void testEvictionFromTertiaryCacheRemovesSecondaryKeyWhenEmpty() throws Exceptio cache.cleanUp(); // Only one secondary key, the first one was removed - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(tautology, cons(ofTypeIntCp0(7), isNotNullInt(7))), Scan))); } @@ -789,14 +791,14 @@ void testPlanReductionViaCosting() throws Exception { // customer 1 issues a query ... planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // customer 2 issues a query that forces a "bad" plan within the same secondary cache entry of the plan above // because of scan boundaries that fall outside the range of the filtered index planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 2005 AND YEAR < 2010", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), Scan); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970, ppe(tautology, cons(ofTypeIntCp0(7), ofTypeIntCp1(11), @@ -804,7 +806,7 @@ void testPlanReductionViaCosting() throws Exception { // we should still get back the "good" plan (scanning i1970) planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970, ppe(tautology, cons(ofTypeIntCp0(7), ofTypeIntCp1(11), @@ -821,7 +823,7 @@ void testConstraintsWithTemporaryFunction() throws Exception { "CREATE TEMPORARY FUNCTION SCI_FI_BOOKS_OF_80S() ON COMMIT DROP FUNCTION AS SELECT * FROM SCI_FI_BOOKS() WHERE YEAR > 1980 AND YEAR < 1989"), "SELECT * FROM SCI_FI_BOOKS_OF_80S()", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), i1980); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"SCI_FI_BOOKS_OF_80S\" ( ) ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), "CREATE TEMPORARY FUNCTION \"SCI_FI_BOOKS\" ( ) " + + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"SCI_FI_BOOKS_OF_80S\" ( ) ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), "CREATE TEMPORARY FUNCTION \"SCI_FI_BOOKS\" ( ) " + "ON COMMIT DROP FUNCTION AS SELECT * FROM \"BOOKS\" WHERE \"TITLE\" LIKE ? ||CREATE TEMPORARY FUNCTION \"SCI_FI_BOOKS_OF_80S\" ( ) " + "ON COMMIT DROP FUNCTION AS SELECT * FROM \"SCI_FI_BOOKS\" ( ) WHERE \"YEAR\" > ? AND \"YEAR\" < ? "), Map.of(ppe(cons( @@ -848,7 +850,7 @@ void testConstraintsWithTemporaryFunctionsIncludingUnusedOnes() throws Exception "CREATE TEMPORARY FUNCTION SCI_FI_BOOKS_OF_80S() ON COMMIT DROP FUNCTION AS SELECT * FROM SCI_FI_BOOKS() WHERE YEAR > 1980 AND YEAR < 1989"), "SELECT * FROM SCI_FI_BOOKS_OF_80S()", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), i1980); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"SCI_FI_BOOKS_OF_80S\" ( ) ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), "CREATE TEMPORARY FUNCTION \"OTHER_BOOKS\" ( ) " + + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"SCI_FI_BOOKS_OF_80S\" ( ) ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), "CREATE TEMPORARY FUNCTION \"OTHER_BOOKS\" ( ) " + "ON COMMIT DROP FUNCTION AS SELECT * FROM \"BOOKS\" WHERE \"TITLE\" LIKE ? ||CREATE TEMPORARY FUNCTION \"SCI_FI_BOOKS\" ( ) " + "ON COMMIT DROP FUNCTION AS SELECT * FROM \"BOOKS\" WHERE \"TITLE\" LIKE ? ||CREATE TEMPORARY FUNCTION \"SCI_FI_BOOKS_OF_80S\" ( ) " + "ON COMMIT DROP FUNCTION AS SELECT * FROM \"SCI_FI_BOOKS\" ( ) WHERE \"YEAR\" > ? AND \"YEAR\" < ? "), @@ -873,7 +875,7 @@ void testConstraintsWithTemporaryFunctionsMultipleReferences() throws Exception "CREATE TEMPORARY FUNCTION SCI_FI_BOOKS_OF_80S() ON COMMIT DROP FUNCTION AS SELECT * FROM SCI_FI_BOOKS() WHERE YEAR > 1980 AND YEAR < 1989"), "SELECT * FROM SCI_FI_BOOKS_OF_80S(), SCI_FI_BOOKS_OF_80S()", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), i1980); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"SCI_FI_BOOKS_OF_80S\" ( ) , \"SCI_FI_BOOKS_OF_80S\" ( ) ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"SCI_FI_BOOKS_OF_80S\" ( ) , \"SCI_FI_BOOKS_OF_80S\" ( ) ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), "CREATE TEMPORARY FUNCTION \"SCI_FI_BOOKS\" ( ) " + "ON COMMIT DROP FUNCTION AS SELECT * FROM \"BOOKS\" WHERE \"TITLE\" LIKE ? ||CREATE TEMPORARY FUNCTION \"SCI_FI_BOOKS_OF_80S\" ( ) " + "ON COMMIT DROP FUNCTION AS SELECT * FROM \"SCI_FI_BOOKS\" ( ) WHERE \"YEAR\" > ? AND \"YEAR\" < ? "), @@ -900,7 +902,7 @@ void testConstraintsWithTemporaryFunctionsMultipleLiterals() throws Exception { "SELECT * FROM SCI_FI_BOOKS_OF_80S() AS A, OTHER_BOOKS() AS B WHERE A.YEAR > 1985 AND A.TITLE = 'OTHER'", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980, i1990), i1980); shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"SCI_FI_BOOKS_OF_80S\" ( ) AS \"A\" , \"OTHER_BOOKS\" ( ) AS \"B\" WHERE \"A\" . \"YEAR\" > ? AND \"A\" . \"TITLE\" = ? ", - "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980, i1990)), "CREATE TEMPORARY FUNCTION \"OTHER_BOOKS\" ( ) " + + PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980, i1990)), "CREATE TEMPORARY FUNCTION \"OTHER_BOOKS\" ( ) " + "ON COMMIT DROP FUNCTION AS SELECT * FROM \"BOOKS\" WHERE \"TITLE\" LIKE ? ||CREATE TEMPORARY FUNCTION \"SCI_FI_BOOKS\" ( ) " + "ON COMMIT DROP FUNCTION AS SELECT * FROM \"BOOKS\" WHERE \"TITLE\" LIKE ? ||CREATE TEMPORARY FUNCTION \"SCI_FI_BOOKS_OF_80S\" ( ) " + "ON COMMIT DROP FUNCTION AS SELECT * FROM \"SCI_FI_BOOKS\" ( ) WHERE \"YEAR\" > ? AND \"YEAR\" < ? "), @@ -929,7 +931,7 @@ void testPlanningQueryWithAndWithoutDisabledPlannerRewriteRules() throws Excepti // customer 1 issues a query with disabled planner rules planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), disabledRulesOption, i1970); shouldBe(cache, Map.of( - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980), disabledRulesOption), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980), disabledRulesOption), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // customer 2 issues the same query, with enabled planner rules. @@ -938,10 +940,10 @@ void testPlanningQueryWithAndWithoutDisabledPlannerRewriteRules() throws Excepti // cache should contain two entries shouldBe(cache, Map.of( // ... one with disabled planner rules. - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980), disabledRulesOption), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980), disabledRulesOption), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970), // ... another one with enabled planner rules. - new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // this is needed because expiration is done passively for better performance. @@ -966,15 +968,15 @@ void testPlanningQueryWithAndWithoutPlanRightDeepOption(boolean specifyInQuery) planQuery(cache, queryWithOption, "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), optionsWithOption, i1970); shouldBe(cache, Map.of( - new Tuple(expectedCononicalString, "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980), rightDeepOption), ""), + new Tuple(expectedCononicalString, PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980), rightDeepOption), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); // now run the plain query (no option either way) planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), Options.none(), i1970); shouldBe(cache, Map.of( - new Tuple(expectedCononicalString, "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980), rightDeepOption), ""), + new Tuple(expectedCononicalString, PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980), rightDeepOption), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970), - new Tuple(expectedCononicalString, "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + new Tuple(expectedCononicalString, PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); cache.cleanUp(); @@ -986,15 +988,15 @@ void testExplainReusesCachedPlan() throws Exception { final var cache = getCache(ticker); planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); planQuery(cache, "SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); planQuery(cache, "EXPLAIN SELECT * FROM BOOKS WHERE YEAR > 1970 AND YEAR < 1979", "SCHEMA_TEMPLATE_1", 10, 100, Set.of(i1970, i1980), i1970); - shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", "SCHEMA_TEMPLATE_1", 10, 100, configOf(Set.of(i1970, i1980)), ""), + shouldBe(cache, Map.of(new Tuple("SELECT * FROM \"BOOKS\" WHERE \"YEAR\" > ? AND \"YEAR\" < ? ", PlanCacheSchemaKey.of("SCHEMA_TEMPLATE_1"), ImmutableSortedMap.of("SCHEMA_TEMPLATE_1", 10), 100, configOf(Set.of(i1970, i1980)), ""), Map.of(ppe(cons(c1970Cp0(7), c1970Cp1(11)), cons(ofTypeIntCp0(7), ofTypeIntCp1(11), isNotNullInt(7), isNotNullInt(11))), i1970))); } } From e5e8f2f56b509c83b2139109171a9aa680d54661 Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Tue, 26 May 2026 00:04:58 +0100 Subject: [PATCH 07/14] Fix AbstractDataAccessRule secondary-schema scan wrapping and add proto serialization to RecordQueryStoreBindingPlan AbstractDataAccessRule.createScansForMatches now wraps scan plans in RecordQueryStoreBindingPlan at creation time when the match candidate belongs to a secondary schema. This places the store-binding node inside any compensation wrappers (MAP, SELECT) produced by Compensation.applyAllNeededCompensations, so the secondary store binding survives to the final physical plan for standalone secondary-schema queries as well as cross-schema joins. The previous approach wrapped only at the dataAccessExpressions level after compensation; the instanceof RecordQueryPlan check failed for logical SelectExpression compensation wrappers, causing the secondary store to be silently dropped. RecordQueryStoreBindingPlan gains full proto serialization: adds PRecordQueryStoreBindingPlan message (field 41 in PRecordQueryPlan.oneof), implements toProto/toRecordQueryPlanProto/fromProto, and registers a @AutoService(PlanDeserializer.class) Deserializer inner class. This enables continuation support for queries spanning secondary schemas. --- .../rules/AbstractDataAccessRule.java | 43 +++++++++++------- .../plans/RecordQueryStoreBindingPlan.java | 45 ++++++++++++++++++- .../src/main/proto/record_query_plan.proto | 9 ++++ 3 files changed, 79 insertions(+), 18 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java index 2041af229d..121a043412 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/AbstractDataAccessRule.java @@ -181,6 +181,23 @@ public void onMatch(@Nonnull final CascadesRuleCall call) { secondarySchemaId = sid; } + // Filter match candidates to only those belonging to the expected schema, preventing + // same-named tables from different schemas from generating mixed plans in the same group. + final var planContext = call.getContext(); + final var schemaFilteredMatches = completeMatches.stream() + .filter(match -> { + final SchemaIdentifier candidateSchema = planContext.getSchemaIdForMatchCandidate(match.getMatchCandidate()); + if (secondarySchemaId != null) { + return secondarySchemaId.equals(candidateSchema); + } else { + return candidateSchema.isCurrentSchema(); + } + }) + .collect(ImmutableList.toImmutableList()); + if (schemaFilteredMatches.isEmpty()) { + return; + } + // // return if there is no pre-determined interesting ordering // @@ -196,7 +213,7 @@ public void onMatch(@Nonnull final CascadesRuleCall call) { // group all successful matches by their sets of compensated aliases final var matchPartitionByMatchAliasMap = - completeMatches + schemaFilteredMatches .stream() .flatMap(match -> { final var compensatedAliases = match.getCompensatedAliases(); @@ -257,20 +274,7 @@ public void onMatch(@Nonnull final CascadesRuleCall call) { dataAccessForMatchPartition(call, requestedOrderings, matchPartition); - if (secondarySchemaId != null) { - final LinkedIdentitySet wrapped = new LinkedIdentitySet<>(); - for (final RelationalExpression expr : dataAccessExpressions) { - if (expr instanceof RecordQueryPlan) { - final Reference innerRef = call.memoizePlan((RecordQueryPlan) expr); - wrapped.add(new RecordQueryStoreBindingPlan(Quantifier.physical(innerRef), secondarySchemaId)); - } else { - wrapped.add(expr); - } - } - call.yieldMixedUnknownExpressions(wrapped); - } else { - call.yieldMixedUnknownExpressions(dataAccessExpressions); - } + call.yieldMixedUnknownExpressions(dataAccessExpressions); } } } @@ -914,9 +918,16 @@ private static Map createScansForMatches(@Nonnull singleMatchedAccessVectored -> { final var singleMatchedAccess = singleMatchedAccessVectored.getElement(); final var partialMatch = singleMatchedAccess.getPartialMatch(); - return partialMatch.getMatchCandidate() + final RecordQueryPlan plan = partialMatch.getMatchCandidate() .toEquivalentPlan(partialMatch, planContext, memoizer, singleMatchedAccess.isReverseScanOrder()); + final SchemaIdentifier schemaId = + planContext.getSchemaIdForMatchCandidate(partialMatch.getMatchCandidate()); + if (!schemaId.isCurrentSchema()) { + return new RecordQueryStoreBindingPlan( + Quantifier.physical(memoizer.memoizePlan(plan)), schemaId); + } + return plan; })); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java index 41e7571158..2b6436b7d5 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java @@ -25,10 +25,12 @@ import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanDeserializer; import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.planprotos.PRecordQueryPlan; +import com.apple.foundationdb.record.planprotos.PRecordQueryStoreBindingPlan; import com.apple.foundationdb.record.provider.common.StoreTimer; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; @@ -47,6 +49,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.auto.service.AutoService; import com.google.protobuf.Message; import javax.annotation.Nonnull; @@ -223,13 +226,51 @@ public PlannerGraph rewritePlannerGraph(@Nonnull final List { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PRecordQueryStoreBindingPlan.class; + } + + @Nonnull + @Override + public RecordQueryStoreBindingPlan fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PRecordQueryStoreBindingPlan proto) { + return RecordQueryStoreBindingPlan.fromProto(serializationContext, proto); + } } @Nonnull diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index f413bbf2af..7632fac63f 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -1739,6 +1739,7 @@ message PRecordQueryPlan { PRecordQueryStreamingAggregationPlan2 streaming_aggregation_plan2 = 38; PRecordQueryMultiIntersectionOnValuesPlan multi_intersection_on_values_plan = 39; PRecordQueryRecursiveDfsJoinPlan recursive_dfs_join_plan = 40; + PRecordQueryStoreBindingPlan store_binding_plan = 41; } } @@ -2386,3 +2387,11 @@ message PRecordQueryRecursiveDfsJoinPlan { optional string prior_value_correlation = 3; optional PDfsTraversalStrategy dfs_traversal_strategy = 4; } + +// +// PRecordQueryStoreBindingPlan +// +message PRecordQueryStoreBindingPlan { + optional PPhysicalQuantifier inner = 1; + optional string schema_name = 2; +} From c5761a7340734b3cf87112dea5b7c9ff9abeda30 Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Tue, 26 May 2026 00:07:11 +0100 Subject: [PATCH 08/14] Wire secondary-schema stores into the execution context at connection time TransactionBoundDatabase now populates storesBySchemaName from the additional stores map on RecordStoreAndRecordContextTransaction, so that loadRecordStore(schemaId) can return the secondary FDBRecordStoreBase that RecordQueryStoreBindingPlan looks up at execution time. --- .../RecordStoreAndRecordContextTransaction.java | 15 +++++++++++++++ .../catalog/TransactionBoundDatabase.java | 8 +++++++- .../recordlayer/storage/BackingRecordStore.java | 4 ++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordStoreAndRecordContextTransaction.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordStoreAndRecordContextTransaction.java index c50a546c50..ba03bf38e2 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordStoreAndRecordContextTransaction.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordStoreAndRecordContextTransaction.java @@ -33,6 +33,8 @@ import com.google.protobuf.Message; import javax.annotation.Nonnull; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; /** @@ -45,6 +47,7 @@ public class RecordStoreAndRecordContextTransaction implements Transaction { FDBRecordStoreBase store; RecordContextTransaction transaction; + Map> additionalStores = new HashMap<>(); /** * the schema template this transaction is bound to. This is mainly needed when accessing the plan cache @@ -105,6 +108,18 @@ public FDBRecordStoreBase getRecordStore() { return store; } + @Nonnull + public Map> getAdditionalStores() { + return additionalStores; + } + + @Nonnull + public RecordStoreAndRecordContextTransaction withAdditionalStore(@Nonnull String schemaName, + @Nonnull FDBRecordStoreBase additionalStore) { + this.additionalStores.put(schemaName, additionalStore); + return this; + } + @Nonnull public SchemaTemplate getBoundSchemaTemplate() { return boundSchemaTemplate; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/TransactionBoundDatabase.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/TransactionBoundDatabase.java index ac15179363..7751b673ad 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/TransactionBoundDatabase.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/TransactionBoundDatabase.java @@ -49,6 +49,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URI; +import java.util.HashMap; +import java.util.Map; /** * There can only be 1 Database object per Connection instance, and its lifecycle is managed by the connection @@ -66,6 +68,7 @@ public class TransactionBoundDatabase extends AbstractDatabase { @Nullable private final KeySpace keySpace; BackingStore store; + Map storesBySchemaName = new HashMap<>(); URI uri; private static final MetadataOperationsFactory onlyTemporaryFunctionOperationsFactory = new AbstractMetadataOperationsFactory() { @@ -97,6 +100,9 @@ public RelationalConnection connect(@Nullable Transaction transaction) throws Re } final var recordStoreAndRecordContextTx = transaction.unwrap(RecordStoreAndRecordContextTransaction.class); store = BackingRecordStore.fromTransactionWithStore(recordStoreAndRecordContextTx); + storesBySchemaName.clear(); + recordStoreAndRecordContextTx.getAdditionalStores().forEach((schemaName, additionalStore) -> + storesBySchemaName.put(schemaName, BackingRecordStore.fromTransactionAndStore(recordStoreAndRecordContextTx, additionalStore))); final var boundSchemaTemplate = recordStoreAndRecordContextTx.getBoundSchemaTemplate(); EmbeddedRelationalConnection connection = new EmbeddedRelationalConnection(this, new HollowStoreCatalog(boundSchemaTemplate, keySpace), ((RecordStoreAndRecordContextTransaction) transaction).getRecordContextTransaction(), options); @@ -106,7 +112,7 @@ public RelationalConnection connect(@Nullable Transaction transaction) throws Re @Override public BackingStore loadRecordStore(@Nonnull String schemaId, @Nonnull FDBRecordStoreBase.StoreExistenceCheck existenceCheck) { - return store; + return storesBySchemaName.getOrDefault(schemaId, store); } @Override diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/storage/BackingRecordStore.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/storage/BackingRecordStore.java index d9cedc6453..d5e39984f4 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/storage/BackingRecordStore.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/storage/BackingRecordStore.java @@ -221,6 +221,10 @@ public static BackingRecordStore fromTransactionWithStore(@Nonnull RecordStoreAn return new BackingRecordStore(txn, txn.getRecordStore()); } + public static BackingRecordStore fromTransactionAndStore(@Nonnull Transaction txn, @Nonnull FDBRecordStoreBase store) { + return new BackingRecordStore(txn, store); + } + @SuppressWarnings("PMD.PreserveStackTrace") public static BackingRecordStore load(@Nonnull Transaction txn, @Nonnull StoreConfig config, @Nonnull FDBRecordStoreBase.StoreExistenceCheck existenceCheck) throws RelationalException { //TODO(bfines) error handling if this store doesn't exist From 942da22b9be831e5cf0bb682f02f15179ab98263 Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Tue, 26 May 2026 00:07:22 +0100 Subject: [PATCH 09/14] Namespace secondary-schema proto types and enumerate all schemas in catalog metadata LogicalOperator prefixes secondary-schema type names with the schema name (e.g. "OTHER_SCHEMA.things") to avoid TypeRepository collisions when two schemas define identically-named tables, while preserving the original FDB record type name as the scan key. CatalogMetaData.getTables/getColumns now enumerates all schemas in the database when no schema filter is provided, enabling cross-schema metadata inspection. --- .../recordlayer/CatalogMetaData.java | 271 ++++++++++-------- .../recordlayer/query/LogicalOperator.java | 12 +- 2 files changed, 154 insertions(+), 129 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java index 759e071d79..55a49d7c6e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java @@ -114,15 +114,32 @@ public RelationalResultSet getTables(String database, String schema, String tabl if (database == null) { throw new SQLFeatureNotSupportedException("Cannot scan across Databases yet", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); } - if (schema == null) { - throw new SQLFeatureNotSupportedException("Cannot scan across Schemas yet", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); - } if (tableName != null) { throw new SQLFeatureNotSupportedException("Table filters on getTables() is not supported yet", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); } if (types != null) { throw new SQLFeatureNotSupportedException("Type filters on getTables() is not supported yet", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); } + if (schema == null) { + return conn.runIsolatedInTransactionIfPossible(() -> { + final List tableList = new ArrayList<>(); + for (final String schemaName : listAllSchemaNames(database)) { + loadSchemaMetadata(database, schemaName).getRecordTypesList().stream() + .map(type -> new ArrayRow(new Object[]{ + database, schemaName, type.getName(), + type.hasSinceVersion() ? type.getSinceVersion() : null + })) + .forEach(tableList::add); + } + final var tablesStructType = DataType.StructType.from("TABLES", List.of( + DataType.StructType.Field.from("TABLE_CAT", DataType.Primitives.NULLABLE_STRING.type(), 0), + DataType.StructType.Field.from("TABLE_SCHEM", DataType.Primitives.NULLABLE_STRING.type(), 1), + DataType.StructType.Field.from("TABLE_NAME", DataType.Primitives.NULLABLE_STRING.type(), 2), + DataType.StructType.Field.from("TABLE_VERSION", DataType.Primitives.NULLABLE_LONG.type(), 3) + ), true); + return new IteratorResultSet(RelationalStructMetaData.of(tablesStructType), tableList.iterator(), 0); + }); + } return conn.runIsolatedInTransactionIfPossible(() -> { final RecordMetaDataProto.MetaData schemaInfo = loadSchemaMetadata(database, schema); List tableList = schemaInfo.getRecordTypesList().stream() @@ -187,86 +204,22 @@ public RelationalResultSet getColumns(String database, String schema, String tab if (database == null || database.isEmpty()) { throw new SQLFeatureNotSupportedException("Cannot scan across Databases yet", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); } - if (schema == null || schema.isEmpty()) { - throw new SQLFeatureNotSupportedException("Cannot scan across Schemas yet", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); - } if (tablePattern == null || tablePattern.isEmpty()) { throw new SQLFeatureNotSupportedException("Table must be specified", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); } if (columnPattern != null) { throw new SQLFeatureNotSupportedException("Column filters on getColumns() is not supported yet", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); } - return conn.runIsolatedInTransactionIfPossible(() -> { - //TODO(bfines) this is a weird way of doing this, is there a better way? - RecordMetaData rmd = new CatalogMetaDataProvider(this.catalog, URI.create(database), schema, conn.getTransaction()).getRecordMetaData(); - Descriptors.FileDescriptor fileDesc = rmd.getRecordsDescriptor(); - //verify that it is in fact a table - try { - rmd.getRecordType(tablePattern); - } catch (MetaDataException mde) { - throw new RelationalException("table <" + tablePattern + "> does not exist", ErrorCode.UNDEFINED_TABLE); - } - //now get its column data - final Descriptors.Descriptor tableDescriptor = fileDesc.findMessageTypeByName(tablePattern); - final List columnDefs = tableDescriptor.getFields().stream() - .map(field -> { - Object[] row = { - database, - schema, - tablePattern, - field.getName(), - ProtobufDdlUtil.getSqlType(field), - ProtobufDdlUtil.getTypeName(field), - -1, - 0, - null, - null, - DatabaseMetaData.columnNullableUnknown, //TODO(bfines) we can probably figure this out - null, - field.getJavaType() != Descriptors.FieldDescriptor.JavaType.MESSAGE ? field.getDefaultValue() : null, - -1, - -1, - -1, - field.getIndex() + 1, - "YES", - null, - null, - null, - null, - "NO", - "NO" - }; - return new ArrayRow(row); - }).collect(Collectors.toList()); - - final var columnsStructType = DataType.StructType.from("PRIMARY_KEYS", List.of( - DataType.StructType.Field.from("TABLE_CAT", DataType.Primitives.NULLABLE_STRING.type(), 0), - DataType.StructType.Field.from("TABLE_SCHEM", DataType.Primitives.NULLABLE_STRING.type(), 1), - DataType.StructType.Field.from("TABLE_NAME", DataType.Primitives.NULLABLE_STRING.type(), 2), - DataType.StructType.Field.from("COLUMN_NAME", DataType.Primitives.NULLABLE_STRING.type(), 3), - DataType.StructType.Field.from("DATA_TYPE", DataType.Primitives.NULLABLE_STRING.type(), 4), - DataType.StructType.Field.from("TYPE_NAME", DataType.Primitives.NULLABLE_STRING.type(), 5), - DataType.StructType.Field.from("COLUMN_SIZE", DataType.Primitives.NULLABLE_INTEGER.type(), 6), - DataType.StructType.Field.from("BUFFER_LENGTH", DataType.Primitives.NULLABLE_INTEGER.type(), 7), - DataType.StructType.Field.from("DECIMAL_DIGITS", DataType.Primitives.NULLABLE_INTEGER.type(), 8), - DataType.StructType.Field.from("NUM_PREC_RADIX", DataType.Primitives.NULLABLE_INTEGER.type(), 9), - DataType.StructType.Field.from("NULLABLE", DataType.Primitives.NULLABLE_INTEGER.type(), 10), - DataType.StructType.Field.from("REMARKS", DataType.Primitives.NULLABLE_STRING.type(), 11), - DataType.StructType.Field.from("COLUMN_DEF", DataType.Primitives.NULLABLE_STRING.type(), 12), - DataType.StructType.Field.from("SQL_DATA_TYPE", DataType.Primitives.NULLABLE_INTEGER.type(), 13), - DataType.StructType.Field.from("SQL_DATETIME_SUB", DataType.Primitives.NULLABLE_INTEGER.type(), 14), - DataType.StructType.Field.from("CHAR_OCTET_LENGTH", DataType.Primitives.NULLABLE_INTEGER.type(), 15), - DataType.StructType.Field.from("ORDINAL_POSITION", DataType.Primitives.NULLABLE_INTEGER.type(), 16), - DataType.StructType.Field.from("IS_NULLABLE", DataType.Primitives.NULLABLE_STRING.type(), 17), - DataType.StructType.Field.from("SCOPE_CATALOG", DataType.Primitives.NULLABLE_STRING.type(), 18), - DataType.StructType.Field.from("SCOPE_SCHEMA", DataType.Primitives.NULLABLE_STRING.type(), 19), - DataType.StructType.Field.from("SCOPE_TABLE", DataType.Primitives.NULLABLE_STRING.type(), 20), - DataType.StructType.Field.from("SOURCE_DATA_TYPE", DataType.Primitives.NULLABLE_INTEGER.type(), 21), - DataType.StructType.Field.from("IS_AUTOINCREMENT", DataType.Primitives.NULLABLE_STRING.type(), 22), - DataType.StructType.Field.from("IS_GENERATEDCOLUMN", DataType.Primitives.NULLABLE_STRING.type(), 23) - ), true); - return new IteratorResultSet(RelationalStructMetaData.of(columnsStructType), columnDefs.iterator(), 0); - }); + if (schema == null || schema.isEmpty()) { + return conn.runIsolatedInTransactionIfPossible(() -> { + final List columnDefs = new ArrayList<>(); + for (final String schemaName : listAllSchemaNames(database)) { + columnDefs.addAll(loadColumnsForTable(database, schemaName, tablePattern)); + } + return buildColumnsResultSet(columnDefs); + }); + } + return conn.runIsolatedInTransactionIfPossible(() -> buildColumnsResultSet(loadColumnsForTable(database, schema, tablePattern))); } // note that approximate is ignored @@ -283,66 +236,132 @@ public RelationalResultSet getIndexInfo(String database, String schema, String t throw new SQLFeatureNotSupportedException("Cannot scan across Databases yet", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); } if (schema == null || schema.isEmpty()) { - throw new SQLFeatureNotSupportedException("Cannot scan across Schemas yet", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); + return conn.runIsolatedInTransactionIfPossible(() -> { + final List indexDefs = new ArrayList<>(); + for (final String schemaName : listAllSchemaNames(database)) { + indexDefs.addAll(loadIndexInfoForTable(database, schemaName, tablePattern)); + } + return buildIndexInfoResultSet(indexDefs); + }); } if (tablePattern == null || tablePattern.isEmpty()) { throw new SQLFeatureNotSupportedException("Table must be specified", ErrorCode.UNSUPPORTED_OPERATION.getErrorCode()); } - return conn.runIsolatedInTransactionIfPossible(() -> { - RecordMetaData rmd = RecordMetaData.build(loadSchemaMetadata(database, schema)); - //verify that it is in fact a table - List indexDefs; - try { - final RecordType recordType = rmd.getRecordType(tablePattern); - final List indexes = recordType.getIndexes(); - indexDefs = indexes.stream() - .map(index -> { - Object[] row = { - database, - schema, - recordType.getName(), - index.isUnique(), - index.getType(), - index.getName(), - DatabaseMetaData.tableIndexOther, //default value--create our own for different index types? - -1, - null, - null, // TODO(bfines) get asc/desc order from options maybe? - -1, - -1, - null //TODO(bfines) filter condition? SQL supports index filters? - }; - return new ArrayRow(row); - }) - .collect(Collectors.toList()); - } catch (MetaDataException mde) { - throw new RelationalException("table <" + tablePattern + "> does not exist", ErrorCode.UNDEFINED_TABLE); - } + return conn.runIsolatedInTransactionIfPossible(() -> buildIndexInfoResultSet(loadIndexInfoForTable(database, schema, tablePattern))); + } - final var indexInfoStructType = DataType.StructType.from("PRIMARY_KEYS", List.of( - DataType.StructType.Field.from("TABLE_CAT", DataType.Primitives.NULLABLE_STRING.type(), 0), - DataType.StructType.Field.from("TABLE_SCHEM", DataType.Primitives.NULLABLE_STRING.type(), 1), - DataType.StructType.Field.from("TABLE_NAME", DataType.Primitives.NULLABLE_STRING.type(), 2), - DataType.StructType.Field.from("NON_UNIQUE", DataType.Primitives.NULLABLE_BOOLEAN.type(), 3), - DataType.StructType.Field.from("INDEX_QUALIFIER", DataType.Primitives.NULLABLE_STRING.type(), 4), - DataType.StructType.Field.from("INDEX_NAME", DataType.Primitives.NULLABLE_STRING.type(), 5), - DataType.StructType.Field.from("TYPE", DataType.Primitives.NULLABLE_STRING.type(), 6), - DataType.StructType.Field.from("ORDINAL_POSITION", DataType.Primitives.NULLABLE_INTEGER.type(), 7), - DataType.StructType.Field.from("COLUMN_NAME", DataType.Primitives.NULLABLE_STRING.type(), 8), - DataType.StructType.Field.from("ASC_OR_DESC", DataType.Primitives.NULLABLE_STRING.type(), 9), - DataType.StructType.Field.from("CARDINALITY", DataType.Primitives.NULLABLE_INTEGER.type(), 10), - DataType.StructType.Field.from("PAGES", DataType.Primitives.NULLABLE_INTEGER.type(), 11), - DataType.StructType.Field.from("FILTER_CONDITION", DataType.Primitives.NULLABLE_STRING.type(), 12) - ), true); - return new IteratorResultSet(RelationalStructMetaData.of(indexInfoStructType), indexDefs.iterator(), 0); - }); + @Nonnull + private List loadColumnsForTable(@Nonnull String database, @Nonnull String schema, @Nonnull String tableName) throws RelationalException { + //TODO(bfines) this is a weird way of doing this, is there a better way? + final RecordMetaData rmd = new CatalogMetaDataProvider(this.catalog, URI.create(database), schema, conn.getTransaction()).getRecordMetaData(); + final Descriptors.Descriptor tableDescriptor; + try { + rmd.getRecordType(tableName); + tableDescriptor = rmd.getRecordsDescriptor().findMessageTypeByName(tableName); + } catch (MetaDataException mde) { + return List.of(); + } + return tableDescriptor.getFields().stream() + .map(field -> new ArrayRow(new Object[]{ + database, schema, tableName, field.getName(), + ProtobufDdlUtil.getSqlType(field), ProtobufDdlUtil.getTypeName(field), + -1, 0, null, null, + DatabaseMetaData.columnNullableUnknown, + null, + field.getJavaType() != Descriptors.FieldDescriptor.JavaType.MESSAGE ? field.getDefaultValue() : null, + -1, -1, -1, field.getIndex() + 1, + "YES", null, null, null, null, "NO", "NO" + })) + .collect(Collectors.toList()); + } + + @Nonnull + private IteratorResultSet buildColumnsResultSet(@Nonnull List columnDefs) { + final var columnsStructType = DataType.StructType.from("PRIMARY_KEYS", List.of( + DataType.StructType.Field.from("TABLE_CAT", DataType.Primitives.NULLABLE_STRING.type(), 0), + DataType.StructType.Field.from("TABLE_SCHEM", DataType.Primitives.NULLABLE_STRING.type(), 1), + DataType.StructType.Field.from("TABLE_NAME", DataType.Primitives.NULLABLE_STRING.type(), 2), + DataType.StructType.Field.from("COLUMN_NAME", DataType.Primitives.NULLABLE_STRING.type(), 3), + DataType.StructType.Field.from("DATA_TYPE", DataType.Primitives.NULLABLE_STRING.type(), 4), + DataType.StructType.Field.from("TYPE_NAME", DataType.Primitives.NULLABLE_STRING.type(), 5), + DataType.StructType.Field.from("COLUMN_SIZE", DataType.Primitives.NULLABLE_INTEGER.type(), 6), + DataType.StructType.Field.from("BUFFER_LENGTH", DataType.Primitives.NULLABLE_INTEGER.type(), 7), + DataType.StructType.Field.from("DECIMAL_DIGITS", DataType.Primitives.NULLABLE_INTEGER.type(), 8), + DataType.StructType.Field.from("NUM_PREC_RADIX", DataType.Primitives.NULLABLE_INTEGER.type(), 9), + DataType.StructType.Field.from("NULLABLE", DataType.Primitives.NULLABLE_INTEGER.type(), 10), + DataType.StructType.Field.from("REMARKS", DataType.Primitives.NULLABLE_STRING.type(), 11), + DataType.StructType.Field.from("COLUMN_DEF", DataType.Primitives.NULLABLE_STRING.type(), 12), + DataType.StructType.Field.from("SQL_DATA_TYPE", DataType.Primitives.NULLABLE_INTEGER.type(), 13), + DataType.StructType.Field.from("SQL_DATETIME_SUB", DataType.Primitives.NULLABLE_INTEGER.type(), 14), + DataType.StructType.Field.from("CHAR_OCTET_LENGTH", DataType.Primitives.NULLABLE_INTEGER.type(), 15), + DataType.StructType.Field.from("ORDINAL_POSITION", DataType.Primitives.NULLABLE_INTEGER.type(), 16), + DataType.StructType.Field.from("IS_NULLABLE", DataType.Primitives.NULLABLE_STRING.type(), 17), + DataType.StructType.Field.from("SCOPE_CATALOG", DataType.Primitives.NULLABLE_STRING.type(), 18), + DataType.StructType.Field.from("SCOPE_SCHEMA", DataType.Primitives.NULLABLE_STRING.type(), 19), + DataType.StructType.Field.from("SCOPE_TABLE", DataType.Primitives.NULLABLE_STRING.type(), 20), + DataType.StructType.Field.from("SOURCE_DATA_TYPE", DataType.Primitives.NULLABLE_INTEGER.type(), 21), + DataType.StructType.Field.from("IS_AUTOINCREMENT", DataType.Primitives.NULLABLE_STRING.type(), 22), + DataType.StructType.Field.from("IS_GENERATEDCOLUMN", DataType.Primitives.NULLABLE_STRING.type(), 23) + ), true); + return new IteratorResultSet(RelationalStructMetaData.of(columnsStructType), columnDefs.iterator(), 0); + } + + @Nonnull + private List loadIndexInfoForTable(@Nonnull String database, @Nonnull String schema, @Nonnull String tableName) throws RelationalException { + final RecordMetaData rmd = RecordMetaData.build(loadSchemaMetadata(database, schema)); + try { + final RecordType recordType = rmd.getRecordType(tableName); + return recordType.getIndexes().stream() + .map(index -> new ArrayRow(new Object[]{ + database, schema, recordType.getName(), + index.isUnique(), index.getType(), index.getName(), + DatabaseMetaData.tableIndexOther, + -1, null, null, -1, -1, null + })) + .collect(Collectors.toList()); + } catch (MetaDataException mde) { + return List.of(); + } + } + + @Nonnull + private IteratorResultSet buildIndexInfoResultSet(@Nonnull List indexDefs) { + final var indexInfoStructType = DataType.StructType.from("PRIMARY_KEYS", List.of( + DataType.StructType.Field.from("TABLE_CAT", DataType.Primitives.NULLABLE_STRING.type(), 0), + DataType.StructType.Field.from("TABLE_SCHEM", DataType.Primitives.NULLABLE_STRING.type(), 1), + DataType.StructType.Field.from("TABLE_NAME", DataType.Primitives.NULLABLE_STRING.type(), 2), + DataType.StructType.Field.from("NON_UNIQUE", DataType.Primitives.NULLABLE_BOOLEAN.type(), 3), + DataType.StructType.Field.from("INDEX_QUALIFIER", DataType.Primitives.NULLABLE_STRING.type(), 4), + DataType.StructType.Field.from("INDEX_NAME", DataType.Primitives.NULLABLE_STRING.type(), 5), + DataType.StructType.Field.from("TYPE", DataType.Primitives.NULLABLE_STRING.type(), 6), + DataType.StructType.Field.from("ORDINAL_POSITION", DataType.Primitives.NULLABLE_INTEGER.type(), 7), + DataType.StructType.Field.from("COLUMN_NAME", DataType.Primitives.NULLABLE_STRING.type(), 8), + DataType.StructType.Field.from("ASC_OR_DESC", DataType.Primitives.NULLABLE_STRING.type(), 9), + DataType.StructType.Field.from("CARDINALITY", DataType.Primitives.NULLABLE_INTEGER.type(), 10), + DataType.StructType.Field.from("PAGES", DataType.Primitives.NULLABLE_INTEGER.type(), 11), + DataType.StructType.Field.from("FILTER_CONDITION", DataType.Primitives.NULLABLE_STRING.type(), 12) + ), true); + return new IteratorResultSet(RelationalStructMetaData.of(indexInfoStructType), indexDefs.iterator(), 0); } @Nonnull private RecordMetaDataProto.MetaData loadSchemaMetadata(@Nonnull final String database, @Nonnull final String schema) throws RelationalException { final var recLayerSchema = this.catalog.loadSchema(conn.getTransaction(), URI.create(database), schema); Assert.thatUnchecked(recLayerSchema instanceof RecordLayerSchema); - return (recLayerSchema.getSchemaTemplate().unwrap(RecordLayerSchemaTemplate.class).toRecordMetadata().toProto()); + return recLayerSchema.getSchemaTemplate().unwrap(RecordLayerSchemaTemplate.class).toRecordMetadata().toProto(); + } + + @Nonnull + private List listAllSchemaNames(@Nonnull String database) throws RelationalException { + final List schemaNames = new ArrayList<>(); + try (RelationalResultSet rrs = catalog.listSchemas(conn.getTransaction(), URI.create(database), ContinuationImpl.BEGIN)) { + while (rrs.next()) { + schemaNames.add(rrs.getString("SCHEMA_NAME")); + } + } catch (SQLException sqle) { + throw new RelationalException(sqle); + } + return schemaNames; } //the position in the array is the key sequence, the value is the name of the column diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java index d73bf12d1f..e9e0210b8c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java @@ -283,15 +283,21 @@ public static LogicalOperator generateTableAccess(@Nonnull Identifier tableId, new AccessHints(indexAccessHints.toArray(new AccessHint[0]))))); final var table = semanticAnalyzer.getTable(tableId); Type.Record type = Assert.castUnchecked(table, RecordLayerTable.class).getType(); + // For secondary schemas, namespace the proto type name to avoid TypeRepository collisions + // when two schemas define identically-named tables. The scan key (FDB record type name) + // must stay as the original table name. + final String scanKey = type.getStorageName(); + if (!schemaId.isCurrentSchema()) { + type = type.withName(schemaId.getSchemaName() + "." + type.getName()); + } if (effectiveTemplate.isStoreRowVersions()) { // Ideally, the RecordLayerTable would have the full type with all the pseudo-fields, // but we need star expansion to skip over these fields. That would be made easier // if we fully supported invisible columns (see: https://github.com/FoundationDB/fdb-record-layer/pull/3787) type = type.addPseudoFields(); } - final String storageName = type.getStorageName(); - Assert.thatUnchecked(storageName != null, "storage name for table access must not be null"); - final var typeFilterExpression = LogicalTypeFilterExpression.of(ImmutableSet.of(storageName), scanExpression, type, schemaId); + Assert.thatUnchecked(scanKey != null, "storage name for table access must not be null"); + final var typeFilterExpression = LogicalTypeFilterExpression.of(ImmutableSet.of(scanKey), scanExpression, type, schemaId); final var resultingQuantifier = Quantifier.forEach(Reference.initialOf(typeFilterExpression)); final ImmutableList.Builder attributesBuilder = ImmutableList.builder(); int colCount = 0; From 3116088a7956359fb933426c21fa12461a12f518 Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Tue, 26 May 2026 00:07:28 +0100 Subject: [PATCH 10/14] Add integration tests for cross-schema join and type-namespace collision avoidance CrossSchemaJoinTest exercises basic cross-schema table access, predicate pushdown, three-schema joins, and continuation support via RecordQueryStoreBindingPlan proto serialization. CrossSchemaTypeCollisionTest verifies that identically-named tables from different schemas do not collide in the TypeRepository when queried together. --- .../query/CrossSchemaJoinTest.java | 79 +++++++++++ .../query/CrossSchemaTypeCollisionTest.java | 131 ++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaTypeCollisionTest.java diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaJoinTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaJoinTest.java index 643094a9ea..9da08d8588 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaJoinTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaJoinTest.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.relational.recordlayer.query; +import com.apple.foundationdb.relational.api.Continuation; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalExtension; import com.apple.foundationdb.relational.recordlayer.RelationalConnectionRule; @@ -30,6 +31,7 @@ import com.apple.foundationdb.relational.utils.SchemaTemplateRule; import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; @@ -38,7 +40,9 @@ import java.net.URI; import java.sql.DriverManager; import java.sql.SQLException; +import java.util.HashSet; import java.util.Map; +import java.util.Set; public class CrossSchemaJoinTest { @@ -50,6 +54,11 @@ public class CrossSchemaJoinTest { "CREATE TABLE TAGS (item_id BIGINT, tag STRING, PRIMARY KEY(item_id))"; private static final String SECONDARY_SCHEMA_NAME = "SECONDARY_SCHEMA"; + private static final String TERTIARY_TEMPLATE_NAME = "CrossSchemaJoinTest_TERTIARY_TEMPLATE"; + private static final String TERTIARY_TEMPLATE_DEF = + "CREATE TABLE PRICES (item_id BIGINT, price BIGINT, PRIMARY KEY(item_id))"; + private static final String TERTIARY_SCHEMA_NAME = "TERTIARY_SCHEMA"; + @RegisterExtension @Order(0) public final EmbeddedRelationalExtension relationalExtension = new EmbeddedRelationalExtension(); @@ -77,6 +86,16 @@ public class CrossSchemaJoinTest { @Order(5) public final RelationalStatementRule statement = new RelationalStatementRule(connection); + @RegisterExtension + @Order(6) + public final SchemaTemplateRule tertiaryTemplateRule = new SchemaTemplateRule( + TERTIARY_TEMPLATE_NAME, Options.none(), null, TERTIARY_TEMPLATE_DEF); + + @RegisterExtension + @Order(7) + public final SchemaRule tertiarySchemaRule = new SchemaRule( + TERTIARY_SCHEMA_NAME, URI.create("/TEST/CrossSchemaJoinTest"), TERTIARY_TEMPLATE_NAME, Options.none()); + public CrossSchemaJoinTest() throws SQLException { } @@ -90,6 +109,12 @@ void setup() throws Exception { stmt.execute("INSERT INTO TAGS VALUES (1, 'fruit'), (2, 'yellow'), (3, 'red')"); } } + try (var tertiaryConn = DriverManager.getConnection(db.getConnectionUri().toString())) { + tertiaryConn.setSchema(TERTIARY_SCHEMA_NAME); + try (var stmt = tertiaryConn.createStatement()) { + stmt.execute("INSERT INTO PRICES VALUES (1, 100), (2, 200), (3, 300)"); + } + } } @Test @@ -125,4 +150,58 @@ void queryFromSecondarySchemaTableAlone() throws Exception { .hasNoNextRow(); } } + + @Test + void innerJoinThreeSchemas() throws Exception { + try (var rs = statement.executeQuery( + "SELECT a.id, a.name, b.tag, c.price FROM ITEMS AS a" + + " JOIN " + SECONDARY_SCHEMA_NAME + ".TAGS AS b ON a.id = b.item_id" + + " JOIN " + TERTIARY_SCHEMA_NAME + ".PRICES AS c ON a.id = c.item_id" + + " ORDER BY a.id")) { + ResultSetAssert.assertThat(rs) + .hasNextRow().hasColumns(Map.of("id", 1L, "name", "Apple", "tag", "fruit", "price", 100L)) + .hasNextRow().hasColumns(Map.of("id", 2L, "name", "Banana", "tag", "yellow", "price", 200L)) + .hasNextRow().hasColumns(Map.of("id", 3L, "name", "Cherry", "tag", "red", "price", 300L)) + .hasNoNextRow(); + } + } + + @Test + void innerJoinWithContinuation() throws Exception { + final Continuation continuation; + statement.setMaxRows(2); + try (var rs = statement.executeQuery( + "SELECT a.id, a.name, b.tag FROM ITEMS AS a JOIN " + SECONDARY_SCHEMA_NAME + ".TAGS AS b ON a.id = b.item_id ORDER BY a.id")) { + ResultSetAssert.assertThat(rs) + .hasNextRow().hasColumns(Map.of("id", 1L, "name", "Apple", "tag", "fruit")) + .hasNextRow().hasColumns(Map.of("id", 2L, "name", "Banana", "tag", "yellow")) + .hasNoNextRow(); + continuation = rs.getContinuation(); + } + Assertions.assertThat(continuation.atEnd()).isFalse(); + + try (var ps = connection.prepareStatement("EXECUTE CONTINUATION ?c")) { + ps.setBytes("c", continuation.serialize()); + try (var rs = ps.executeQuery()) { + ResultSetAssert.assertThat(rs) + .hasNextRow().hasColumns(Map.of("id", 3L, "name", "Cherry", "tag", "red")) + .hasNoNextRow(); + } + } + } + + @Test + void crossSchemaCatalogGetTables() throws Exception { + final String database = db.getDatabasePath().getPath(); + final Set tables = new HashSet<>(); + try (var rs = connection.getMetaData().getTables(database, null, null, null)) { + while (rs.next()) { + tables.add(rs.getString("TABLE_SCHEM") + "." + rs.getString("TABLE_NAME")); + } + } + Assertions.assertThat(tables) + .contains(db.getSchemaName() + ".ITEMS") + .contains(SECONDARY_SCHEMA_NAME + ".TAGS") + .contains(TERTIARY_SCHEMA_NAME + ".PRICES"); + } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaTypeCollisionTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaTypeCollisionTest.java new file mode 100644 index 0000000000..4b916d0ea0 --- /dev/null +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/CrossSchemaTypeCollisionTest.java @@ -0,0 +1,131 @@ +/* + * CrossSchemaTypeCollisionTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.query; + +import com.apple.foundationdb.relational.api.Options; +import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalExtension; +import com.apple.foundationdb.relational.recordlayer.RelationalConnectionRule; +import com.apple.foundationdb.relational.recordlayer.RelationalStatementRule; +import com.apple.foundationdb.relational.recordlayer.Utils; +import com.apple.foundationdb.relational.utils.ResultSetAssert; +import com.apple.foundationdb.relational.utils.SchemaRule; +import com.apple.foundationdb.relational.utils.SchemaTemplateRule; +import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.net.URI; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Map; + +/** + * Tests that two schemas defining a table with the same name but different columns do not + * cause TypeRepository collisions. The type namespacing fix in LogicalOperator prefixes the + * proto type name with the schema name (e.g. "SECONDARY.THINGS" vs "THINGS") so that + * TypeRepository can distinguish them. + */ +public class CrossSchemaTypeCollisionTest { + + // Primary schema: THINGS(id, description) — note: different columns from secondary + private static final String PRIMARY_TEMPLATE_DEF = + "CREATE TABLE THINGS (id BIGINT, description STRING, PRIMARY KEY(id))"; + + // Secondary schema: also named THINGS, but with a different column (quantity instead of description) + private static final String SECONDARY_TEMPLATE_NAME = "CrossSchemaTypeCollisionTest_SECONDARY_TEMPLATE"; + private static final String SECONDARY_TEMPLATE_DEF = + "CREATE TABLE THINGS (id BIGINT, quantity BIGINT, PRIMARY KEY(id))"; + private static final String SECONDARY_SCHEMA_NAME = "SECONDARY_SCHEMA"; + + @RegisterExtension + @Order(0) + public final EmbeddedRelationalExtension relationalExtension = new EmbeddedRelationalExtension(); + + @RegisterExtension + @Order(1) + public final SimpleDatabaseRule db = new SimpleDatabaseRule(CrossSchemaTypeCollisionTest.class, PRIMARY_TEMPLATE_DEF); + + @RegisterExtension + @Order(2) + public final SchemaTemplateRule secondaryTemplateRule = new SchemaTemplateRule( + SECONDARY_TEMPLATE_NAME, Options.none(), null, SECONDARY_TEMPLATE_DEF); + + @RegisterExtension + @Order(3) + public final SchemaRule secondarySchemaRule = new SchemaRule( + SECONDARY_SCHEMA_NAME, URI.create("/TEST/CrossSchemaTypeCollisionTest"), SECONDARY_TEMPLATE_NAME, Options.none()); + + @RegisterExtension + @Order(4) + public final RelationalConnectionRule connection = new RelationalConnectionRule(db::getConnectionUri) + .withSchema(db.getSchemaName()); + + @RegisterExtension + @Order(5) + public final RelationalStatementRule statement = new RelationalStatementRule(connection); + + public CrossSchemaTypeCollisionTest() throws SQLException { + } + + @BeforeEach + void setup() throws Exception { + Utils.enableCascadesDebugger(); + statement.execute("INSERT INTO THINGS VALUES (1, 'Widget'), (2, 'Gadget'), (3, 'Doohickey')"); + try (var secondaryConn = DriverManager.getConnection(db.getConnectionUri().toString())) { + secondaryConn.setSchema(SECONDARY_SCHEMA_NAME); + try (var stmt = secondaryConn.createStatement()) { + stmt.execute("INSERT INTO THINGS VALUES (1, 10), (2, 20), (3, 30)"); + } + } + } + + @Test + void joinSameNamedTablesFromDifferentSchemas() throws Exception { + // Both schemas define THINGS but with different columns. + // Without type namespacing, TypeRepository would throw an IllegalArgumentException + // because two Type.Record objects with different fields but the same storageName ("THINGS") + // cannot both be registered. + try (var rs = statement.executeQuery( + "SELECT a.description, b.quantity FROM THINGS AS a" + + " JOIN " + SECONDARY_SCHEMA_NAME + ".THINGS AS b ON a.id = b.id ORDER BY a.id")) { + ResultSetAssert.assertThat(rs) + .hasNextRow().hasColumns(Map.of("description", "Widget", "quantity", 10L)) + .hasNextRow().hasColumns(Map.of("description", "Gadget", "quantity", 20L)) + .hasNextRow().hasColumns(Map.of("description", "Doohickey", "quantity", 30L)) + .hasNoNextRow(); + } + } + + @Test + void querySameNamedTableInSecondarySchemaAlone() throws Exception { + try (var rs = statement.executeQuery( + "SELECT quantity FROM " + SECONDARY_SCHEMA_NAME + ".THINGS ORDER BY id")) { + ResultSetAssert.assertThat(rs) + .hasNextRow().hasColumns(Map.of("quantity", 10L)) + .hasNextRow().hasColumns(Map.of("quantity", 20L)) + .hasNextRow().hasColumns(Map.of("quantity", 30L)) + .hasNoNextRow(); + } + } +} From 582f228ea0a40db3ad3d52da8d01447b9f6d9d11 Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Tue, 26 May 2026 09:08:02 +0100 Subject: [PATCH 11/14] Fix redundant import in RelationalPlanCacheTests --- .../recordlayer/query/cache/RelationalPlanCacheTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java index 9b8ff69384..91bd650ce9 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/RelationalPlanCacheTests.java @@ -53,7 +53,6 @@ import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; import com.apple.foundationdb.relational.utils.TestSchemas; import com.apple.test.BooleanSource; -import com.apple.foundationdb.relational.recordlayer.query.cache.PlanCacheSchemaKey; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSortedMap; import com.google.common.testing.FakeTicker; From 0fac65fb686f7d671759a0ec0fb3b8d57ff7f4db Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Tue, 26 May 2026 09:37:32 +0100 Subject: [PATCH 12/14] Fix PMD and checkstyle violations in cross-schema classes --- .../record/query/plan/plans/RecordQueryStoreBindingPlan.java | 1 + .../foundationdb/relational/recordlayer/query/PlanContext.java | 1 - .../foundationdb/relational/recordlayer/query/QueryPlan.java | 1 - .../relational/recordlayer/query/SemanticAnalyzer.java | 2 +- 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java index 2b6436b7d5..f487e80b81 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryStoreBindingPlan.java @@ -172,6 +172,7 @@ public int computeHashCodeWithoutChildren() { } @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") public boolean equalsWithoutChildren(@Nonnull final RelationalExpression otherExpression, @Nonnull final AliasMap equivalencesMap) { if (this == otherExpression) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanContext.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanContext.java index 0934625ea6..2b16db1068 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanContext.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanContext.java @@ -40,7 +40,6 @@ import javax.annotation.Nonnull; import java.net.URI; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java index 95530439bb..d13290e707 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java @@ -35,7 +35,6 @@ import com.apple.foundationdb.record.query.plan.cascades.CascadesPlanner; import com.apple.foundationdb.record.query.plan.cascades.PlannerPhase; import com.apple.foundationdb.record.query.plan.cascades.Reference; -import com.apple.foundationdb.record.query.plan.cascades.SchemaIdentifier; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStoreBindingPlan; import com.apple.foundationdb.record.query.plan.cascades.events.ExecutingTaskPlannerEvent; import com.apple.foundationdb.record.query.plan.cascades.events.InsertIntoMemoPlannerEvent; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 2962884f77..289e6e3a1d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -128,7 +128,7 @@ public class SemanticAnalyzer { private java.util.function.Function> secondarySchemaLookup; @Nonnull - private final LinkedHashMap loadedSecondarySchemas; + private final Map loadedSecondarySchemas; public SemanticAnalyzer(@Nonnull SchemaTemplate metadataCatalog, @Nonnull SqlFunctionCatalog functionCatalog, From b53d98628054f41e932d2eec9a55ebce09cdb7d8 Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Tue, 26 May 2026 09:46:38 +0100 Subject: [PATCH 13/14] Fix unused import in CatalogMetaData --- .../foundationdb/relational/recordlayer/CatalogMetaData.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java index 55a49d7c6e..82728ecf82 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java @@ -24,7 +24,6 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; -import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.MetaDataException; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.relational.api.RelationalDatabaseMetaData; From f0c1db0dd478fd0f6fc8e4fded581bb6200b4fde Mon Sep 17 00:00:00 2001 From: Arnaud Lacurie Date: Tue, 26 May 2026 09:51:54 +0100 Subject: [PATCH 14/14] Add unit tests for PlanCacheSchemaKey.of(Collection) and toString --- .../query/cache/PlanCacheSchemaKeyTest.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/PlanCacheSchemaKeyTest.java diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/PlanCacheSchemaKeyTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/PlanCacheSchemaKeyTest.java new file mode 100644 index 0000000000..66debd3a4d --- /dev/null +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/cache/PlanCacheSchemaKeyTest.java @@ -0,0 +1,71 @@ +/* + * PlanCacheSchemaKeyTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.query.cache; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +class PlanCacheSchemaKeyTest { + + @Test + void ofCollectionProducesSortedKey() { + final List names = Arrays.asList("schema_b", "schema_a", "schema_c"); + final PlanCacheSchemaKey key = PlanCacheSchemaKey.of(names); + assertEquals(List.of("schema_a", "schema_b", "schema_c"), List.copyOf(key.getSchemaNames())); + } + + @Test + void ofCollectionEqualsSingletonOfForSingleElement() { + final PlanCacheSchemaKey fromCollection = PlanCacheSchemaKey.of(List.of("my_schema")); + final PlanCacheSchemaKey fromSingle = PlanCacheSchemaKey.of("my_schema"); + assertEquals(fromSingle, fromCollection); + assertEquals(fromSingle.hashCode(), fromCollection.hashCode()); + } + + @Test + void ofCollectionOrderIndependent() { + final PlanCacheSchemaKey k1 = PlanCacheSchemaKey.of(Arrays.asList("alpha", "beta")); + final PlanCacheSchemaKey k2 = PlanCacheSchemaKey.of(Arrays.asList("beta", "alpha")); + assertEquals(k1, k2); + assertEquals(k1.hashCode(), k2.hashCode()); + } + + @Test + void toStringSingleSchema() { + assertEquals("my_schema", PlanCacheSchemaKey.of("my_schema").toString()); + } + + @Test + void toStringMultiSchemaJoinedWithPipe() { + final PlanCacheSchemaKey key = PlanCacheSchemaKey.of(Arrays.asList("schema_b", "schema_a")); + assertEquals("schema_a|schema_b", key.toString()); + } + + @Test + void differentSchemasDontMatch() { + assertNotEquals(PlanCacheSchemaKey.of("schema_x"), PlanCacheSchemaKey.of("schema_y")); + } +}