From ad4fee9114de1382230b69722051c3d2c3397f17 Mon Sep 17 00:00:00 2001 From: mck Date: Tue, 15 Sep 2026 22:30:32 +0200 Subject: [PATCH 1/4] Support integration tests after Transactional Cluster Metadata Select topology-change hooks for legacy Cassandra and Transactional Cluster Metadata. Create schema before supported transitions and configure three metadata replicas on Cassandra 6.0 and later. Retain single-node success and failure scenarios, including multi-datacenter coverage, across supported versions. Preserve concurrent joins and leaves and cluster-doubling/halving scenarios for Cassandra older than 6.0, with their required range-movement settings scoped to those tests and restored during teardown. Exercise concurrent TCM joins and leaves with Sidecar's CASSSIDECAR-277 disjoint token placement, including a join in an unreplicated datacenter. Preserve the existing consistency-level and per-node data checks. Wait for joining and replacement startup before shutdown, require successful joins to reach Normal, and await decommission results before validating writes. Inject terminal leave failures and verify the expected error. Add completion regression tests and raise the integration JVM datagram-socket limit for multi-node tests. patch by Mick Semb Wever for CASSANALYTICS-112 Assisted-by: Codex:GPT-6 --- DEV-README.md | 26 +-- .../distributed/impl/CassandraCluster.java | 2 +- .../SharedClusterIntegrationTestBase.java | 8 + .../testing/ClusterBuilderConfiguration.java | 14 ++ .../apache/cassandra/testing/TestUtils.java | 7 - .../build.gradle | 2 + .../analytics/ResiliencyTestBase.java | 115 ++++++++++--- .../analytics/TopologyChangeBBUtils.java | 79 +++++++++ .../analytics/TopologyChangeBBUtilsTest.java | 101 ++++++++++++ .../expansion/JoiningCompletionTest.java | 71 ++++++++ ...ingDisjointRangesSingleReplicatedTest.java | 64 ++++++++ .../expansion/JoiningDisjointRangesTest.java | 104 ++++++++++++ .../JoiningDoubleClusterFailureTest.java | 6 + .../expansion/JoiningDoubleClusterTest.java | 6 + .../expansion/JoiningMultiDCFailureTest.java | 45 ++--- ...ingMultiDCSingleReplicatedFailureTest.java | 43 ++--- .../JoiningMultiDCSingleReplicatedTest.java | 36 +--- .../expansion/JoiningMultiDCTest.java | 38 +---- .../expansion/JoiningMultipleNodesTest.java | 6 + .../JoiningSingleNodeFailureTest.java | 32 +--- .../expansion/JoiningSingleNodeTest.java | 25 +-- .../analytics/expansion/JoiningTestBase.java | 76 ++++++--- .../LegacyJoiningMultiDCFailureTest.java | 105 ++++++++++++ ...ingMultiDCSingleReplicatedFailureTest.java | 105 ++++++++++++ ...acyJoiningMultiDCSingleReplicatedTest.java | 118 ++++++++++++++ .../expansion/LegacyJoiningMultiDCTest.java | 154 ++++++++++++++++++ .../movement/NodeMovementFailureTest.java | 31 +--- .../NodeMovementMultiDCFailureTest.java | 31 +--- .../movement/NodeMovementMultiDCTest.java | 26 +-- .../analytics/movement/NodeMovementTest.java | 26 +-- .../movement/NodeMovementTestBase.java | 9 +- .../HostReplacementFailureTest.java | 32 +--- .../HostReplacementMultiDCFailureTest.java | 27 +-- ...cementMultiDCInsufficientReplicasTest.java | 31 +--- .../HostReplacementMultiDCTest.java | 25 +-- .../replacement/HostReplacementTest.java | 25 +-- .../replacement/HostReplacementTestBase.java | 28 ++-- .../shrink/LeavingCompletionTest.java | 110 +++++++++++++ .../shrink/LeavingDisjointRangesTest.java | 102 ++++++++++++ .../analytics/shrink/LeavingHalfTest.java | 6 + .../shrink/LeavingMultiDCFailureTest.java | 45 ++--- ...LeavingMultiDCHalveClusterFailureTest.java | 11 +- .../LeavingMultiDCHalveClusterTest.java | 6 + .../analytics/shrink/LeavingMultiDCTest.java | 36 +--- .../shrink/LeavingMultipleFailureTest.java | 11 +- .../analytics/shrink/LeavingMultipleTest.java | 6 + .../shrink/LeavingSingleFailureTest.java | 34 +--- .../analytics/shrink/LeavingSingleTest.java | 25 +-- .../analytics/shrink/LeavingTestBase.java | 95 +++++++---- .../LegacyLeavingMultiDCFailureTest.java | 146 +++++++++++++++++ .../shrink/LegacyLeavingMultiDCTest.java | 140 ++++++++++++++++ 51 files changed, 1835 insertions(+), 617 deletions(-) create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/TopologyChangeBBUtils.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/TopologyChangeBBUtilsTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningCompletionTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDisjointRangesSingleReplicatedTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDisjointRangesTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCFailureTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCSingleReplicatedFailureTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCSingleReplicatedTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingCompletionTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingDisjointRangesTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LegacyLeavingMultiDCFailureTest.java create mode 100644 cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LegacyLeavingMultiDCTest.java diff --git a/DEV-README.md b/DEV-README.md index a973174c7..b1844d5ba 100644 --- a/DEV-README.md +++ b/DEV-README.md @@ -125,21 +125,21 @@ Without these entries an upstream DNS server can answer `localhost2` with a publ connects to that address and the test fails with `java.net.ConnectException: Operation timed out` in `CassandraDataLayer.initialize`. -### Topology-change tests skip on Cassandra 6.0 +### Topology-change tests -Twenty-eight test classes under `expansion`, `shrink`, `replacement` and `movement` pause a topology change with a -ByteBuddy hook, then run the bulk writer while the node is in the transitional state. The hooks target -`StorageService.bootstrap(Collection, long)`, `StorageService.unbootstrap()` and `RangeRelocator.stream()`. +The tests under `expansion`, `shrink`, `replacement` and `movement` pause a topology change with ByteBuddy, then run the bulk writer while the node is in the transitional state. `TopologyChangeBBUtils` selects the hook from the classes available in the Cassandra instance classloader and fails immediately if the target method is absent. -CEP-21 Transactional Cluster Metadata removed all three in Cassandra 6.0. The work now belongs to -`org.apache.cassandra.tcm.sequences`: `BootstrapAndJoin.bootstrap(...)` for a join, `BootstrapAndReplace` for a -replacement, `UnbootstrapAndLeave.executeNext()` with `LeaveStreams` for a decommission, and `Move` for a token move. -A hook that fails to install is silent, so each class waited two minutes for a latch that never counted down. +| Operation | Cassandra 4.x / 5.0 | Cassandra 6.0 (Transactional Cluster Metadata) | +| --- | --- | --- | +| Join or replace | `StorageService.bootstrap(Collection, long)` | `BootstrapAndJoin.bootstrap(...)`, also called by replacement | +| Decommission | `StorageService.unbootstrap()` | `UnbootstrapStreams.execute(...)` | +| Move | `RangeRelocator.stream()` | `Move.executeNext()` | -`ResiliencyTestBase.assumeTopologyChangeHooksSupported()` now skips these classes on 6.0 and later. The four base -classes call it from `beforeClusterProvisioning()`. Bulk write during a topology change is therefore untested on -6.0. To close the gap, retarget each hook at the sequence types named above, and keep the 4.0 and 5.0 targets for -the older runs. +The shared single-node scenarios create their schema before starting a topology change. On Cassandra 6.0 and later, they configure three Cluster Metadata Service members before a node can leave or stop. These scenarios also cover multi-datacenter clusters. + +The `JoiningDisjointRangesTest` and `LeavingDisjointRangesTest` scenarios exercise two concurrent transitions on Cassandra 6.0 and later. They use Sidecar's CASSSIDECAR-277 token placement: six tokens per datacenter with the tokens for nodes 6 and 11 exchanged. The single-replicated variant also checks that the joining node in the unreplicated datacenter receives no rows. Successful joins must reach `Normal`; decommission tests await the nodetool result and check success or the injected failure before validating data again. + +Cassandra versions older than 6.0 additionally run the concurrent join/leave and cluster-doubling/halving tests. The `LegacyJoiningMultiDC*` and `LegacyLeavingMultiDC*` tests preserve the original concurrent multi-datacenter scenarios alongside the shared single-node tests. These legacy scenarios retain schema creation during the transition and enable their required range-movement overrides only for the lifetime of the test class. They skip before cluster provisioning on Cassandra 6.0 and later because their concurrent transitions can require overlapping range locks. ## IntelliJ @@ -172,4 +172,4 @@ compiled classes of both earlier majors, and Gradle never recompiles an inherite changed between two majors is silent at build time and fails at runtime with `NoSuchMethodError`. After you add a new major version, compile the union of the inherited sources against the new shaded jar and confirm that the resulting error set matches the error set that the previous major produces. Every error the new version adds names -a file you must copy forward and override. \ No newline at end of file +a file you must copy forward and override. diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/distributed/impl/CassandraCluster.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/distributed/impl/CassandraCluster.java index 69686b900..2f07e48d3 100644 --- a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/distributed/impl/CassandraCluster.java +++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/distributed/impl/CassandraCluster.java @@ -134,7 +134,7 @@ public AbstractCluster initializeCluster(String versionString, { instanceConfigUpdater = instanceConfigUpdater.andThen(config -> configuration.additionalInstanceConfig.forEach(config::set)); } - clusterBuilder.withTokenSupplier(tokenSupplier) + clusterBuilder.withTokenSupplier(configuration.tokenSupplier == null ? tokenSupplier : configuration.tokenSupplier) .withConfig(instanceConfigUpdater); if (dcCount > 1) diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/SharedClusterIntegrationTestBase.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/SharedClusterIntegrationTestBase.java index e4b867f31..e43f6c545 100644 --- a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/SharedClusterIntegrationTestBase.java +++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/SharedClusterIntegrationTestBase.java @@ -190,6 +190,7 @@ protected void setup() throws Exception assertThat(cluster).isNotNull(); afterClusterProvisioned(); initializeSchemaForTest(); + afterSchemaInitialized(); mtlsTestHelper = new MtlsTestHelper(secretsPath); startSidecar(cluster); beforeTestStart(); @@ -309,6 +310,13 @@ protected void afterClusterProvisioned() */ protected abstract void initializeSchemaForTest(); + /** + * Runs after schema creation, before Sidecar starts. + */ + protected void afterSchemaInitialized() + { + } + /** * Override to perform an action before the tests start */ diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/ClusterBuilderConfiguration.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/ClusterBuilderConfiguration.java index 4a0d64488..c012f0086 100644 --- a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/ClusterBuilderConfiguration.java +++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/ClusterBuilderConfiguration.java @@ -27,6 +27,7 @@ import com.google.common.base.Preconditions; import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.NetworkTopology; /** @@ -44,6 +45,7 @@ public class ClusterBuilderConfiguration public String partitioner; public Map additionalInstanceConfig = null; public int tokenCount = 1; + public TokenSupplier tokenSupplier; public IntFunction dcAndRackSupplier; /** @@ -180,6 +182,18 @@ public ClusterBuilderConfiguration tokenCount(int tokenCount) return this; } + /** + * Overrides the default token allocation for topology-change scenarios. + * + * @param tokenSupplier the token allocation for existing and joining nodes + * @return this configuration instance + */ + public ClusterBuilderConfiguration tokenSupplier(TokenSupplier tokenSupplier) + { + this.tokenSupplier = tokenSupplier; + return this; + } + /** * Sets a supplier function that provides datacenter and rack information for each node in the cluster. * diff --git a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/TestUtils.java b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/TestUtils.java index a233d377f..9fe4f5f6b 100644 --- a/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/TestUtils.java +++ b/cassandra-analytics-integration-framework/src/main/java/org/apache/cassandra/testing/TestUtils.java @@ -107,8 +107,6 @@ public static void configureDefaultDTestJarProperties() { // Settings to reduce the test setup delay incurred if gossip is enabled System.setProperty("cassandra.ring_delay_ms", "5000"); // down from 30s default - System.setProperty("cassandra.consistent.rangemovement", "false"); - System.setProperty("cassandra.consistent.simultaneousmoves.allow", "true"); // End gossip delay settings // Set the location of dtest jars System.setProperty("cassandra.test.dtest_jar_path", System.getProperty("cassandra.test.dtest_jar_path", "dependencies")); @@ -125,11 +123,6 @@ public static void configureDefaultDTestJarProperties() System.setProperty("cassandra.require_native_file_hints", "true"); // Disable all native stuff in Netty as streaming isn't functional with native enabled System.setProperty("shaded.io.netty.transport.noNative", "true"); - // Lifted from the Simulation runner (we're running into similar errors): - // this property is used to allow non-members of the ring to exist in gossip without breaking RF changes - // it would be nice not to rely on this, but hopefully we'll have consistent range movements before it matters - System.setProperty("cassandra.allow_alter_rf_during_range_movement", "true"); - System.setProperty("cassandra.minimum_replication_factor", "1"); } diff --git a/cassandra-analytics-integration-tests/build.gradle b/cassandra-analytics-integration-tests/build.gradle index 38cff0681..644522a77 100644 --- a/cassandra-analytics-integration-tests/build.gradle +++ b/cassandra-analytics-integration-tests/build.gradle @@ -100,6 +100,8 @@ def configureIntegrationTestTask = { Test task, String majorMinor = null -> // use testCassandra40/testCassandra41 for targeted runs. CI covers all versions via CASSANDRA_VERSION. task.systemProperty "cassandra.sidecar.versions_to_test", "5.0" } + // Dtest installs a SecurityManager; its default limit of 25 UDP sockets is too small for multi-node tests. + task.systemProperty "sun.net.maxDatagramSockets", "1024" task.systemProperty "SKIP_STARTUP_VALIDATIONS", "true" task.systemProperty "logback.configurationFile", "src/test/resources/logback-test.xml" task.systemProperty "cassandra.integration.sidecar.test.enable_mtls", integrationEnableMtls diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/ResiliencyTestBase.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/ResiliencyTestBase.java index 3e1dd81d3..2137ab7dd 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/ResiliencyTestBase.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/ResiliencyTestBase.java @@ -28,12 +28,18 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import com.google.common.collect.Range; +import org.junit.jupiter.api.AfterAll; + import org.apache.cassandra.bridge.CassandraVersion; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.ICluster; @@ -41,6 +47,7 @@ import org.apache.cassandra.distributed.api.IInstanceConfig; import org.apache.cassandra.distributed.api.Row; import org.apache.cassandra.distributed.api.SimpleQueryResult; +import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.sidecar.common.server.JmxClient; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.spark.bulkwriter.DecoratedKey; @@ -48,11 +55,13 @@ import org.apache.cassandra.spark.common.schema.ColumnType; import org.apache.cassandra.spark.common.schema.ColumnTypes; import org.apache.cassandra.testing.ClusterBuilderConfiguration; +import org.apache.cassandra.testing.TestTokenSupplier; +import org.apache.cassandra.testing.utils.WithProperties; import scala.Tuple2; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.junit.jupiter.api.Assumptions.assumeFalse; /** * Base class for resiliency tests. Contains helper methods for data generation and validation @@ -61,35 +70,91 @@ public abstract class ResiliencyTestBase extends SharedClusterSparkIntegrationTe { public static final String QUERY_ALL_ROWS = "SELECT * FROM %s"; + private final WithProperties topologyChangeProperties = new WithProperties(); + /** - * Skips the test class when the version under test declares none of the internals that the ByteBuddy hooks of a - * topology-change test intercept: {@code StorageService.bootstrap(Collection, long)}, - * {@code StorageService.unbootstrap()} and {@code RangeRelocator.stream()}. CEP-21 Transactional Cluster Metadata - * removed all three in Cassandra 6.0, in favour of {@code org.apache.cassandra.tcm.sequences}. A hook that fails - * to install is silent, so the test would instead wait on a latch that never counts down. - * - *

CASSANALYTICS-112 tracks the port of these hooks to {@code BootstrapAndJoin.bootstrap}, - * {@code UnbootstrapStreams.execute} and {@code Move.executeNext}, and the decision on the tests that move - * several nodes at once, which Transactional Cluster Metadata no longer permits.

- * - *

Call this from {@link #beforeClusterProvisioning()}, which runs before the cluster starts.

+ * Legacy scenarios that need multiple nodes to remain in transition at once. */ - protected void assumeTopologyChangeHooksSupported() + protected boolean requiresConcurrentTopologyChanges() + { + return false; + } + + @Override + protected void beforeClusterProvisioning() + { + super.beforeClusterProvisioning(); + if (requiresConcurrentTopologyChanges()) + { + assumeFalse(usesTcm(), "Concurrent topology scenarios require Cassandra older than 6.0"); + // Preserve the permissive configuration used by the legacy concurrent scenarios. + topologyChangeProperties.with("cassandra.consistent.rangemovement", "false", + "cassandra.consistent.simultaneousmoves.allow", "true", + "cassandra.allow_alter_rf_during_range_movement", "true"); + } + } + + @Override + @AfterAll + protected void tearDown() throws Exception + { + try + { + super.tearDown(); + } + finally + { + topologyChangeProperties.close(); + } + } + + protected boolean usesTcm() + { + return CassandraVersion.fromVersion(testVersion.version()) + .orElseThrow(() -> new IllegalStateException("Unsupported Cassandra version: " + testVersion)) + .versionNumber() >= CassandraVersion.SIXZERO.versionNumber(); + } + + protected void prepareTopologyChange() + { + if (usesTcm()) + { + // Keep a metadata quorum when a test stops or replaces a node. + cluster.get(1).nodetoolResult("cms", "reconfigure", "3").asserts().success(); + } + } + + protected static TokenSupplier disjointMultiDcTokens() + { + TokenSupplier tokens = TestTokenSupplier.evenlyDistributedTokens(6, 0, 2, 1); + // Match Sidecar CASSSIDECAR-277's swap(5, 10), whose indices are zero-based. + // Nodes 11 and 12 can then join or leave concurrently without overlapping range locks. + return node -> tokens.tokens(node == 6 ? 11 : node == 11 ? 6 : node); + } + + protected static T awaitTopologyChange(Future task, boolean failureExpected) { - String version = testVersion.version(); - CassandraVersion underTest = CassandraVersion.fromVersion(version) - .orElseThrow(() -> new IllegalStateException( - "Unsupported Cassandra version for topology-change tests: " + version)); - boolean supported = underTest.versionNumber() < CassandraVersion.SIXZERO.versionNumber(); - if (!supported) + try + { + return task.get(2, TimeUnit.MINUTES); + } + catch (ExecutionException exception) + { + if (!failureExpected) + { + throw new AssertionError("Topology change failed", exception.getCause()); + } + return null; + } + catch (InterruptedException exception) + { + Thread.currentThread().interrupt(); + throw new AssertionError("Interrupted waiting for topology change", exception); + } + catch (TimeoutException exception) { - // An aborted @BeforeAll produces no test event, so Gradle reports nothing - logger.warn("Skipping {}: a topology-change test intercepts Cassandra internals that CEP-21 removed " - + "in 6.0, and the test version is {}", getClass().getSimpleName(), version); + throw new AssertionError("Topology change did not finish", exception); } - assumeTrue(supported, - "A topology-change test intercepts Cassandra internals that CEP-21 removed in 6.0, " - + "but the test version is " + version); } public Set getDataForRange(Range range, int rowCount) diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/TopologyChangeBBUtils.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/TopologyChangeBBUtils.java new file mode 100644 index 000000000..3bdb10c4a --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/TopologyChangeBBUtils.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics; + +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.description.method.MethodDescription; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.dynamic.TypeResolutionStrategy; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.matcher.ElementMatcher; +import net.bytebuddy.pool.TypePool; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +/** + * Installs topology-change hooks in the Cassandra instance classloader, before or after Transactional Cluster Metadata (TCM). + */ +public final class TopologyChangeBBUtils +{ + private TopologyChangeBBUtils() + { + } + + public static void installBootstrap(ClassLoader loader, Class interceptor) + { + install(loader, interceptor, "org.apache.cassandra.tcm.sequences.BootstrapAndJoin", named("bootstrap"), + "org.apache.cassandra.service.StorageService", named("bootstrap").and(takesArguments(2))); + } + + public static void installLeaving(ClassLoader loader, Class interceptor) + { + install(loader, interceptor, "org.apache.cassandra.tcm.sequences.UnbootstrapStreams", named("execute"), + "org.apache.cassandra.service.StorageService", named("unbootstrap")); + } + + public static void installMoving(ClassLoader loader, Class interceptor) + { + install(loader, interceptor, "org.apache.cassandra.tcm.sequences.Move", named("executeNext"), + "org.apache.cassandra.service.RangeRelocator", named("stream")); + } + + private static void install(ClassLoader loader, Class interceptor, + String tcmClass, ElementMatcher.Junction tcmMethod, + String legacyClass, ElementMatcher.Junction legacyMethod) + { + TypePool pool = TypePool.Default.of(loader); + TypePool.Resolution tcm = pool.describe(tcmClass); + TypeDescription type = tcm.isResolved() ? tcm.resolve() : pool.describe(legacyClass).resolve(); + ElementMatcher.Junction method = tcm.isResolved() ? tcmMethod : legacyMethod; + if (type.getDeclaredMethods().filter(method).isEmpty()) + { + throw new IllegalStateException("Missing topology-change interception point in " + type.getName()); + } + new ByteBuddy().rebase(type, ClassFileLocator.ForClassLoader.of(loader)) + .method(method) + .intercept(MethodDelegation.to(interceptor)) + .make(TypeResolutionStrategy.Lazy.INSTANCE, pool) + .load(loader, ClassLoadingStrategy.Default.INJECTION); + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/TopologyChangeBBUtilsTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/TopologyChangeBBUtilsTest.java new file mode 100644 index 000000000..dc26188ca --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/TopologyChangeBBUtilsTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics; + +import java.net.URLClassLoader; +import java.util.concurrent.Callable; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import com.vdurmont.semver4j.Semver; +import net.bytebuddy.implementation.bind.annotation.RuntimeType; +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import org.apache.cassandra.distributed.shared.Versions; +import org.apache.cassandra.testing.TestVersion; +import org.apache.cassandra.testing.TestVersionSupplier; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Checks hook binding against the actual dtest jar without starting a cluster. + */ +class TopologyChangeBBUtilsTest +{ + @ParameterizedTest + @ValueSource(strings = {"bootstrap", "leaving", "moving"}) + void installsHookInDtestJar(String operation) throws Exception + { + TestVersion version = TestVersionSupplier.testVersions().findFirst().orElseThrow(); + Versions.Version dtest = Versions.find().getLatest(new Semver(version.version(), Semver.SemverType.LOOSE)); + try (URLClassLoader loader = new URLClassLoader(dtest.classpath, getClass().getClassLoader())) + { + String target; + boolean tcm = loader.getResource("org/apache/cassandra/tcm/sequences/BootstrapAndJoin.class") != null; + switch (operation) + { + case "bootstrap": + TopologyChangeBBUtils.installBootstrap(loader, BootstrapInterceptor.class); + target = tcm ? "org.apache.cassandra.tcm.sequences.BootstrapAndJoin" + : "org.apache.cassandra.service.StorageService"; + break; + case "leaving": + TopologyChangeBBUtils.installLeaving(loader, LeavingInterceptor.class); + target = tcm ? "org.apache.cassandra.tcm.sequences.UnbootstrapStreams" + : "org.apache.cassandra.service.StorageService"; + break; + case "moving": + TopologyChangeBBUtils.installMoving(loader, MovingInterceptor.class); + target = tcm ? "org.apache.cassandra.tcm.sequences.Move" + : "org.apache.cassandra.service.RangeRelocator"; + break; + default: + throw new IllegalArgumentException(operation); + } + Class intercepted = Class.forName(target, false, loader); + assertThat(intercepted.getClassLoader()).isSameAs(loader); + assertThat(intercepted.getDeclaredMethods()).anyMatch(method -> method.getName().contains("$original$")); + } + } + + public static class BootstrapInterceptor + { + public static boolean bootstrap(@SuperCall Callable original) throws Exception + { + return original.call(); + } + } + + public static class LeavingInterceptor + { + public static void unbootstrap(@SuperCall Callable original) throws Exception + { + original.call(); + } + } + + public static class MovingInterceptor + { + @RuntimeType + public static Object stream(@SuperCall Callable original) throws Exception + { + return original.call(); + } + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningCompletionTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningCompletionTest.java new file mode 100644 index 000000000..14f0264af --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningCompletionTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.expansion; + +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; + +import org.apache.cassandra.distributed.api.IInstance; +import org.apache.cassandra.testing.IClusterExtension; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class JoiningCompletionTest +{ + @Test + void rejectsAJoinThatHasNotReachedNormal() + { + IClusterExtension cluster = mock(IClusterExtension.class); + IInstance seed = mock(IInstance.class); + IInstance joining = mock(IInstance.class); + when(cluster.get(1)).thenReturn(seed); + doThrow(new AssertionError("Node is still Joining")).when(cluster).awaitRingState(seed, joining, "Normal"); + + Scenario scenario = new Scenario(cluster, joining); + assertThatThrownBy(() -> scenario.completeTransitionsAndValidateWrites(new CountDownLatch(0), Stream.empty(), false)) + .isInstanceOf(AssertionError.class) + .hasMessage("Node is still Joining"); + } + + private static class Scenario extends JoiningTestBase + { + Scenario(IClusterExtension cluster, IInstance joining) + { + this.cluster = cluster; + newInstances = Collections.singletonList(joining); + } + + @Override + protected void initializeSchemaForTest() + { + } + + @Override + protected CountDownLatch transitioningStateStart() + { + return new CountDownLatch(0); + } + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDisjointRangesSingleReplicatedTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDisjointRangesSingleReplicatedTest.java new file mode 100644 index 000000000..b362c3c8f --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDisjointRangesSingleReplicatedTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.expansion; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.cassandra.sidecar.testing.QualifiedName; +import org.apache.cassandra.testing.ClusterBuilderConfiguration; + +import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; +import static org.apache.cassandra.testing.TestUtils.DC1_RF3; +import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The concurrent join in the unreplicated datacenter must receive no bulk-written rows. + */ +class JoiningDisjointRangesSingleReplicatedTest extends JoiningDisjointRangesTest +{ + @Override + protected void initializeSchemaForTest() + { + createTestKeyspace(TEST_KEYSPACE, DC1_RF3); + multiDCTestInputs().forEach(arguments -> { + QualifiedName table = uniqueTestTableFullName(TEST_KEYSPACE, arguments.get()); + createTestTable(table, CREATE_TEST_TABLE_STATEMENT); + }); + } + + @Override + protected ClusterBuilderConfiguration testClusterConfiguration() + { + ClusterBuilderConfiguration configuration = super.testClusterConfiguration(); + Map settings = new HashMap<>(configuration.additionalInstanceConfig); + // As in Sidecar's single-replicated-DC test, an empty DC cannot acknowledge EACH_QUORUM. + settings.put("progress_barrier_default_consistency_level", "QUORUM"); + return configuration.additionalInstanceConfig(settings); + } + + @Override + protected void beforeTestStart() + { + super.beforeTestStart(); + assertThat(expectedInstanceData.get(cluster.get(11))).isNotEmpty(); + assertThat(expectedInstanceData.get(cluster.get(12))).isEmpty(); + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDisjointRangesTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDisjointRangesTest.java new file mode 100644 index 000000000..57f2058ab --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDisjointRangesTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.expansion; + +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.testing.ClusterBuilderConfiguration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Bulk writes during concurrent TCM joins with Sidecar's CASSSIDECAR-277 token placement. + */ +class JoiningDisjointRangesTest extends JoiningMultiDCTest +{ + @Override + protected void beforeClusterProvisioning() + { + super.beforeClusterProvisioning(); + assumeTrue(usesTcm(), "Disjoint range locks require TCM"); + } + + @Override + protected ClusterBuilderConfiguration testClusterConfiguration() + { + return clusterConfig().nodesPerDc(5) + .newNodesPerDc(1) + .dcCount(2) + .tokenSupplier(disjointMultiDcTokens()) + .requestFeature(Feature.NETWORK) + .instanceInitializer(BBHelper::install); + } + + @Override + protected int joiningDatacenters() + { + return 2; + } + + @Override + protected CountDownLatch transitioningStateStart() + { + return BBHelper.transitioningStateStart; + } + + @Override + protected void beforeTestStart() + { + super.beforeTestStart(); + assertThat(newInstances).extracting(instance -> instance.config().localDatacenter()) + .containsExactly("datacenter1", "datacenter2"); + } + + @Override + protected void beforeClusterShutdown() + { + completeTransitionsAndValidateWrites(BBHelper.transitioningStateEnd, multiDCTestInputs(), false); + } + + public static class BBHelper + { + static final CountDownLatch transitioningStateStart = new CountDownLatch(2); + static final CountDownLatch transitioningStateEnd = new CountDownLatch(1); + + public static void install(ClassLoader loader, Integer nodeNumber) + { + if (nodeNumber == 11 || nodeNumber == 12) + { + TopologyChangeBBUtils.installBootstrap(loader, BBHelper.class); + } + } + + public static boolean bootstrap(@SuperCall Callable original) throws Exception + { + boolean result = original.call(); + transitioningStateStart.countDown(); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); + return result; + } + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDoubleClusterFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDoubleClusterFailureTest.java index dd3c42cf9..dd2f9abb9 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDoubleClusterFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDoubleClusterFailureTest.java @@ -45,6 +45,12 @@ */ class JoiningDoubleClusterFailureTest extends JoiningSingleNodeTest { + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + @Override protected void beforeClusterShutdown() { diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDoubleClusterTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDoubleClusterTest.java index 0ac7b2aa0..9f1cc0495 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDoubleClusterTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningDoubleClusterTest.java @@ -44,6 +44,12 @@ */ class JoiningDoubleClusterTest extends JoiningSingleNodeTest { + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + @Override protected void beforeClusterShutdown() { diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCFailureTest.java index 2a7cdbeff..79ea5c02e 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCFailureTest.java @@ -18,26 +18,16 @@ package org.apache.cassandra.analytics.expansion; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.Uninterruptibles; - -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; class JoiningMultiDCFailureTest extends JoiningMultiDCTest { @@ -53,7 +43,7 @@ protected void beforeClusterShutdown() protected ClusterBuilderConfiguration testClusterConfiguration() { return clusterConfig().nodesPerDc(3) - .newNodesPerDc(3) + .newNodesPerDc(1) .dcCount(2) .requestFeature(Feature.NETWORK) .instanceInitializer(BBHelperMultiDCFailure::install); @@ -66,39 +56,28 @@ protected CountDownLatch transitioningStateStart() } /** - * ByteBuddy helper for multiple joining nodes failure scenario in multiDC + * ByteBuddy helper for a joining node failure scenario in multiDC */ public static class BBHelperMultiDCFailure { - static final CountDownLatch transitioningStateStart = new CountDownLatch(6); - static final CountDownLatch transitioningStateEnd = new CountDownLatch(6); + static final CountDownLatch transitioningStateStart = new CountDownLatch(1); + static final CountDownLatch transitioningStateEnd = new CountDownLatch(1); public static void install(ClassLoader cl, Integer nodeNumber) { - // Test case involves doubling the size of a 6 node cluster (3 per DC) - // We intercept the bootstrap of nodes (7-12) to validate token ranges - if (nodeNumber > 6) + // Intercept the joining node in datacenter1. + if (nodeNumber == 7) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperMultiDCFailure.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperMultiDCFailure.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { - boolean result = orig.call(); + orig.call(); // trigger bootstrap start and wait until bootstrap is ready from test transitioningStateStart.countDown(); - Uninterruptibles.awaitUninterruptibly(transitioningStateEnd, 2, TimeUnit.MINUTES); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); throw new UnsupportedOperationException("Simulated failure"); } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCSingleReplicatedFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCSingleReplicatedFailureTest.java index a05b16d28..a9f66ad16 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCSingleReplicatedFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCSingleReplicatedFailureTest.java @@ -18,26 +18,16 @@ package org.apache.cassandra.analytics.expansion; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.Uninterruptibles; - -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; class JoiningMultiDCSingleReplicatedFailureTest extends JoiningMultiDCSingleReplicatedTest { @@ -66,39 +56,28 @@ protected CountDownLatch transitioningStateStart() } /** - * ByteBuddy helper for multiple joining nodes failure + * ByteBuddy helper for a joining node failure */ public static class BBHelperMultiDCFailure { - static final CountDownLatch transitioningStateStart = new CountDownLatch(2); - static final CountDownLatch transitioningStateEnd = new CountDownLatch(2); + static final CountDownLatch transitioningStateStart = new CountDownLatch(1); + static final CountDownLatch transitioningStateEnd = new CountDownLatch(1); public static void install(ClassLoader cl, Integer nodeNumber) { - // Test case involves adding 2 nodes to a 10 node cluster (5 per DC) - // We intercept the bootstrap of nodes (11,12) to validate token ranges - if (nodeNumber > 10) + // Intercept the joining node in datacenter1. + if (nodeNumber == 11) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperMultiDCFailure.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperMultiDCFailure.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { - boolean result = orig.call(); + orig.call(); // trigger bootstrap start and wait until bootstrap is ready from test transitioningStateStart.countDown(); - Uninterruptibles.awaitUninterruptibly(transitioningStateEnd, 2, TimeUnit.MINUTES); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); throw new IllegalStateException("Unable to contact any seeds: "); } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCSingleReplicatedTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCSingleReplicatedTest.java index 172a44357..98d0525c4 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCSingleReplicatedTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCSingleReplicatedTest.java @@ -18,26 +18,17 @@ package org.apache.cassandra.analytics.expansion; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; @@ -79,34 +70,23 @@ protected CountDownLatch transitioningStateStart() } /** - * ByteBuddy helper for multiple joining nodes + * ByteBuddy helper for a joining node */ public static class BBHelperMultiDC { - static final CountDownLatch transitioningStateStart = new CountDownLatch(2); - static final CountDownLatch transitioningStateEnd = new CountDownLatch(2); + static final CountDownLatch transitioningStateStart = new CountDownLatch(1); + static final CountDownLatch transitioningStateEnd = new CountDownLatch(1); public static void install(ClassLoader cl, Integer nodeNumber) { - // Test case involves adding 2 nodes to a 10 node cluster (5 per DC) - // We intercept the bootstrap of nodes (11,12) to validate token ranges - if (nodeNumber > 10) + // Intercept the joining node in datacenter1. + if (nodeNumber == 11) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperMultiDC.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperMultiDC.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { boolean result = orig.call(); // trigger bootstrap start and wait until bootstrap is ready from test diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCTest.java index 8695203db..d1fdf2301 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultiDCTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.analytics.expansion; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -28,16 +27,10 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.testing.ClusterBuilderConfiguration; @@ -47,8 +40,6 @@ import static org.apache.cassandra.distributed.api.ConsistencyLevel.LOCAL_QUORUM; import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE; import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3_DC2_RF3; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; @@ -84,7 +75,7 @@ protected void beforeClusterShutdown() protected ClusterBuilderConfiguration testClusterConfiguration() { return clusterConfig().nodesPerDc(3) - .newNodesPerDc(3) + .newNodesPerDc(1) .dcCount(2) .requestFeature(Feature.NETWORK) .instanceInitializer(BBHelperMultiDC::install); @@ -109,34 +100,23 @@ static Stream multiDCTestInputs() } /** - * ByteBuddy helper for multiple joining nodes + * ByteBuddy helper for a joining node */ public static class BBHelperMultiDC { - static final CountDownLatch transitioningStateStart = new CountDownLatch(6); - static final CountDownLatch transitioningStateEnd = new CountDownLatch(6); + static final CountDownLatch transitioningStateStart = new CountDownLatch(1); + static final CountDownLatch transitioningStateEnd = new CountDownLatch(1); public static void install(ClassLoader cl, Integer nodeNumber) { - // Test case involves doubling the size of a 6 node cluster (3 per DC) - // We intercept the bootstrap of nodes (7-12) to validate token ranges - if (nodeNumber > 6) + // Intercept the joining node in datacenter1. + if (nodeNumber == 7) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperMultiDC.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperMultiDC.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { boolean result = orig.call(); // trigger bootstrap start and wait until bootstrap is ready from test diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultipleNodesTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultipleNodesTest.java index 00173bedf..5e1b57359 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultipleNodesTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningMultipleNodesTest.java @@ -43,6 +43,12 @@ */ class JoiningMultipleNodesTest extends JoiningSingleNodeTest { + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + @Override protected void beforeClusterShutdown() { diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningSingleNodeFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningSingleNodeFailureTest.java index 036cad93b..407e93ea7 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningSingleNodeFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningSingleNodeFailureTest.java @@ -18,25 +18,15 @@ package org.apache.cassandra.analytics.expansion; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.Uninterruptibles; - -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; /** * Integration tests to validate bulk writes during a Cassandra instance join operation, where the @@ -78,26 +68,16 @@ public static void install(ClassLoader cl, Integer nodeNumber) // We intercept the bootstrap of the leaving node (4) to validate token ranges if (nodeNumber == 4) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperSingleJoiningNodeFailure.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperSingleJoiningNodeFailure.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { - boolean result = orig.call(); + orig.call(); // trigger bootstrap start and wait until bootstrap is ready from test transitioningStateStart.countDown(); - Uninterruptibles.awaitUninterruptibly(transitioningStateEnd, 2, TimeUnit.MINUTES); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); throw new UnsupportedOperationException("Simulated failure"); } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningSingleNodeTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningSingleNodeTest.java index a74d8d8b9..4840f06d2 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningSingleNodeTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningSingleNodeTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.analytics.expansion; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -28,16 +27,10 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.testing.ClusterBuilderConfiguration; @@ -45,8 +38,6 @@ import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE; import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; @@ -116,21 +107,11 @@ public static void install(ClassLoader cl, Integer nodeNumber) // We intercept the bootstrap of the leaving node (4) to validate token ranges if (nodeNumber == 4) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperSingleJoiningNode.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperSingleJoiningNode.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { boolean result = orig.call(); // trigger bootstrap start and wait until bootstrap is ready from test diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningTestBase.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningTestBase.java index 406a7c4b8..0f43b7fec 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningTestBase.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/JoiningTestBase.java @@ -24,6 +24,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -53,6 +54,7 @@ abstract class JoiningTestBase extends ResiliencyTestBase Dataset df; Map> expectedInstanceData; List newInstances; + private final List> joiningStartups = new ArrayList<>(); protected void runJoiningTestScenario(TestConsistencyLevel cl) { @@ -76,16 +78,37 @@ protected void beforeTestStart() } @Override - protected void beforeClusterProvisioning() + protected void afterClusterProvisioned() { - assumeTopologyChangeHooksSupported(); + if (requiresConcurrentTopologyChanges()) + { + startTopologyChange(); + } } @Override - protected void afterClusterProvisioned() + protected void afterSchemaInitialized() + { + if (!requiresConcurrentTopologyChanges()) + { + startTopologyChange(); + } + } + + private void startTopologyChange() { + prepareTopologyChange(); + newInstances = new ArrayList<>(); ClusterBuilderConfiguration configuration = testClusterConfiguration(); - newInstances = addNewInstances(cluster, configuration.newNodesPerDc, configuration.dcCount); + int nodesPerDc = requiresConcurrentTopologyChanges() ? configuration.newNodesPerDc : 1; + int datacenters = joiningDatacenters(); + for (int i = 0; i < nodesPerDc; i++) + { + for (int dc = 1; dc <= datacenters; dc++) + { + newInstances.add(addJoiningInstance(cluster, cluster.get(dc))); + } + } TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateStart(), 2, TimeUnit.MINUTES); newInstances.forEach(instance -> cluster.awaitRingState(instance, instance, "Joining")); } @@ -100,6 +123,12 @@ protected void completeTransitionsAndValidateWrites(CountDownLatch transitionalS transitionalStateEnd.countDown(); } + joiningStartups.forEach(startup -> awaitTopologyChange(startup, failureExpected)); + if (!failureExpected) + { + newInstances.forEach(instance -> cluster.awaitRingState(cluster.get(1), instance, "Normal")); + } + testInputs.forEach(arguments -> { TestConsistencyLevel cl = (TestConsistencyLevel) arguments.get()[0]; @@ -127,29 +156,24 @@ protected void completeTransitionsAndValidateWrites(CountDownLatch transitionalS */ protected abstract CountDownLatch transitioningStateStart(); - private static List addNewInstances(IClusterExtension cluster, int newNodesPerDc, int numDcs) + protected int joiningDatacenters() { - List newInstances = new ArrayList<>(); - // Go over new nodes and add them once for each DC - for (int i = 0; i < newNodesPerDc; i++) - { - int dcNodeIdx = 1; // Use node 2's DC - for (int dc = 1; dc <= numDcs; dc++) - { - IInstance dcNode = cluster.get(dcNodeIdx++); - IInstance newInstance = cluster.addInstance(dcNode.config().localDatacenter(), - dcNode.config().localRack(), - inst -> { - inst.set("auto_bootstrap", true); - inst.with(Feature.GOSSIP, - Feature.JMX, - Feature.NATIVE_PROTOCOL); - }); - new Thread(() -> newInstance.startup(cluster.delegate())).start(); - newInstances.add(newInstance); - } - } - return newInstances; + return requiresConcurrentTopologyChanges() ? testClusterConfiguration().dcCount : 1; + } + + private IInstance addJoiningInstance(IClusterExtension cluster, IInstance seed) + { + IInstance joining = cluster.addInstance(seed.config().localDatacenter(), + seed.config().localRack(), + config -> { + config.set("auto_bootstrap", true); + config.set("dtest.api.startup.failure_as_shutdown", false); + config.with(Feature.GOSSIP, Feature.JMX, Feature.NATIVE_PROTOCOL); + }); + FutureTask joiningStartup = new FutureTask<>(() -> joining.startup(cluster.delegate()), null); + joiningStartups.add(joiningStartup); + new Thread(joiningStartup, "joining-node-startup").start(); + return joining; } Optional getMatchingInstanceFromRing(IInstance seed, diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCFailureTest.java new file mode 100644 index 000000000..724029c6e --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCFailureTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.expansion; + +import java.util.Collection; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.google.common.util.concurrent.Uninterruptibles; + +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.dynamic.TypeResolutionStrategy; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.testing.ClusterBuilderConfiguration; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +class LegacyJoiningMultiDCFailureTest extends LegacyJoiningMultiDCTest +{ + @Override + protected void beforeClusterShutdown() + { + completeTransitionsAndValidateWrites(BBHelperMultiDCFailure.transitioningStateEnd, + multiDCTestInputs(), + true); + } + + @Override + protected ClusterBuilderConfiguration testClusterConfiguration() + { + return clusterConfig().nodesPerDc(3) + .newNodesPerDc(3) + .dcCount(2) + .requestFeature(Feature.NETWORK) + .instanceInitializer(BBHelperMultiDCFailure::install); + } + + @Override + protected CountDownLatch transitioningStateStart() + { + return BBHelperMultiDCFailure.transitioningStateStart; + } + + /** + * ByteBuddy helper for multiple joining nodes failure scenario in multiDC + */ + public static class BBHelperMultiDCFailure + { + static final CountDownLatch transitioningStateStart = new CountDownLatch(6); + static final CountDownLatch transitioningStateEnd = new CountDownLatch(6); + + public static void install(ClassLoader cl, Integer nodeNumber) + { + // Test case involves doubling the size of a 6 node cluster (3 per DC) + // We intercept the bootstrap of nodes (7-12) to validate token ranges + if (nodeNumber > 6) + { + TypePool typePool = TypePool.Default.of(cl); + TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") + .resolve(); + new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) + .method(named("bootstrap").and(takesArguments(2))) + .intercept(MethodDelegation.to(BBHelperMultiDCFailure.class)) + // Defer class loading until all dependencies are loaded + .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) + .load(cl, ClassLoadingStrategy.Default.INJECTION); + } + } + + public static boolean bootstrap(Collection tokens, + long bootstrapTimeoutMillis, + @SuperCall Callable orig) throws Exception + { + boolean result = orig.call(); + // trigger bootstrap start and wait until bootstrap is ready from test + transitioningStateStart.countDown(); + Uninterruptibles.awaitUninterruptibly(transitioningStateEnd, 2, TimeUnit.MINUTES); + throw new UnsupportedOperationException("Simulated failure"); + } + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCSingleReplicatedFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCSingleReplicatedFailureTest.java new file mode 100644 index 000000000..bbf667154 --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCSingleReplicatedFailureTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.expansion; + +import java.util.Collection; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import com.google.common.util.concurrent.Uninterruptibles; + +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.dynamic.TypeResolutionStrategy; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.testing.ClusterBuilderConfiguration; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +class LegacyJoiningMultiDCSingleReplicatedFailureTest extends LegacyJoiningMultiDCSingleReplicatedTest +{ + @Override + protected void beforeClusterShutdown() + { + completeTransitionsAndValidateWrites(BBHelperMultiDCFailure.transitioningStateEnd, + multiDCTestInputs(), + true); + } + + @Override + protected ClusterBuilderConfiguration testClusterConfiguration() + { + return clusterConfig().nodesPerDc(5) + .newNodesPerDc(1) + .dcCount(2) + .requestFeature(Feature.NETWORK) + .instanceInitializer(BBHelperMultiDCFailure::install); + } + + @Override + protected CountDownLatch transitioningStateStart() + { + return BBHelperMultiDCFailure.transitioningStateStart; + } + + /** + * ByteBuddy helper for multiple joining nodes failure + */ + public static class BBHelperMultiDCFailure + { + static final CountDownLatch transitioningStateStart = new CountDownLatch(2); + static final CountDownLatch transitioningStateEnd = new CountDownLatch(2); + + public static void install(ClassLoader cl, Integer nodeNumber) + { + // Test case involves adding 2 nodes to a 10 node cluster (5 per DC) + // We intercept the bootstrap of nodes (11,12) to validate token ranges + if (nodeNumber > 10) + { + TypePool typePool = TypePool.Default.of(cl); + TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") + .resolve(); + new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) + .method(named("bootstrap").and(takesArguments(2))) + .intercept(MethodDelegation.to(BBHelperMultiDCFailure.class)) + // Defer class loading until all dependencies are loaded + .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) + .load(cl, ClassLoadingStrategy.Default.INJECTION); + } + } + + public static boolean bootstrap(Collection tokens, + long bootstrapTimeoutMillis, + @SuperCall Callable orig) throws Exception + { + boolean result = orig.call(); + // trigger bootstrap start and wait until bootstrap is ready from test + transitioningStateStart.countDown(); + Uninterruptibles.awaitUninterruptibly(transitioningStateEnd, 2, TimeUnit.MINUTES); + throw new IllegalStateException("Unable to contact any seeds: "); + } + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCSingleReplicatedTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCSingleReplicatedTest.java new file mode 100644 index 000000000..162621067 --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCSingleReplicatedTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.expansion; + +import java.util.Collection; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.dynamic.TypeResolutionStrategy; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.sidecar.testing.QualifiedName; +import org.apache.cassandra.testing.ClusterBuilderConfiguration; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; +import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; +import static org.apache.cassandra.testing.TestUtils.DC1_RF3; +import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; + +class LegacyJoiningMultiDCSingleReplicatedTest extends LegacyJoiningMultiDCTest +{ + @Override + protected void initializeSchemaForTest() + { + createTestKeyspace(TEST_KEYSPACE, DC1_RF3); + multiDCTestInputs().forEach(arguments -> { + QualifiedName tableName = uniqueTestTableFullName(TEST_KEYSPACE, arguments.get()); + createTestTable(tableName, CREATE_TEST_TABLE_STATEMENT); + }); + } + + @Override + protected void beforeClusterShutdown() + { + completeTransitionsAndValidateWrites(BBHelperMultiDC.transitioningStateEnd, + multiDCTestInputs(), + false); + } + + @Override + protected ClusterBuilderConfiguration testClusterConfiguration() + { + return clusterConfig().nodesPerDc(5) + .newNodesPerDc(1) + .dcCount(2) + .requestFeature(Feature.NETWORK) + .instanceInitializer(BBHelperMultiDC::install); + } + + @Override + protected CountDownLatch transitioningStateStart() + { + return BBHelperMultiDC.transitioningStateStart; + } + + /** + * ByteBuddy helper for multiple joining nodes + */ + public static class BBHelperMultiDC + { + static final CountDownLatch transitioningStateStart = new CountDownLatch(2); + static final CountDownLatch transitioningStateEnd = new CountDownLatch(2); + + public static void install(ClassLoader cl, Integer nodeNumber) + { + // Test case involves adding 2 nodes to a 10 node cluster (5 per DC) + // We intercept the bootstrap of nodes (11,12) to validate token ranges + if (nodeNumber > 10) + { + TypePool typePool = TypePool.Default.of(cl); + TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") + .resolve(); + new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) + .method(named("bootstrap").and(takesArguments(2))) + .intercept(MethodDelegation.to(BBHelperMultiDC.class)) + // Defer class loading until all dependencies are loaded + .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) + .load(cl, ClassLoadingStrategy.Default.INJECTION); + } + } + + public static boolean bootstrap(Collection tokens, + long bootstrapTimeoutMillis, + @SuperCall Callable orig) throws Exception + { + boolean result = orig.call(); + // trigger bootstrap start and wait until bootstrap is ready from test + transitioningStateStart.countDown(); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); + return result; + } + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCTest.java new file mode 100644 index 000000000..981018cde --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/expansion/LegacyJoiningMultiDCTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.expansion; + +import java.util.Collection; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.dynamic.TypeResolutionStrategy; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestConsistencyLevel; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.sidecar.testing.QualifiedName; +import org.apache.cassandra.testing.ClusterBuilderConfiguration; + +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.EACH_QUORUM; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.LOCAL_QUORUM; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; +import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; +import static org.apache.cassandra.testing.TestUtils.DC1_RF3_DC2_RF3; +import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; + +class LegacyJoiningMultiDCTest extends JoiningTestBase +{ + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + + @ParameterizedTest(name = "{index} => {0}") + @MethodSource("multiDCTestInputs") + void testJoiningNodeInMultiDCTest(TestConsistencyLevel cl) + { + runJoiningTestScenario(cl); + } + + @Override + protected void initializeSchemaForTest() + { + createTestKeyspace(TEST_KEYSPACE, DC1_RF3_DC2_RF3); + multiDCTestInputs().forEach(arguments -> { + QualifiedName tableName = uniqueTestTableFullName(TEST_KEYSPACE, arguments.get()); + createTestTable(tableName, CREATE_TEST_TABLE_STATEMENT); + }); + } + + @Override + protected void beforeClusterShutdown() + { + completeTransitionsAndValidateWrites(BBHelperMultiDC.transitioningStateEnd, + multiDCTestInputs(), + false); + } + + @Override + protected ClusterBuilderConfiguration testClusterConfiguration() + { + return clusterConfig().nodesPerDc(3) + .newNodesPerDc(3) + .dcCount(2) + .requestFeature(Feature.NETWORK) + .instanceInitializer(BBHelperMultiDC::install); + } + + @Override + protected CountDownLatch transitioningStateStart() + { + return BBHelperMultiDC.transitioningStateStart; + } + + static Stream multiDCTestInputs() + { + return Stream.of( + Arguments.of(TestConsistencyLevel.of(ALL, ONE)), + Arguments.of(TestConsistencyLevel.of(LOCAL_QUORUM, LOCAL_QUORUM)), + Arguments.of(TestConsistencyLevel.of(EACH_QUORUM, LOCAL_QUORUM)), + Arguments.of(TestConsistencyLevel.of(QUORUM, QUORUM)), + Arguments.of(TestConsistencyLevel.of(LOCAL_QUORUM, EACH_QUORUM)), + Arguments.of(TestConsistencyLevel.of(ONE, ALL)) + ); + } + + /** + * ByteBuddy helper for multiple joining nodes + */ + public static class BBHelperMultiDC + { + static final CountDownLatch transitioningStateStart = new CountDownLatch(6); + static final CountDownLatch transitioningStateEnd = new CountDownLatch(6); + + public static void install(ClassLoader cl, Integer nodeNumber) + { + // Test case involves doubling the size of a 6 node cluster (3 per DC) + // We intercept the bootstrap of nodes (7-12) to validate token ranges + if (nodeNumber > 6) + { + TypePool typePool = TypePool.Default.of(cl); + TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") + .resolve(); + new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) + .method(named("bootstrap").and(takesArguments(2))) + .intercept(MethodDelegation.to(BBHelperMultiDC.class)) + // Defer class loading until all dependencies are loaded + .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) + .load(cl, ClassLoadingStrategy.Default.INJECTION); + } + } + + public static boolean bootstrap(Collection tokens, + long bootstrapTimeoutMillis, + @SuperCall Callable orig) throws Exception + { + boolean result = orig.call(); + // trigger bootstrap start and wait until bootstrap is ready from test + transitioningStateStart.countDown(); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); + return result; + } + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementFailureTest.java index d5d00f3c2..682282162 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementFailureTest.java @@ -21,22 +21,14 @@ import java.io.IOException; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.Uninterruptibles; - -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.RuntimeType; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; /** * Integration tests to verify bulk writes when a Cassandra's node range is moved and the operation is expected to @@ -76,24 +68,17 @@ public static void install(ClassLoader cl, Integer nodeNumber) // Moving the 5th node in the test case if (nodeNumber == SINGLE_DC_MOVING_NODE_IDX) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.RangeRelocator") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("stream")) - .intercept(MethodDelegation.to(BBHelperMovingNodeFailure.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installMoving(cl, BBHelperMovingNodeFailure.class); } } @SuppressWarnings("unused") - public static Future stream(@SuperCall Callable> orig) throws Exception + @RuntimeType + public static Object stream(@SuperCall Callable orig) throws Exception { - Future res = orig.call(); + orig.call(); transitioningStateStart.countDown(); - Uninterruptibles.awaitUninterruptibly(transitioningStateEnd, 2, TimeUnit.MINUTES); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); throw new IOException("Simulated node movement failures"); // Throws exception to nodetool } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementMultiDCFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementMultiDCFailureTest.java index 3a0441d25..2deb9450f 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementMultiDCFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementMultiDCFailureTest.java @@ -21,22 +21,14 @@ import java.io.IOException; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.Uninterruptibles; - -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.RuntimeType; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; /** * Integration tests to verify bulk writes when a Cassandra's node range is moved in a multi-DC cluster, and the @@ -77,24 +69,17 @@ public static void install(ClassLoader cl, Integer nodeNumber) // Moving the 5th node in the test case if (nodeNumber == MULTI_DC_MOVING_NODE_IDX) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.RangeRelocator") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("stream")) - .intercept(MethodDelegation.to(BBHelperMultiDCMovingNodeFailure.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installMoving(cl, BBHelperMultiDCMovingNodeFailure.class); } } @SuppressWarnings("unused") - public static Future stream(@SuperCall Callable> orig) throws Exception + @RuntimeType + public static Object stream(@SuperCall Callable orig) throws Exception { - Future res = orig.call(); + orig.call(); transitioningStateStart.countDown(); - Uninterruptibles.awaitUninterruptibly(transitioningStateEnd, 2, TimeUnit.MINUTES); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); throw new IOException("Simulated node movement failure"); // Throws exception to nodetool } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementMultiDCTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementMultiDCTest.java index 5819f1676..29c1b6e9a 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementMultiDCTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementMultiDCTest.java @@ -20,7 +20,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -28,16 +27,11 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.RuntimeType; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.testing.ClusterBuilderConfiguration; @@ -47,7 +41,6 @@ import static org.apache.cassandra.distributed.api.ConsistencyLevel.LOCAL_QUORUM; import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE; import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; -import static net.bytebuddy.matcher.ElementMatchers.named; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3_DC2_RF3; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; @@ -120,22 +113,15 @@ public static void install(ClassLoader cl, Integer nodeNumber) // Moving the 5th node in the test case if (nodeNumber == MULTI_DC_MOVING_NODE_IDX) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.RangeRelocator") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("stream")) - .intercept(MethodDelegation.to(BBHelperMovingNodeMultiDC.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installMoving(cl, BBHelperMovingNodeMultiDC.class); } } @SuppressWarnings("unused") - public static Future stream(@SuperCall Callable> orig) throws Exception + @RuntimeType + public static Object stream(@SuperCall Callable orig) throws Exception { - Future res = orig.call(); + Object res = orig.call(); transitioningStateStart.countDown(); TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); return res; diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementTest.java index 9d58b51ca..b11fc7af0 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementTest.java @@ -20,7 +20,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; @@ -28,16 +27,11 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.RuntimeType; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.testing.ClusterBuilderConfiguration; @@ -45,7 +39,6 @@ import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; import static org.apache.cassandra.distributed.api.ConsistencyLevel.ONE; import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; -import static net.bytebuddy.matcher.ElementMatchers.named; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; @@ -115,22 +108,15 @@ public static void install(ClassLoader cl, Integer nodeNumber) // Moving the 5th node in the test case if (nodeNumber == SINGLE_DC_MOVING_NODE_IDX) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.RangeRelocator") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("stream")) - .intercept(MethodDelegation.to(BBHelperMovingNode.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installMoving(cl, BBHelperMovingNode.class); } } @SuppressWarnings("unused") - public static Future stream(@SuperCall Callable> orig) throws Exception + @RuntimeType + public static Object stream(@SuperCall Callable orig) throws Exception { - Future res = orig.call(); + Object res = orig.call(); transitioningStateStart.countDown(); TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); return res; diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementTestBase.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementTestBase.java index be74d1091..30d27fc60 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementTestBase.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/movement/NodeMovementTestBase.java @@ -77,14 +77,9 @@ protected void beforeTestStart() } @Override - protected void beforeClusterProvisioning() - { - assumeTopologyChangeHooksSupported(); - } - - @Override - protected void afterClusterProvisioned() + protected void afterSchemaInitialized() { + prepareTopologyChange(); ClusterBuilderConfiguration configuration = testClusterConfiguration(); int movingNodeIndex = configuration.dcCount > 1 ? MULTI_DC_MOVING_NODE_IDX : SINGLE_DC_MOVING_NODE_IDX; movingNode = cluster.get(movingNodeIndex); diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementFailureTest.java index 305f53f87..ad15ee973 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementFailureTest.java @@ -18,25 +18,15 @@ package org.apache.cassandra.analytics.replacement; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.Uninterruptibles; - -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; /** * Integration tests that verify bulk writes during a host replacement operation in the Cassandra cluster where @@ -80,27 +70,17 @@ public static void install(ClassLoader cl, Integer nodeNumber) // We intercept the bootstrap of the replacement (6th) node to validate token ranges if (nodeNumber == 6) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperReplacementsNodeFailure.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperReplacementsNodeFailure.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { - boolean result = orig.call(); + orig.call(); nodeStart.countDown(); // trigger bootstrap start and wait until bootstrap is ready from test transitioningStateStart.countDown(); - Uninterruptibles.awaitUninterruptibly(transitioningStateEnd, 2, TimeUnit.MINUTES); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); throw new UnsupportedOperationException("Simulated failure"); } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCFailureTest.java index a4f8771de..e724cc6f8 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCFailureTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.analytics.replacement; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -27,24 +26,16 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.provider.Arguments; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.spark.bulkwriter.WriterOptions; import org.apache.cassandra.testing.ClusterBuilderConfiguration; import static org.apache.cassandra.distributed.api.ConsistencyLevel.LOCAL_QUORUM; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF2_DC2_RF2; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; @@ -114,23 +105,13 @@ public static void install(ClassLoader cl, Integer nodeNumber) // We intercept the bootstrap of the replacement (7th) node to validate token ranges if (nodeNumber == 7) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperReplacementFailureMultiDC.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperReplacementFailureMultiDC.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { - boolean result = orig.call(); + orig.call(); nodeStart.countDown(); // trigger bootstrap start and wait until bootstrap is ready from test transitionalStateStart.countDown(); diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCInsufficientReplicasTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCInsufficientReplicasTest.java index 4567035a2..010de51c9 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCInsufficientReplicasTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCInsufficientReplicasTest.java @@ -18,22 +18,15 @@ package org.apache.cassandra.analytics.replacement; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.Uninterruptibles; import org.junit.jupiter.api.Test; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.spark.bulkwriter.WriterOptions; @@ -43,8 +36,6 @@ import org.apache.spark.sql.Row; import static org.apache.cassandra.distributed.api.ConsistencyLevel.EACH_QUORUM; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3_DC2_RF3; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; @@ -119,28 +110,18 @@ public static void install(ClassLoader cl, Integer nodeNumber) // We intercept the bootstrap of the replacement (7th) node to validate token ranges if (nodeNumber == 7) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperNodeReplacementMultiDCInsufficientReplicas.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperNodeReplacementMultiDCInsufficientReplicas.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { - boolean result = orig.call(); + orig.call(); nodeStart.countDown(); // trigger bootstrap start and wait until bootstrap is ready from test transitionalStateStart.countDown(); - Uninterruptibles.awaitUninterruptibly(transitionalStateEnd, 2, TimeUnit.MINUTES); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitionalStateEnd, 2, TimeUnit.MINUTES); throw new UnsupportedOperationException("Simulated failure"); } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCTest.java index 19c902dbe..fcd68c72f 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementMultiDCTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.analytics.replacement; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -28,23 +27,15 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.spark.bulkwriter.WriterOptions; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import static org.apache.cassandra.distributed.api.ConsistencyLevel.EACH_QUORUM; import static org.apache.cassandra.distributed.api.ConsistencyLevel.LOCAL_QUORUM; import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; @@ -126,21 +117,11 @@ public static void install(ClassLoader cl, Integer nodeNumber) // We intercept the bootstrap of the replacement (7th) node to validate token ranges if (nodeNumber == 7) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperNodeReplacementMultiDC.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperNodeReplacementMultiDC.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { boolean result = orig.call(); nodeStart.countDown(); diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementTest.java index cc5fa5e34..77f61705b 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementTest.java @@ -18,7 +18,6 @@ package org.apache.cassandra.analytics.replacement; -import java.util.Collection; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -26,23 +25,15 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.spark.bulkwriter.WriterOptions; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; @@ -110,21 +101,11 @@ public static void install(ClassLoader cl, Integer nodeNumber) // We intercept the bootstrap of the replacement (6th) node to validate token ranges if (nodeNumber == 6) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("bootstrap").and(takesArguments(2))) - .intercept(MethodDelegation.to(BBHelperReplacementsNode.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installBootstrap(cl, BBHelperReplacementsNode.class); } } - public static boolean bootstrap(Collection tokens, - long bootstrapTimeoutMillis, - @SuperCall Callable orig) throws Exception + public static boolean bootstrap(@SuperCall Callable orig) throws Exception { boolean result = orig.call(); nodeStart.countDown(); diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementTestBase.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementTestBase.java index efa07be6c..6934a0f57 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementTestBase.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/replacement/HostReplacementTestBase.java @@ -26,17 +26,18 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.common.util.concurrent.Uninterruptibles; import org.junit.jupiter.params.provider.Arguments; import org.apache.cassandra.analytics.DataGenerationUtils; import org.apache.cassandra.analytics.ResiliencyTestBase; import org.apache.cassandra.analytics.TestConsistencyLevel; +import org.apache.cassandra.analytics.TestUninterruptibles; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.IInstance; import org.apache.cassandra.distributed.api.IInstanceConfig; @@ -60,16 +61,12 @@ abstract class HostReplacementTestBase extends ResiliencyTestBase Map> expectedInstanceData; List newNodes; List removedNodeAddresses; + private final List> replacementStartups = new ArrayList<>(); @Override - protected void beforeClusterProvisioning() - { - assumeTopologyChangeHooksSupported(); - } - - @Override - protected void afterClusterProvisioned() + protected void afterSchemaInitialized() { + prepareTopologyChange(); assertThat(additionalNodesToStop()).isLessThan(cluster.size() - 1); IInstance seed = cluster.get(1); @@ -105,7 +102,10 @@ protected void afterClusterProvisioned() for (IInstance newInstance : newNodes) { cluster.awaitRingState(newInstance, newInstance, "Joining"); - cluster.awaitGossipStatus(newInstance, newInstance, "BOOT_REPLACE"); + if (!usesTcm()) + { + cluster.awaitGossipStatus(newInstance, newInstance, "BOOT_REPLACE"); + } String newAddress = newInstance.config().broadcastAddress().getAddress().getHostAddress(); Optional replacementInstance = getMatchingInstanceFromRing(newInstance, newAddress); @@ -125,6 +125,8 @@ protected void completeTransitionsAndValidateWrites(CountDownLatch transitionalS transitionalStateEnd.countDown(); } + replacementStartups.forEach(startup -> awaitTopologyChange(startup, expectFailure)); + assertThat(newNodes).isNotNull(); assertThat(removedNodeAddresses).isNotNull(); @@ -241,7 +243,7 @@ private List startReplacementNodes(CountDownLatch nodeStart, ICluster }, remPort); - new Thread(() -> ClusterUtils.start(replacement, (properties) -> { + FutureTask startup = new FutureTask<>(() -> ClusterUtils.start(replacement, (properties) -> { replacement.config().set("storage_port", remPort); properties.with("cassandra.skip_schema_check", "true"); properties.with("cassandra.schema_delay_ms", String.valueOf(TimeUnit.SECONDS.toMillis(10L))); @@ -252,9 +254,11 @@ private List startReplacementNodes(CountDownLatch nodeStart, ICluster // This property tells cassandra that this new instance is replacing the node with // address remAddress and port remPort properties.with("cassandra.replace_address_first_boot", remAddress + ":" + remPort); - })).start(); + }), null); + replacementStartups.add(startup); + new Thread(startup, "replacement-node-startup").start(); - Uninterruptibles.awaitUninterruptibly(nodeStart, 2, TimeUnit.MINUTES); + TestUninterruptibles.awaitUninterruptiblyOrThrow(nodeStart, 2, TimeUnit.MINUTES); newNodes.add(replacement); } return newNodes; diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingCompletionTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingCompletionTest.java new file mode 100644 index 000000000..db0d31e1d --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingCompletionTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.shrink; + +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; + +import org.apache.cassandra.distributed.api.IInstance; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.NodeToolResult; +import org.apache.cassandra.testing.IClusterExtension; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class LeavingCompletionTest +{ + @Test + void propagatesDecommissionFailureToTheTestThread() + { + Scenario scenario = new Scenario(result(1, "Unexpected streaming failure")); + assertThatThrownBy(() -> scenario.completeTransitionsAndValidateWrites(new CountDownLatch(0), Stream.empty())) + .isInstanceOf(AssertionError.class); + } + + @Test + void rejectsSuccessWhenDecommissionFailureWasExpected() + { + Scenario scenario = new Scenario(result(0, null)); + assertThatThrownBy(() -> scenario.completeTransitionsAndValidateWrites(new CountDownLatch(0), Stream.empty(), true)) + .isInstanceOf(AssertionError.class); + } + + @Test + void rejectsAnUnrelatedDecommissionFailure() + { + Scenario scenario = new Scenario(result(1, "Unexpected streaming failure")); + assertThatThrownBy(() -> scenario.completeTransitionsAndValidateWrites(new CountDownLatch(0), Stream.empty(), true)) + .isInstanceOf(AssertionError.class); + } + + @Test + void acceptsTheInjectedDecommissionFailure() + { + Scenario scenario = new Scenario(result(1, "Simulated leave failure")); + assertThatCode(() -> scenario.completeTransitionsAndValidateWrites(new CountDownLatch(0), Stream.empty(), true)) + .doesNotThrowAnyException(); + } + + private static NodeToolResult result(int rc, String error) + { + return new NodeToolResult(new String[]{"decommission"}, rc, Collections.emptyList(), + error == null ? null : new RuntimeException(error)); + } + + private static class Scenario extends LeavingTestBase + { + Scenario(NodeToolResult result) + { + IClusterExtension cluster = mock(IClusterExtension.class); + IInstance leaving = mock(IInstance.class); + IInstanceConfig config = mock(IInstanceConfig.class); + when(config.num()).thenReturn(1); + when(leaving.config()).thenReturn(config); + when(cluster.size()).thenReturn(1); + when(cluster.get(1)).thenReturn(leaving); + when(cluster.getFirstRunningInstance()).thenReturn(leaving); + when(leaving.nodetoolResult("decommission")).thenReturn(result); + this.cluster = cluster; + afterSchemaInitialized(); + } + + @Override + protected void prepareTopologyChange() + { + } + + @Override + protected void initializeSchemaForTest() + { + } + + @Override + protected CountDownLatch transitioningStateStart() + { + return new CountDownLatch(0); + } + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingDisjointRangesTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingDisjointRangesTest.java new file mode 100644 index 000000000..3b866e326 --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingDisjointRangesTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.shrink; + +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.testing.ClusterBuilderConfiguration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Bulk writes during concurrent TCM leaves with Sidecar's CASSSIDECAR-277 token placement. + */ +class LeavingDisjointRangesTest extends LeavingMultiDCTest +{ + @Override + protected void beforeClusterProvisioning() + { + super.beforeClusterProvisioning(); + assumeTrue(usesTcm(), "Disjoint range locks require TCM"); + } + + @Override + protected ClusterBuilderConfiguration testClusterConfiguration() + { + return clusterConfig().nodesPerDc(6) + .dcCount(2) + .tokenSupplier(disjointMultiDcTokens()) + .requestFeature(Feature.NETWORK) + .instanceInitializer(BBHelper::install); + } + + @Override + protected int leavingNodeCount() + { + return 2; + } + + @Override + protected CountDownLatch transitioningStateStart() + { + return BBHelper.transitioningStateStart; + } + + @Override + protected void beforeTestStart() + { + super.beforeTestStart(); + assertThat(leavingNodes).extracting(instance -> instance.config().localDatacenter()) + .containsExactlyInAnyOrder("datacenter1", "datacenter2"); + } + + @Override + protected void beforeClusterShutdown() + { + completeTransitionsAndValidateWrites(BBHelper.transitioningStateEnd, multiDCTestInputs()); + } + + public static class BBHelper + { + static final CountDownLatch transitioningStateStart = new CountDownLatch(2); + static final CountDownLatch transitioningStateEnd = new CountDownLatch(1); + + public static void install(ClassLoader loader, Integer nodeNumber) + { + if (nodeNumber == 11 || nodeNumber == 12) + { + TopologyChangeBBUtils.installLeaving(loader, BBHelper.class); + } + } + + public static void unbootstrap(@SuperCall Callable original) throws Exception + { + transitioningStateStart.countDown(); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateEnd, 2, TimeUnit.MINUTES); + original.call(); + } + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingHalfTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingHalfTest.java index 399b2774c..120fc0bcc 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingHalfTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingHalfTest.java @@ -50,6 +50,12 @@ */ class LeavingHalfTest extends LeavingTestBase { + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + @ParameterizedTest(name = "{index} => {0}") @MethodSource("singleDCTestInputs") void halveClusterSize(TestConsistencyLevel cl) diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCFailureTest.java index b7caf3510..c138f71bb 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCFailureTest.java @@ -20,27 +20,21 @@ import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.Uninterruptibles; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.spark.bulkwriter.WriterOptions; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3_DC2_RF3; import static org.apache.cassandra.testing.TestUtils.ROW_COUNT; @@ -74,7 +68,7 @@ protected void initializeSchemaForTest() @Override protected void beforeClusterShutdown() { - completeTransitionsAndValidateWrites(BBHelperLeavingNodesMultiDCFailure.transitionalStateEnd, multiDCTestInputs()); + completeTransitionsAndValidateWrites(BBHelperLeavingNodesMultiDCFailure.transitionalStateEnd, multiDCTestInputs(), true); // For tests that involve LEAVE failures, we validate that the leaving nodes are part of the cluster // check leave node are part of cluster when leave fails @@ -90,12 +84,6 @@ protected ClusterBuilderConfiguration testClusterConfiguration() .instanceInitializer(BBHelperLeavingNodesMultiDCFailure::install); } - @Override - protected int leavingNodesPerDc() - { - return 1; - } - @Override protected CountDownLatch transitioningStateStart() { @@ -103,28 +91,19 @@ protected CountDownLatch transitioningStateStart() } /** - * ByteBuddy helper for multiple leaving nodes multi-DC failure scenario + * ByteBuddy helper for a leaving node in a multi-DC cluster failure scenario */ public static class BBHelperLeavingNodesMultiDCFailure { - static final CountDownLatch transitionalStateStart = new CountDownLatch(2); - static final CountDownLatch transitionalStateEnd = new CountDownLatch(2); + static final CountDownLatch transitionalStateStart = new CountDownLatch(1); + static final CountDownLatch transitionalStateEnd = new CountDownLatch(1); public static void install(ClassLoader cl, Integer nodeNumber) { - // Test case involves 10 node cluster (5 nodes per DC) with a 2 leaving nodes (1 per DC) - // We intercept the shutdown of the leaving nodes (9, 10) to validate token ranges - if (nodeNumber > 8) + // Intercept the last node leaving the two-datacenter cluster. + if (nodeNumber == 10) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("unbootstrap")) - .intercept(MethodDelegation.to(BBHelperLeavingNodesMultiDCFailure.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installLeaving(cl, BBHelperLeavingNodesMultiDCFailure.class); } } @@ -132,8 +111,8 @@ public static void install(ClassLoader cl, Integer nodeNumber) public static void unbootstrap(@SuperCall Callable orig) throws Exception { transitionalStateStart.countDown(); - Uninterruptibles.awaitUninterruptibly(transitionalStateEnd, 2, TimeUnit.MINUTES); - throw new UnsupportedOperationException("Simulate leave failure"); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitionalStateEnd, 2, TimeUnit.MINUTES); + throw new ExecutionException(new UnsupportedOperationException("Simulated leave failure")); } } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCHalveClusterFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCHalveClusterFailureTest.java index d4dbd2854..183750fd9 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCHalveClusterFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCHalveClusterFailureTest.java @@ -20,6 +20,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.junit.jupiter.params.ParameterizedTest; @@ -49,6 +50,12 @@ class LeavingMultiDCHalveClusterFailureTest extends LeavingTestBase { + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + static final int LEAVING_NODES_PER_DC = 3; @ParameterizedTest(name = "{index} => {0}") @@ -76,7 +83,7 @@ protected void initializeSchemaForTest() @Override protected void beforeClusterShutdown() { - completeTransitionsAndValidateWrites(BBHelperHalveClusterMultiDCFailure.transitionalStateEnd, multiDCTestInputs()); + completeTransitionsAndValidateWrites(BBHelperHalveClusterMultiDCFailure.transitionalStateEnd, multiDCTestInputs(), true); // For tests that involve LEAVE failures, we validate that the leaving nodes are part of the cluster // check leave node are part of cluster when leave fails @@ -136,7 +143,7 @@ public static void unbootstrap(@SuperCall Callable orig) throws Exception transitionalStateStart.countDown(); TestUninterruptibles.awaitUninterruptiblyOrThrow(transitionalStateStart, 4, TimeUnit.MINUTES); TestUninterruptibles.awaitUninterruptiblyOrThrow(transitionalStateEnd, 2, TimeUnit.MINUTES); - throw new UnsupportedOperationException("Simulated failure"); + throw new ExecutionException(new UnsupportedOperationException("Simulated leave failure")); } } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCHalveClusterTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCHalveClusterTest.java index 8570695db..996129552 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCHalveClusterTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCHalveClusterTest.java @@ -51,6 +51,12 @@ class LeavingMultiDCHalveClusterTest extends LeavingTestBase { + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + static final int LEAVING_NODES_PER_DC = 3; @ParameterizedTest(name = "{index} => {0}") diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCTest.java index 4b09bd90f..b3b886193 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultiDCTest.java @@ -25,22 +25,15 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.spark.bulkwriter.WriterOptions; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3_DC2_RF3; import static org.apache.cassandra.testing.TestUtils.ROW_COUNT; @@ -85,12 +78,6 @@ protected ClusterBuilderConfiguration testClusterConfiguration() .instanceInitializer(BBHelperLeavingNodesMultiDC::install); } - @Override - protected int leavingNodesPerDc() - { - return 1; - } - @Override protected CountDownLatch transitioningStateStart() { @@ -98,28 +85,19 @@ protected CountDownLatch transitioningStateStart() } /** - * ByteBuddy helper for multiple leaving nodes multi-DC + * ByteBuddy helper for a leaving node in a multi-DC cluster */ public static class BBHelperLeavingNodesMultiDC { - static final CountDownLatch transitionalStateStart = new CountDownLatch(2); - static final CountDownLatch transitionalStateEnd = new CountDownLatch(2); + static final CountDownLatch transitionalStateStart = new CountDownLatch(1); + static final CountDownLatch transitionalStateEnd = new CountDownLatch(1); public static void install(ClassLoader cl, Integer nodeNumber) { - // Test case involves 10 node cluster (5 nodes per DC) with a 2 leaving nodes (1 per DC) - // We intercept the shutdown of the leaving nodes (9, 10) to validate token ranges - if (nodeNumber > 8) + // Intercept the last node leaving the two-datacenter cluster. + if (nodeNumber == 10) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("unbootstrap")) - .intercept(MethodDelegation.to(BBHelperLeavingNodesMultiDC.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installLeaving(cl, BBHelperLeavingNodesMultiDC.class); } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultipleFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultipleFailureTest.java index 2b23cb983..4e4d4e847 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultipleFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultipleFailureTest.java @@ -20,6 +20,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import com.google.common.util.concurrent.Uninterruptibles; @@ -51,6 +52,12 @@ */ class LeavingMultipleFailureTest extends LeavingTestBase { + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + @ParameterizedTest(name = "{index} => {0}") @MethodSource("singleDCTestInputs") void multipleLeavingFailure(TestConsistencyLevel cl) @@ -71,7 +78,7 @@ protected void initializeSchemaForTest() @Override protected void beforeClusterShutdown() { - completeTransitionsAndValidateWrites(BBHelperMultipleLeavingNodesFailure.transitionalStateEnd, singleDCTestInputs()); + completeTransitionsAndValidateWrites(BBHelperMultipleLeavingNodesFailure.transitionalStateEnd, singleDCTestInputs(), true); // For tests that involve LEAVE failures, we validate that the leaving nodes are part of the cluster // check leave node are part of cluster when leave fails @@ -129,7 +136,7 @@ public static void unbootstrap(@SuperCall Callable orig) throws Exception { transitionalStateStart.countDown(); Uninterruptibles.awaitUninterruptibly(transitionalStateEnd, 2, TimeUnit.MINUTES); - throw new UnsupportedOperationException("Simulate leave failure"); + throw new ExecutionException(new UnsupportedOperationException("Simulated leave failure")); } } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultipleTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultipleTest.java index 625ae0eb6..d102a8021 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultipleTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingMultipleTest.java @@ -50,6 +50,12 @@ */ class LeavingMultipleTest extends LeavingTestBase { + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + @ParameterizedTest(name = "{index} => {0}") @MethodSource("singleDCTestInputs") void multipleLeavingNodes(TestConsistencyLevel cl) diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingSingleFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingSingleFailureTest.java index 79240e762..5ee035aba 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingSingleFailureTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingSingleFailureTest.java @@ -20,26 +20,20 @@ import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.Uninterruptibles; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; @@ -71,7 +65,7 @@ protected void initializeSchemaForTest() @Override protected void beforeClusterShutdown() { - completeTransitionsAndValidateWrites(BBHelperSingleLeavingNodeFailure.transitionalStateEnd, singleDCTestInputs()); + completeTransitionsAndValidateWrites(BBHelperSingleLeavingNodeFailure.transitionalStateEnd, singleDCTestInputs(), true); // For tests that involve LEAVE failures, we validate that the leaving nodes are part of the cluster // check leave node are part of cluster when leave fails @@ -86,12 +80,6 @@ protected ClusterBuilderConfiguration testClusterConfiguration() .instanceInitializer(BBHelperSingleLeavingNodeFailure::install); } - @Override - protected int leavingNodesPerDc() - { - return 1; - } - @Override protected CountDownLatch transitioningStateStart() { @@ -112,15 +100,7 @@ public static void install(ClassLoader cl, Integer nodeNumber) // We intercept the shutdown of the leaving node (5) to validate token ranges if (nodeNumber == 5) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("unbootstrap")) - .intercept(MethodDelegation.to(BBHelperSingleLeavingNodeFailure.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installLeaving(cl, BBHelperSingleLeavingNodeFailure.class); } } @@ -128,8 +108,8 @@ public static void install(ClassLoader cl, Integer nodeNumber) public static void unbootstrap(@SuperCall Callable orig) throws Exception { transitionalStateStart.countDown(); - Uninterruptibles.awaitUninterruptibly(transitionalStateEnd, 2, TimeUnit.MINUTES); - throw new UnsupportedOperationException("Simulate leave failure"); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitionalStateEnd, 2, TimeUnit.MINUTES); + throw new ExecutionException(new UnsupportedOperationException("Simulated leave failure")); } } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingSingleTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingSingleTest.java index 64d83d553..1c44edeb0 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingSingleTest.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingSingleTest.java @@ -25,21 +25,14 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.dynamic.ClassFileLocator; -import net.bytebuddy.dynamic.TypeResolutionStrategy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.pool.TypePool; import org.apache.cassandra.analytics.TestConsistencyLevel; import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.analytics.TopologyChangeBBUtils; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.testing.ClusterBuilderConfiguration; -import static net.bytebuddy.matcher.ElementMatchers.named; import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; import static org.apache.cassandra.testing.TestUtils.DC1_RF3; import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; @@ -81,12 +74,6 @@ protected ClusterBuilderConfiguration testClusterConfiguration() .instanceInitializer(BBHelperSingleLeavingNode::install); } - @Override - protected int leavingNodesPerDc() - { - return 1; - } - @Override protected CountDownLatch transitioningStateStart() { @@ -107,15 +94,7 @@ public static void install(ClassLoader cl, Integer nodeNumber) // We intercept the shutdown of the leaving node (5) to validate token ranges if (nodeNumber == 5) { - TypePool typePool = TypePool.Default.of(cl); - TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") - .resolve(); - new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) - .method(named("unbootstrap")) - .intercept(MethodDelegation.to(BBHelperSingleLeavingNode.class)) - // Defer class loading until all dependencies are loaded - .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) - .load(cl, ClassLoadingStrategy.Default.INJECTION); + TopologyChangeBBUtils.installLeaving(cl, BBHelperSingleLeavingNode.class); } } diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingTestBase.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingTestBase.java index db7a8803d..3d6f730b5 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingTestBase.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LeavingTestBase.java @@ -23,25 +23,22 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; import org.junit.jupiter.params.provider.Arguments; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.cassandra.analytics.DataGenerationUtils; import org.apache.cassandra.analytics.ResiliencyTestBase; import org.apache.cassandra.analytics.TestConsistencyLevel; import org.apache.cassandra.analytics.TestUninterruptibles; import org.apache.cassandra.testing.utils.ClusterUtils; -import org.apache.cassandra.distributed.api.ICluster; import org.apache.cassandra.distributed.api.IInstance; import org.apache.cassandra.distributed.api.NodeToolResult; import org.apache.cassandra.sidecar.testing.QualifiedName; import org.apache.cassandra.spark.bulkwriter.WriterOptions; -import org.apache.cassandra.testing.ClusterBuilderConfiguration; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; @@ -56,8 +53,8 @@ abstract class LeavingTestBase extends ResiliencyTestBase { - private static final Logger LOGGER = LoggerFactory.getLogger(LeavingTestBase.class); - List leavingNodes; + private final List> decommissions = new ArrayList<>(); + List leavingNodes; Dataset df; private Map> expectedInstanceData; @@ -83,17 +80,35 @@ protected void beforeTestStart() } @Override - protected void beforeClusterProvisioning() + protected void afterClusterProvisioned() { - assumeTopologyChangeHooksSupported(); + if (requiresConcurrentTopologyChanges()) + { + startTopologyChange(); + } } @Override - protected void afterClusterProvisioned() + protected void afterSchemaInitialized() { - ClusterBuilderConfiguration configuration = testClusterConfiguration(); + if (!requiresConcurrentTopologyChanges()) + { + startTopologyChange(); + } + } + + private void startTopologyChange() + { + prepareTopologyChange(); IInstance seed = cluster.getFirstRunningInstance(); - leavingNodes = decommissionNodes(cluster, leavingNodesPerDc(), configuration.dcCount); + leavingNodes = new ArrayList<>(); + int count = leavingNodeCount(); + for (int i = 0; i < count; i++) + { + IInstance node = cluster.get(cluster.size() - i); + decommissionNode(node); + leavingNodes.add(node); + } // Wait until nodes have reached expected state TestUninterruptibles.awaitUninterruptiblyOrThrow(transitioningStateStart(), 4, TimeUnit.MINUTES); @@ -102,11 +117,30 @@ protected void afterClusterProvisioned() protected void completeTransitionsAndValidateWrites(CountDownLatch transitionalStateEnd, Stream testInputs) { - for (int i = 0; i < leavingNodesPerDc(); i++) + completeTransitionsAndValidateWrites(transitionalStateEnd, testInputs, false); + } + + protected void completeTransitionsAndValidateWrites(CountDownLatch transitionalStateEnd, + Stream testInputs, + boolean failureExpected) + { + while (transitionalStateEnd.getCount() > 0) { transitionalStateEnd.countDown(); } + decommissions.forEach(task -> { + NodeToolResult result = awaitTopologyChange(task, false); + if (failureExpected) + { + result.asserts().failure().errorContains("Simulated leave failure"); + } + else + { + result.asserts().success(); + } + }); + testInputs.forEach(arguments -> { TestConsistencyLevel cl = (TestConsistencyLevel) arguments.get()[0]; @@ -121,10 +155,15 @@ protected void completeTransitionsAndValidateWrites(CountDownLatch transitionalS */ protected abstract CountDownLatch transitioningStateStart(); - /** - * @return the number of nodes per datacenter that are expected to leave the cluster - */ - protected abstract int leavingNodesPerDc(); + protected int leavingNodesPerDc() + { + return 1; + } + + protected int leavingNodeCount() + { + return requiresConcurrentTopologyChanges() ? leavingNodesPerDc() * testClusterConfiguration().dcCount : 1; + } protected static Stream singleDCTestInputs() { @@ -145,27 +184,11 @@ protected static Stream multiDCTestInputs() ); } - protected List decommissionNodes(ICluster cluster, - int leavingNodesPerDC, - int numDcs) + private void decommissionNode(IInstance node) { - List leavingNodes = new ArrayList<>(); - for (int i = 0; i < leavingNodesPerDC * numDcs; i++) - { - IInstance node = cluster.get(cluster.size() - i); - new Thread(() -> { - NodeToolResult decommission = node.nodetoolResult("decommission"); - if (decommission.getRc() != 0 || decommission.getError() != null) - { - LOGGER.error("Failed to decommission instance={}", - node.config().num(), decommission.getError()); - } - decommission.asserts().success(); - }).start(); - leavingNodes.add(node); - } - - return leavingNodes; + FutureTask decommission = new FutureTask<>(() -> node.nodetoolResult("decommission")); + decommissions.add(decommission); + new Thread(decommission, "decommission-node").start(); } protected boolean areLeavingNodesPartOfCluster(IInstance seed, List leavingNodes) diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LegacyLeavingMultiDCFailureTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LegacyLeavingMultiDCFailureTest.java new file mode 100644 index 000000000..e086a9a75 --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LegacyLeavingMultiDCFailureTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.shrink; + +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import com.google.common.util.concurrent.Uninterruptibles; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.dynamic.TypeResolutionStrategy; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.sidecar.testing.QualifiedName; +import org.apache.cassandra.spark.bulkwriter.WriterOptions; +import org.apache.cassandra.testing.ClusterBuilderConfiguration; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; +import static org.apache.cassandra.testing.TestUtils.DC1_RF3_DC2_RF3; +import static org.apache.cassandra.testing.TestUtils.ROW_COUNT; +import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; +import static org.assertj.core.api.Assertions.assertThat; + +class LegacyLeavingMultiDCFailureTest extends LeavingTestBase +{ + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + + @ParameterizedTest(name = "{index} => {0}") + @MethodSource("multiDCTestInputs") + void testLeavingFailureScenario(TestConsistencyLevel cl) + { + QualifiedName table = uniqueTestTableFullName(TEST_KEYSPACE, cl.readCL, cl.writeCL); + bulkWriterDataFrameWriter(df, table).option(WriterOptions.BULK_WRITER_CL.name(), cl.writeCL.name()) + .save(); + // validate data right after bulk writes + validateData(table, cl.readCL, ROW_COUNT); + validateNodeSpecificData(table, generateExpectedInstanceData(cluster, leavingNodes, ROW_COUNT), false); + } + + @Override + protected void initializeSchemaForTest() + { + createTestKeyspace(TEST_KEYSPACE, DC1_RF3_DC2_RF3); + multiDCTestInputs().forEach(arguments -> { + QualifiedName tableName = uniqueTestTableFullName(TEST_KEYSPACE, arguments.get()); + createTestTable(tableName, CREATE_TEST_TABLE_STATEMENT); + }); + } + + @Override + protected void beforeClusterShutdown() + { + completeTransitionsAndValidateWrites(BBHelperLeavingNodesMultiDCFailure.transitionalStateEnd, multiDCTestInputs(), true); + + // For tests that involve LEAVE failures, we validate that the leaving nodes are part of the cluster + // check leave node are part of cluster when leave fails + assertThat(areLeavingNodesPartOfCluster(cluster.get(1), leavingNodes)).isTrue(); + } + + @Override + protected ClusterBuilderConfiguration testClusterConfiguration() + { + return clusterConfig().dcCount(2) + .nodesPerDc(5) + .requestFeature(Feature.NETWORK) + .instanceInitializer(BBHelperLeavingNodesMultiDCFailure::install); + } + + @Override + protected int leavingNodesPerDc() + { + return 1; + } + + @Override + protected CountDownLatch transitioningStateStart() + { + return BBHelperLeavingNodesMultiDCFailure.transitionalStateStart; + } + + /** + * ByteBuddy helper for multiple leaving nodes multi-DC failure scenario + */ + public static class BBHelperLeavingNodesMultiDCFailure + { + static final CountDownLatch transitionalStateStart = new CountDownLatch(2); + static final CountDownLatch transitionalStateEnd = new CountDownLatch(2); + + public static void install(ClassLoader cl, Integer nodeNumber) + { + // Test case involves 10 node cluster (5 nodes per DC) with a 2 leaving nodes (1 per DC) + // We intercept the shutdown of the leaving nodes (9, 10) to validate token ranges + if (nodeNumber > 8) + { + TypePool typePool = TypePool.Default.of(cl); + TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") + .resolve(); + new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) + .method(named("unbootstrap")) + .intercept(MethodDelegation.to(BBHelperLeavingNodesMultiDCFailure.class)) + // Defer class loading until all dependencies are loaded + .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) + .load(cl, ClassLoadingStrategy.Default.INJECTION); + } + } + + @SuppressWarnings("unused") + public static void unbootstrap(@SuperCall Callable orig) throws Exception + { + transitionalStateStart.countDown(); + Uninterruptibles.awaitUninterruptibly(transitionalStateEnd, 2, TimeUnit.MINUTES); + throw new ExecutionException(new UnsupportedOperationException("Simulated leave failure")); + } + } +} diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LegacyLeavingMultiDCTest.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LegacyLeavingMultiDCTest.java new file mode 100644 index 000000000..b7f221732 --- /dev/null +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/shrink/LegacyLeavingMultiDCTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.analytics.shrink; + +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.dynamic.ClassFileLocator; +import net.bytebuddy.dynamic.TypeResolutionStrategy; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import net.bytebuddy.pool.TypePool; +import org.apache.cassandra.analytics.TestConsistencyLevel; +import org.apache.cassandra.analytics.TestUninterruptibles; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.sidecar.testing.QualifiedName; +import org.apache.cassandra.spark.bulkwriter.WriterOptions; +import org.apache.cassandra.testing.ClusterBuilderConfiguration; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static org.apache.cassandra.testing.TestUtils.CREATE_TEST_TABLE_STATEMENT; +import static org.apache.cassandra.testing.TestUtils.DC1_RF3_DC2_RF3; +import static org.apache.cassandra.testing.TestUtils.ROW_COUNT; +import static org.apache.cassandra.testing.TestUtils.TEST_KEYSPACE; + +class LegacyLeavingMultiDCTest extends LeavingTestBase +{ + @Override + protected boolean requiresConcurrentTopologyChanges() + { + return true; + } + + @ParameterizedTest(name = "{index} => {0}") + @MethodSource("multiDCTestInputs") + void testLeavingScenario(TestConsistencyLevel cl) + { + QualifiedName table = uniqueTestTableFullName(TEST_KEYSPACE, cl.readCL, cl.writeCL); + bulkWriterDataFrameWriter(df, table).option(WriterOptions.BULK_WRITER_CL.name(), cl.writeCL.name()) + .save(); + // validate data right after bulk writes + validateData(table, cl.readCL, ROW_COUNT); + validateNodeSpecificData(table, generateExpectedInstanceData(cluster, leavingNodes, ROW_COUNT), false); + } + + @Override + protected void initializeSchemaForTest() + { + createTestKeyspace(TEST_KEYSPACE, DC1_RF3_DC2_RF3); + multiDCTestInputs().forEach(arguments -> { + QualifiedName tableName = uniqueTestTableFullName(TEST_KEYSPACE, arguments.get()); + createTestTable(tableName, CREATE_TEST_TABLE_STATEMENT); + }); + } + + @Override + protected void beforeClusterShutdown() + { + completeTransitionsAndValidateWrites(BBHelperLeavingNodesMultiDC.transitionalStateEnd, multiDCTestInputs()); + } + + @Override + protected ClusterBuilderConfiguration testClusterConfiguration() + { + return clusterConfig().dcCount(2) + .nodesPerDc(5) + .requestFeature(Feature.NETWORK) + .instanceInitializer(BBHelperLeavingNodesMultiDC::install); + } + + @Override + protected int leavingNodesPerDc() + { + return 1; + } + + @Override + protected CountDownLatch transitioningStateStart() + { + return BBHelperLeavingNodesMultiDC.transitionalStateStart; + } + + /** + * ByteBuddy helper for multiple leaving nodes multi-DC + */ + public static class BBHelperLeavingNodesMultiDC + { + static final CountDownLatch transitionalStateStart = new CountDownLatch(2); + static final CountDownLatch transitionalStateEnd = new CountDownLatch(2); + + public static void install(ClassLoader cl, Integer nodeNumber) + { + // Test case involves 10 node cluster (5 nodes per DC) with a 2 leaving nodes (1 per DC) + // We intercept the shutdown of the leaving nodes (9, 10) to validate token ranges + if (nodeNumber > 8) + { + TypePool typePool = TypePool.Default.of(cl); + TypeDescription description = typePool.describe("org.apache.cassandra.service.StorageService") + .resolve(); + new ByteBuddy().rebase(description, ClassFileLocator.ForClassLoader.of(cl)) + .method(named("unbootstrap")) + .intercept(MethodDelegation.to(BBHelperLeavingNodesMultiDC.class)) + // Defer class loading until all dependencies are loaded + .make(TypeResolutionStrategy.Lazy.INSTANCE, typePool) + .load(cl, ClassLoadingStrategy.Default.INJECTION); + } + } + + @SuppressWarnings("unused") + public static void unbootstrap(@SuperCall Callable orig) throws Exception + { + transitionalStateStart.countDown(); + TestUninterruptibles.awaitUninterruptiblyOrThrow(transitionalStateEnd, 2, TimeUnit.MINUTES); + orig.call(); + } + } +} From c72fca86069a66c6de82139a7a8472c1c15bc2ed Mon Sep 17 00:00:00 2001 From: mck Date: Sun, 4 Oct 2026 14:35:16 +0200 Subject: [PATCH 2/4] =?UTF-8?q?SQUASH=20=E2=80=93=20review=20comment:=20ad?= =?UTF-8?q?d=206.0-alpha2=20to=20the=20gha=20integration=20test=20matrix?= =?UTF-8?q?=20(for=20spark=203=20and=204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2bc40c45a..6e6c7e109 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -77,7 +77,7 @@ jobs: path: ${{ github.workspace }} key: build-jdk11-${{ github.sha }} - # JDK17 dependency build for the Spark 4 / Scala 2.13 / Cassandra 5.0 matrix. + # JDK17 dependency build for the Spark 4 / Scala 2.13 / Cassandra 5.0 and 6.0 matrix. # Produces a distinct workspace cache (build-jdk17-...) so downstream test jobs # must restore from exactly one of build-jdk11 / build-jdk17 — mixing them # would clobber dependency jars. @@ -117,7 +117,7 @@ jobs: export SCALA_VERSION="2.13" # JDK17 only targets Cassandra 5.0+; skip 4.0 / 4.1 dtest jar builds # (build-dtest-jars.sh reads this var to filter CANDIDATE_BRANCHES). - export BRANCHES="cassandra-5.0" + export BRANCHES="cassandra-5.0 cassandra-6.0" ./scripts/build-dependencies.sh @@ -217,16 +217,21 @@ jobs: needs: [build-jdk11, build-jdk17] runs-on: ubuntu-latest strategy: - # GitHub Actions generate a cross-product of 'config' × 'job_index' (4 × 5 = 20 jobs). + # GitHub Actions generate a cross-product of 'config' × 'job_index' (6 × 5 = 30 jobs). # The 'include' entries don't add new combinations — they augment existing ones # by matching on 'config' and injecting 'scala', 'cassandra', 'jdk', and 'spark' # into each match. To add a new version: add one entry to 'config' and one to # 'include'. matrix: - config: ['s2.13-c5.0.7', 's2.12-c4.1.4', 's2.12-c4.0.17', 's2.13-c5.0.7-spark4'] + config: ['s2.13-c6.0a2', 's2.13-c5.0.7', 's2.12-c4.1.4', 's2.12-c4.0.17', 's2.13-c6.0a2-spark4', 's2.13-c5.0.7-spark4'] job_index: [0, 1, 2, 3, 4] job_total: [5] include: + - config: 's2.13-c6.0a2' + scala: '2.13' + cassandra: '6.0-alpha2' + jdk: '11' + spark: '3' - config: 's2.13-c5.0.7' scala: '2.13' cassandra: '5.0.7' @@ -242,6 +247,11 @@ jobs: cassandra: '4.0.17' jdk: '11' spark: '3' + - config: 's2.13-c6.0a2-spark4' + scala: '2.13' + cassandra: '6.0-alpha2' + jdk: '17' + spark: '4' - config: 's2.13-c5.0.7-spark4' scala: '2.13' cassandra: '5.0.7' From ea6d9faff0a63fed5a94693a457361c64e5dc7eb Mon Sep 17 00:00:00 2001 From: mck Date: Sun, 4 Oct 2026 18:39:35 +0200 Subject: [PATCH 3/4] =?UTF-8?q?SQUASH=20=E2=80=93=20test=20failure=20debug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/apache/cassandra/analytics/SparkTestUtils.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/SparkTestUtils.java b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/SparkTestUtils.java index dddb030ed..9d0395c01 100644 --- a/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/SparkTestUtils.java +++ b/cassandra-analytics-integration-tests/src/test/java/org/apache/cassandra/analytics/SparkTestUtils.java @@ -256,9 +256,13 @@ public void assertExpectedBulkWriteFailure(String writeCL, DataFrameWriter cause = cause.getCause(); } - assertThat(cause).isNotNull() - .hasMessageFindingMatch("Failed to write (\\d+) ranges with " + writeCL + - " for job ([a-zA-Z0-9-]+) in phase .*"); + if (cause == null) + { + throw new AssertionError("Expected a range write failure with " + writeCL, thrown); + } + + assertThat(cause).hasMessageFindingMatch("Failed to write (\\d+) ranges with " + writeCL + + " for job ([a-zA-Z0-9-]+) in phase .*"); } /** From f6737054790aee3c8687c6b22f4a6eb7bb4a2250 Mon Sep 17 00:00:00 2001 From: mck Date: Sun, 4 Oct 2026 20:44:13 +0000 Subject: [PATCH 4/4] SQUASH: preserve Cassandra 6.0 keyspace instances during SSTable writer construction Support integration tests after Transactional Cluster Metadata Transactional Cluster Metadata (TCM) changes the hooks and sequencing needed to test writes during topology changes. Create schema before transitions and configure three metadata replicas before stopping or removing nodes on Cassandra 6.0. Await transition completion so data checks cannot pass while a join, replacement, or decommission is still running. Keep concurrent transitions that require overlapping range locks on legacy Cassandra. Exercise concurrent TCM joins and leaves with Sidecar's CASSSIDECAR-277 disjoint token placement, including a join in an unreplicated datacenter. Retain consistency-level and per-node data checks, and scope legacy range-movement overrides to the tests that require them. Run Cassandra 6.0-alpha2 integration tests with Spark 3 and Spark 4, building the dtest dependency for each Java version. Increase the integration JVM datagram-socket limit to accommodate multi-node clusters. Offline Cassandra 6.0 schema commits do not notify the listener that creates keyspace instances. SSTable writer construction can therefore replace the schema while another task builds or reads a table. Preserve existing instances across those commits and use the same lock for writer construction, schema building, and scanner table lookup. This prevents missing-keyspace failures from masking the consistency and corruption failures under test. patch by Mick Semb Wever for CASSANALYTICS-112 Assisted-by: Codex:GPT-6 --- .../bridge/CassandraBridgeImplementation.java | 16 +- .../bridge/SSTableWriterImplementation.java | 160 ++++++++++++++++++ .../spark/reader/CompactionStreamScanner.java | 157 +++++++++++++++++ .../SSTableWriterImplementationTest.java | 27 +++ .../cassandra/bridge/SchemaUpdater.java | 27 +++ 5 files changed, 379 insertions(+), 8 deletions(-) create mode 100644 cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/bridge/SSTableWriterImplementation.java create mode 100644 cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/spark/reader/CompactionStreamScanner.java diff --git a/cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/bridge/CassandraBridgeImplementation.java b/cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/bridge/CassandraBridgeImplementation.java index f50f00303..9db0f46c5 100644 --- a/cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/bridge/CassandraBridgeImplementation.java +++ b/cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/bridge/CassandraBridgeImplementation.java @@ -590,7 +590,7 @@ public synchronized void writeSSTable(Partitioner partitioner, builder.withType(statement); } - try (CQLSSTableWriter ssTable = builder.build()) + try (CQLSSTableWriter ssTable = SchemaUpdater.withKeyspaceInstances(builder::build)) { writer.accept(values -> { try @@ -695,13 +695,13 @@ public void writeTombstoneSSTable(Partitioner partitioner, // Cassandra 6.0 CQLSSTableWriter takes any modification statement, a DELETE included, and writes the // range tombstones of a slice delete through the same DeleteStatement.createSlices path the earlier // bridges need SSTableTombstoneWriter for - try (CQLSSTableWriter writer = CQLSSTableWriter.builder() - .inDirectory(directory.toFile().getAbsolutePath()) - .forTable(createStatement) - .withPartitioner(getPartitioner(partitioner)) - .using(deleteStatement) - .withBufferSizeInMB(128) - .build()) + CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder() + .inDirectory(directory.toFile().getAbsolutePath()) + .forTable(createStatement) + .withPartitioner(getPartitioner(partitioner)) + .using(deleteStatement) + .withBufferSizeInMB(128); + try (CQLSSTableWriter writer = SchemaUpdater.withKeyspaceInstances(builder::build)) { consumer.accept(values -> { try diff --git a/cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/bridge/SSTableWriterImplementation.java b/cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/bridge/SSTableWriterImplementation.java new file mode 100644 index 000000000..cc1053239 --- /dev/null +++ b/cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/bridge/SSTableWriterImplementation.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cassandra.bridge; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import javax.annotation.concurrent.NotThreadSafe; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.config.Config; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.RandomPartitioner; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.io.sstable.CQLSSTableWriter; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.jetbrains.annotations.NotNull; + +@NotThreadSafe // The underlying CQLSSTableWriter is not thread-safe +public class SSTableWriterImplementation implements SSTableWriter +{ + static + { + Config.setClientMode(true); + } + + private final CQLSSTableWriter writer; + private Consumer> producedSSTablesListener; + + public SSTableWriterImplementation(String inDirectory, + String partitioner, + String createStatement, + String insertStatement, + @NotNull Set userDefinedTypeStatements, + int bufferSizeMB) + { + this(inDirectory, determineSupportedPartitioner(partitioner), createStatement, insertStatement, userDefinedTypeStatements, bufferSizeMB); + } + + @VisibleForTesting + public SSTableWriterImplementation(String inDirectory, + IPartitioner partitioner, + String createStatement, + String insertStatement, + @NotNull Set userDefinedTypeStatements, + int bufferSizeMB) + { + CQLSSTableWriter.Builder builder = configureBuilder(inDirectory, + createStatement, + insertStatement, + bufferSizeMB, + userDefinedTypeStatements, + this::onSSTablesProduced, + partitioner); + this.writer = SchemaUpdater.withKeyspaceInstances(builder::build); + } + + private static IPartitioner determineSupportedPartitioner(String partitioner) + { + // Cassandra 6.0 makes the partitioner constructors non-public. + return partitioner.toLowerCase().contains("random") + ? RandomPartitioner.instance + : Murmur3Partitioner.instance; + } + + private void onSSTablesProduced(Collection sstables) + { + Objects.requireNonNull(producedSSTablesListener, "producedSSTablesListener is not set"); + Set sstableDescriptors = sstables + .stream() + .map(sstable -> { + String baseFilename = CassandraBridgeImplementation.baseFilename(sstable.descriptor); + // TODO: for now, the sstableReader is closed immediately, + // TODO (CONTI): we can potentially read from the reader to validate the underlying sstable, + // TODO (CONTI): replacing org.apache.cassandra.spark.bulkwriter.SortedSSTableWriter.validateSSTables + sstable.selfRef().close(); + return new SSTableDescriptor(baseFilename); + }) + .collect(Collectors.toSet()); + + producedSSTablesListener.accept(sstableDescriptors); + } + + @Override + public void addRow(Map values) throws IOException + { + try + { + writer.addRow(values); + } + catch (InvalidRequestException exception) + { + throw new RuntimeException(exception); + } + } + + @Override + public void setSSTablesProducedListener(Consumer> listener) + { + producedSSTablesListener = Objects.requireNonNull(listener); + } + + @Override + public void close() throws IOException + { + writer.close(); + } + + @VisibleForTesting + static CQLSSTableWriter.Builder configureBuilder(String inDirectory, + String createStatement, + String insertStatement, + int bufferSizeMB, + Set udts, + Consumer> producedSSTablesListener, + IPartitioner cassPartitioner) + { + CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder(); + + for (String udt : udts) + { + builder.withType(udt); + } + + return builder.inDirectory(inDirectory) + .forTable(createStatement) + .withPartitioner(cassPartitioner) + .using(insertStatement) + // The data frame to write is always sorted, + // see org.apache.cassandra.spark.bulkwriter.CassandraBulkSourceRelation.insert + .sorted() + .withSSTableProducedListener(producedSSTablesListener) + .openSSTableOnProduced() + .withMaxSSTableSizeInMiB(bufferSizeMB); + } +} diff --git a/cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/spark/reader/CompactionStreamScanner.java b/cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/spark/reader/CompactionStreamScanner.java new file mode 100644 index 000000000..fcd6c75e1 --- /dev/null +++ b/cassandra-six-zero-bridge/src/main/java/org/apache/cassandra/spark/reader/CompactionStreamScanner.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cassandra.spark.reader; + +import java.math.BigInteger; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.LongPredicate; +import java.util.stream.Collectors; + +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.bridge.CassandraSchema; +import org.apache.cassandra.db.AbstractCompactionController; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.compaction.AbstractCompactionStrategy; +import org.apache.cassandra.db.compaction.CompactionIterator; +import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.schema.CompactionParams; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.spark.data.partitioner.Partitioner; +import org.apache.cassandra.spark.utils.IOUtils; +import org.apache.cassandra.spark.utils.TimeProvider; +import org.apache.cassandra.utils.TimeUUID; +import org.jetbrains.annotations.NotNull; + +public class CompactionStreamScanner extends AbstractStreamScanner +{ + private final Collection toCompact; + private final TimeUUID taskId; + + private PurgingCompactionController controller; + private AbstractCompactionStrategy.ScannerList scanners; + private CompactionIterator ci; + + @VisibleForTesting + CompactionStreamScanner(@NotNull TableMetadata cfMetaData, + @NotNull Partitioner partitionerType, + @NotNull Collection toCompact) + { + this(cfMetaData, partitionerType, TimeProvider.DEFAULT, toCompact); + } + + public CompactionStreamScanner(@NotNull TableMetadata cfMetaData, + @NotNull Partitioner partitionerType, + @NotNull TimeProvider timeProvider, + @NotNull Collection toCompact) + { + super(cfMetaData, partitionerType, timeProvider); + this.toCompact = toCompact; + this.taskId = TimeUUID.Generator.nextTimeUUID(); + } + + @Override + public void close() + { + Arrays.asList(controller, scanners, ci) + .forEach(IOUtils::closeQuietly); + } + + @Override + protected void handleRowTombstone(BigInteger token, Row row) + { + throw new IllegalStateException("Row tombstone found. " + + "It should have been purged in CompactionIterator." + + "Partition key token: " + token); + } + + @Override + protected void handlePartitionTombstone(BigInteger token, UnfilteredRowIterator partition) + { + throw new IllegalStateException("Partition tombstone found. " + + "It should have been purged in CompactionIterator. " + + "Partition key token: " + token); + } + + @Override + protected void handleCellTombstone(BigInteger token) + { + throw new IllegalStateException("Cell tombstone found. " + + "It should have been purged in CompactionIterator. " + + "Partition key token: " + token); + } + + @Override + protected void handleCellTombstoneInComplex(BigInteger token, Cell cell) + { + throw new IllegalStateException("Cell tombstone in complex type found. " + + "It should have been purged in CompactionIterator. " + + "Partition key token: " + token); + } + + @Override + UnfilteredPartitionIterator initializePartitions() + { + long nowInSec = timeProvider.referenceEpochInSeconds(); + ColumnFamilyStore cfStore = CassandraSchema.apply(schema -> schema.getKeyspaceInstance(metadata.keyspace) + .getColumnFamilyStore(metadata.name)); + controller = new PurgingCompactionController(cfStore, CompactionParams.TombstoneOption.NONE); + List scannerList = toCompact.stream() + .map(Scannable::scanner) + .collect(Collectors.toList()); + scanners = new AbstractCompactionStrategy.ScannerList(scannerList); + ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, taskId); + return ci; + } + + private static class PurgingCompactionController extends AbstractCompactionController implements AutoCloseable + { + PurgingCompactionController(ColumnFamilyStore cfs, CompactionParams.TombstoneOption tombstoneOption) + { + super(cfs, Integer.MAX_VALUE, tombstoneOption); + } + + @Override + public boolean compactingRepaired() + { + return false; + } + + @Override + public LongPredicate getPurgeEvaluator(DecoratedKey key) + { + // Purge all tombstones + return time -> true; + } + + @Override + public void close() + { + } + } +} diff --git a/cassandra-six-zero-bridge/src/test/java/org/apache/cassandra/bridge/SSTableWriterImplementationTest.java b/cassandra-six-zero-bridge/src/test/java/org/apache/cassandra/bridge/SSTableWriterImplementationTest.java index 8dba1acfd..cd99b72e2 100644 --- a/cassandra-six-zero-bridge/src/test/java/org/apache/cassandra/bridge/SSTableWriterImplementationTest.java +++ b/cassandra-six-zero-bridge/src/test/java/org/apache/cassandra/bridge/SSTableWriterImplementationTest.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import com.google.common.collect.ImmutableMap; @@ -41,6 +42,10 @@ import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.spark.TestUtils; import org.apache.cassandra.util.ReflectionUtils; @@ -79,6 +84,28 @@ void testSSTableWriterConfiguration() throws NoSuchFieldException, IllegalAccess assertThat(peekBufferSizeInMB(builder)).isEqualTo(250); } + @Test + void testWriterPreservesKeyspaceInstances() throws IOException + { + String keyspaceName = "existing_" + UUID.randomUUID().toString().replace("-", ""); + SchemaUpdater.load(Schema.instance, KeyspaceMetadata.create(keyspaceName, KeyspaceParams.simple(1))); + Keyspace keyspace = Schema.instance.getKeyspaceInstance(keyspaceName); + assertThat(keyspace).isNotNull(); + String tableName = "table_" + UUID.randomUUID().toString().replace("-", ""); + + try (SSTableWriterImplementation writer = new SSTableWriterImplementation(writeDirectory.getAbsolutePath(), + Murmur3Partitioner.instance, + CREATE_STATEMENT.replace("test_table", tableName), + INSERT_STATEMENT.replace("test_table", tableName), + Collections.emptySet(), + 1)) + { + writer.setSSTablesProducedListener(sstables -> {}); + assertThat(Schema.instance.getKeyspaceInstance(keyspaceName)).isSameAs(keyspace); + assertThat(Schema.instance.getKeyspaceInstance("test_keyspace")).isNotNull(); + } + } + @Test void testGetProducedSSTables() throws IOException { diff --git a/cassandra-six-zero-types/src/main/java/org/apache/cassandra/bridge/SchemaUpdater.java b/cassandra-six-zero-types/src/main/java/org/apache/cassandra/bridge/SchemaUpdater.java index 5802d4de3..567cac92e 100644 --- a/cassandra-six-zero-types/src/main/java/org/apache/cassandra/bridge/SchemaUpdater.java +++ b/cassandra-six-zero-types/src/main/java/org/apache/cassandra/bridge/SchemaUpdater.java @@ -19,9 +19,12 @@ package org.apache.cassandra.bridge; +import java.util.function.Supplier; + import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Keyspaces; +import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaProvider; import org.apache.cassandra.schema.SchemaTransformation; import org.apache.cassandra.schema.SchemaTransformations; @@ -42,6 +45,30 @@ private SchemaUpdater() { } + /** + * Preserves keyspace instances across schema commits made by an offline SSTable writer. + * Readers use the same lock to avoid observing a committed schema before its instances are ready. + */ + public static T withKeyspaceInstances(Supplier action) + { + synchronized (Schema.instance) + { + ClusterMetadata before = ClusterMetadata.current(); + try + { + return action.get(); + } + finally + { + ClusterMetadata after = ClusterMetadata.current(); + if (after.schema != before.schema) + { + after.schema.initializeKeyspaceInstances(before.schema, false); + } + } + } + } + /** * Commits a schema transformation, then creates the keyspace instances that the transformation adds. *