diff --git a/pom.xml b/pom.xml index f956113943..354e6c26b3 100644 --- a/pom.xml +++ b/pom.xml @@ -183,6 +183,7 @@ 1.63.0 + 2.28.1 true _ @@ -403,6 +404,11 @@ opentelemetry-context ${opentelemetry.version} + + io.opentelemetry.javaagent + opentelemetry-javaagent + ${opentelemetry-javaagent.version} + diff --git a/ratis-assembly/pom.xml b/ratis-assembly/pom.xml index 0558693039..9987d91a15 100644 --- a/ratis-assembly/pom.xml +++ b/ratis-assembly/pom.xml @@ -289,6 +289,11 @@ org.apache.ratis ratis-shell + + + io.opentelemetry.javaagent + opentelemetry-javaagent + diff --git a/ratis-assembly/src/main/assembly/bin.xml b/ratis-assembly/src/main/assembly/bin.xml index 12161b00a8..6a4f3a62b1 100644 --- a/ratis-assembly/src/main/assembly/bin.xml +++ b/ratis-assembly/src/main/assembly/bin.xml @@ -33,6 +33,12 @@ examples/lib + + lib/trace + + io.opentelemetry.javaagent:* + + diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java index f24360f62b..b27616feb3 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java @@ -44,6 +44,7 @@ import org.apache.ratis.thirdparty.com.google.common.cache.Cache; import org.apache.ratis.thirdparty.com.google.common.cache.CacheBuilder; import org.apache.ratis.trace.TraceUtils; +import org.apache.ratis.trace.otel.OTelTraceUtils; import org.apache.ratis.util.CollectionUtils; import org.apache.ratis.util.IOUtils; import org.apache.ratis.util.JavaUtils; @@ -292,6 +293,9 @@ RaftClientRequest newRaftClientRequest( b.setLeaderId(getLeaderId()) .setRepliedCallIds(repliedCallIds.get(callId)); } + if (TraceUtils.isEnabled()) { + b.setSpanContext(OTelTraceUtils.injectContextToProto()); + } return b.setClientId(clientId) .setGroupId(groupId) .setCallId(callId) diff --git a/ratis-common/src/main/java/org/apache/ratis/trace/otel/OTelTraceUtils.java b/ratis-common/src/main/java/org/apache/ratis/trace/otel/OTelTraceUtils.java index 662503a8d3..fda0ca5c09 100644 --- a/ratis-common/src/main/java/org/apache/ratis/trace/otel/OTelTraceUtils.java +++ b/ratis-common/src/main/java/org/apache/ratis/trace/otel/OTelTraceUtils.java @@ -32,6 +32,10 @@ public final class OTelTraceUtils { private OTelTraceUtils() { } + public static SpanContextProto injectContextToProto() { + return injectContextToProto(Context.current()); + } + public static SpanContextProto injectContextToProto(Context context) { final Map carrier = new TreeMap<>(); getTextMapPropagator().inject(context, carrier, (map, key, value) -> map.put(key, value)); diff --git a/ratis-examples/README.md b/ratis-examples/README.md index 1e50058dc1..8ae5eeae18 100644 --- a/ratis-examples/README.md +++ b/ratis-examples/README.md @@ -146,3 +146,53 @@ by using the `run_all_tests.sh` script found in [dev-support/vagrant/](../dev-su See the [dev-support/vagrant/README.md](../dev-support/vagrant/README.md) for more on dependencies and what is setup. This will allow one to try a fully setup three server Ratis cluster on a single VM image, preventing resource contention with your development host and allowing failure injection too. + + +## Enable Tracing when testing examples +Ratis uses OpenTelemetry to provide distributed tracing of requests across the cluster. +Tracing is configured through `RaftProperties`; Ratis itself does not read JVM system +properties for configuration. + +The example shell scripts pass `-Draft.otel.tracing.enabled=true` as a JVM option when the +OpenTelemetry agent is found. `Runner` reads that property through `ExampleLauncher` and +applies it to `RaftProperties` before starting an example (e.g. "filestore" or +"arithmetic"). + +For example, to enable tracing for the FileStore server, you can run the following command: + +```bash +# build the assembly jar first +mvn clean package -DskipTests + +# extract the assembly jar, mainly we need the dependencies for tracing +pushd ratis-assembly/target +file=$(ls -1t ratis-assembly-*.tar.gz | head -n1) +tar -zxvf "$file" +popd + +# run the server, the tracing will be enabled because it found the +# opentelemetry agent jar +BIN=ratis-examples/src/main/bin +PEERS=n0:127.0.0.1:6000,n1:127.0.0.1:6001,n2:127.0.0.1:6002 + +ID=n0; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS} +ID=n1; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS} +ID=n2; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS} + +# some message should be shown +[otel.javaagent 2026-04-15 15:55:11:320 -0700] [main] INFO io.opentelemetry.javaagent.tooling.VersionLogger - opentelemetry-javaagent - version: 2.26.1 +``` + +If you want to configure the tracing further, you can set the following system properties + +```bash +# below is an example to configure the OTLP exporter to send the traces to a local collector, +# you can change the endpoint and protocol as needed. ExampleLauncher reads +# -Draft.otel.tracing.enabled=true and applies it to RaftProperties. +export OTEL_EXPORTER_OPTS="-Dotel.traces.exporter=otlp \ +-Dotel.exporter.otlp.protocol=grpc \ +-Dotel.exporter.otlp.endpoint=http://localhost:4317 \ +-Dotel.metrics.exporter=none \ +-Dotel.logs.exporter=none \ +-Draft.otel.tracing.enabled=true" +``` \ No newline at end of file diff --git a/ratis-examples/src/main/bin/client.sh b/ratis-examples/src/main/bin/client.sh index e42786fa3c..d5e00268ea 100755 --- a/ratis-examples/src/main/bin/client.sh +++ b/ratis-examples/src/main/bin/client.sh @@ -23,7 +23,8 @@ if [ "$#" -lt 2 ]; then exit 1 fi -source $DIR/common.sh +OTEL_SERVICE_NAME=ratis.client +source "${DIR}/common.sh" # One of the examples, e.g. "filestore" or "arithmetic" example="$1" @@ -32,4 +33,4 @@ shift subcommand="$1" shift -java ${LOGGER_OPTS} -jar $ARTIFACT "$example" "$subcommand" "$@" +java ${OTEL_OPTS} ${LOGGER_OPTS} -jar $ARTIFACT "$example" "$subcommand" "$@" diff --git a/ratis-examples/src/main/bin/common.sh b/ratis-examples/src/main/bin/common.sh index b05bf4dbf0..8fa3b3ba7d 100755 --- a/ratis-examples/src/main/bin/common.sh +++ b/ratis-examples/src/main/bin/common.sh @@ -46,3 +46,43 @@ if [[ -d "${CONF_DIR}" ]]; then else LOGGER_OPTS="-Dlog4j.configuration=file:${DIR}/../resources/log4j.properties" fi + + +# for opentelemetry: release tarball uses lib/trace next to examples/; +# dev tree uses ratis-assembly/target/apache-ratis-*-bin/lib/trace after package. +if [[ -n "${OTEL_JAR:-}" && -f "${OTEL_JAR}" ]]; then + : # use OTEL_JAR from environment +else + otel_jar_candidates=() + for _jar in "${SCRIPT_DIR}"/../../lib/trace/opentelemetry-javaagent*.jar; do + [[ -f "${_jar}" ]] && otel_jar_candidates+=("${_jar}") + done + for _otel_trace_dir in "${SCRIPT_DIR}"/../../../../ratis-assembly/target/apache-ratis-*-bin/lib/trace; do + [[ -d "${_otel_trace_dir}" ]] || continue + for _jar in "${_otel_trace_dir}"/opentelemetry-javaagent*.jar; do + [[ -f "${_jar}" ]] && otel_jar_candidates+=("${_jar}") + done + done + if ((${#otel_jar_candidates[@]} > 0)); then + OTEL_JAR="$(printf '%s\n' "${otel_jar_candidates[@]}" | sort -V | tail -n 1)" + else + echo "Warning: OpenTelemetry agent jar not found; OTEL disabled" + OTEL_JAR="" + fi +fi + +OTEL_SERVICE_NAME="${OTEL_SERVICE_NAME:-ratis.server}" +OTEL_DEFAULT_EXPORTER_OPTS="-Dotel.traces.exporter=logging \ +-Dotel.metrics.exporter=none \ +-Dotel.logs.exporter=logging" +# Unset or empty → default exporter flags (empty is not a supported override). +OTEL_EXPORTER_OPTS="${OTEL_EXPORTER_OPTS:-${OTEL_DEFAULT_EXPORTER_OPTS}}" + +# Enable javaagent when OTEL_JAR resolves to a file (env or discovery above). +if [[ -n "${OTEL_JAR}" && -f "${OTEL_JAR}" ]]; then + OTEL_OPTS="-javaagent:${OTEL_JAR} -Dotel.resource.attributes=service.name=${OTEL_SERVICE_NAME} ${OTEL_EXPORTER_OPTS} -Draft.otel.tracing.enabled=true" +else + OTEL_OPTS="" +fi + +echo "Using OTEL_OPTS: ${OTEL_OPTS}" \ No newline at end of file diff --git a/ratis-examples/src/main/bin/server.sh b/ratis-examples/src/main/bin/server.sh index 3cc069edf1..c0468c15bf 100755 --- a/ratis-examples/src/main/bin/server.sh +++ b/ratis-examples/src/main/bin/server.sh @@ -16,5 +16,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" source $DIR/common.sh - -java ${LOGGER_OPTS} -jar $ARTIFACT "$@" +java ${OTEL_OPTS} ${LOGGER_OPTS} -jar $ARTIFACT "$@" diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/cli/Client.java b/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/cli/Client.java index 5c01520cef..bb2ad31881 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/cli/Client.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/cli/Client.java @@ -37,7 +37,7 @@ public abstract class Client extends SubCommandBase { @Override public void run() throws Exception { - RaftProperties raftProperties = new RaftProperties(); + RaftProperties raftProperties = newRaftProperties(); final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())), getPeers()); diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/cli/Server.java b/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/cli/Server.java index a93997425e..a3bbf5a9fb 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/cli/Server.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/cli/Server.java @@ -56,7 +56,7 @@ public class Server extends SubCommandBase { @Override public void run() throws Exception { RaftPeerId peerId = RaftPeerId.valueOf(id); - RaftProperties properties = new RaftProperties(); + RaftProperties properties = newRaftProperties(); final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort(); GrpcConfigKeys.Server.setPort(properties, port); diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/common/ExampleLauncher.java b/ratis-examples/src/main/java/org/apache/ratis/examples/common/ExampleLauncher.java new file mode 100644 index 0000000000..373eeb0527 --- /dev/null +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/common/ExampleLauncher.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ratis.examples.common; + +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.trace.TraceConfigKeys; +import org.apache.ratis.util.StringUtils; + +/** + * Applies example-only JVM system properties to {@link RaftProperties}. + * {@link Runner} calls {@link #init()} at startup; example subcommands obtain + * configured properties via {@link #newRaftProperties()}. + */ +public final class ExampleLauncher { + + private static volatile boolean initialized; + private static Boolean tracingEnabled; + + private ExampleLauncher() { + } + + /** Reads example JVM system properties. Called once from {@link Runner#main}. */ + public static void init() { + initFromSystemProperties(); + } + + /** + * Creates {@link RaftProperties} for examples, applying any tracing configuration + * read from the JVM system property {@value TraceConfigKeys#ENABLED_KEY}. + */ + public static RaftProperties newRaftProperties() { + initFromSystemProperties(); + final RaftProperties properties = new RaftProperties(); + if (tracingEnabled != null) { + TraceConfigKeys.setEnabled(properties, tracingEnabled); + } + return properties; + } + + private static void initFromSystemProperties() { + if (initialized) { + return; + } + synchronized (ExampleLauncher.class) { + if (initialized) { + return; + } + final String value = System.getProperty(TraceConfigKeys.ENABLED_KEY); + if (value != null) { + tracingEnabled = StringUtils.string2boolean(value.trim(), TraceConfigKeys.ENABLED_DEFAULT); + } + initialized = true; + } + } +} diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/common/Runner.java b/ratis-examples/src/main/java/org/apache/ratis/examples/common/Runner.java index f8090ff795..5dd4c7a18c 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/common/Runner.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/common/Runner.java @@ -40,6 +40,7 @@ public static void main(String[] args) throws Exception { System.err.println("No command type specified: "); return; } + ExampleLauncher.init(); List commands = initializeCommands(args[0]); Runner runner = new Runner(); diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/common/SubCommandBase.java b/ratis-examples/src/main/java/org/apache/ratis/examples/common/SubCommandBase.java index d650a5cfe8..a965b7883f 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/common/SubCommandBase.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/common/SubCommandBase.java @@ -18,6 +18,7 @@ package org.apache.ratis.examples.common; import com.beust.jcommander.Parameter; +import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.protocol.RaftPeer; import org.apache.ratis.protocol.RaftPeerId; import org.apache.ratis.protocol.RoutingTable; @@ -73,6 +74,10 @@ public RaftPeer getPrimary() { public abstract void run() throws Exception; + protected RaftProperties newRaftProperties() { + return ExampleLauncher.newRaftProperties(); + } + public String getRaftGroupId() { return raftGroupId; } diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java index caf2aa59b2..4cdad21fe0 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java @@ -94,7 +94,7 @@ public int getNumFiles() { @Override public void run() throws Exception { int raftSegmentPreallocatedSize = 1024 * 1024 * 1024; - RaftProperties raftProperties = new RaftProperties(); + RaftProperties raftProperties = newRaftProperties(); RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC); GrpcConfigKeys.setMessageSizeMax(raftProperties, SizeInBytes.valueOf(raftSegmentPreallocatedSize)); diff --git a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Server.java b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Server.java index 5204ee21b0..071fdd1702 100644 --- a/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Server.java +++ b/ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Server.java @@ -77,7 +77,7 @@ public void run() throws Exception { JvmMetrics.initJvmMetrics(TimeDuration.valueOf(10, TimeUnit.SECONDS)); RaftPeerId peerId = RaftPeerId.valueOf(id); - RaftProperties properties = new RaftProperties(); + RaftProperties properties = newRaftProperties(); // Avoid leader change affect the performance RaftServerConfigKeys.Rpc.setTimeoutMin(properties, TimeDuration.valueOf(2, TimeUnit.SECONDS));