RATIS-2398. Add opentelemetry-javaagent to ratis-examples and assembly

pom.xml

@@ -183,6 +183,7 @@
 
     <!-- OpenTelemetry versions -->
     <opentelemetry.version>1.63.0</opentelemetry.version>
+    <opentelemetry-javaagent.version>2.28.1</opentelemetry-javaagent.version>
     <!-- Test properties -->
     <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
     <test.exclude.pattern>_</test.exclude.pattern>
@@ -403,6 +404,11 @@
         <artifactId>opentelemetry-context</artifactId>
         <version>${opentelemetry.version}</version>
       </dependency>
+      <dependency>
+        <groupId>io.opentelemetry.javaagent</groupId>
+        <artifactId>opentelemetry-javaagent</artifactId>
+        <version>${opentelemetry-javaagent.version}</version>
+      </dependency>
     </dependencies>
 
   </dependencyManagement>

ratis-assembly/pom.xml

@@ -289,6 +289,11 @@
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-shell</artifactId>
     </dependency>
+    <!-- Include OpenTelemetry agent -->
+    <dependency>
+      <groupId>io.opentelemetry.javaagent</groupId>
+      <artifactId>opentelemetry-javaagent</artifactId>
+    </dependency>
   </dependencies>
 
   <profiles>

ratis-assembly/src/main/assembly/bin.xml

@@ -33,6 +33,12 @@
       </includes>
       <outputDirectory>examples/lib</outputDirectory>
     </dependencySet>
+    <dependencySet>
+      <outputDirectory>lib/trace</outputDirectory>
+      <includes>
+        <include>io.opentelemetry.javaagent:*</include>
+      </includes>
+    </dependencySet>
   </dependencySets>
   <moduleSets>
     <moduleSet>

ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java

@@ -44,6 +44,7 @@
 import org.apache.ratis.thirdparty.com.google.common.cache.Cache;
 import org.apache.ratis.thirdparty.com.google.common.cache.CacheBuilder;
 import org.apache.ratis.trace.TraceUtils;
+import org.apache.ratis.trace.otel.OTelTraceUtils;
 import org.apache.ratis.util.CollectionUtils;
 import org.apache.ratis.util.IOUtils;
 import org.apache.ratis.util.JavaUtils;
@@ -292,6 +293,9 @@ RaftClientRequest newRaftClientRequest(
       b.setLeaderId(getLeaderId())
        .setRepliedCallIds(repliedCallIds.get(callId));
     }
+    if (TraceUtils.isEnabled()) {
+      b.setSpanContext(OTelTraceUtils.injectContextToProto());
+    }
     return b.setClientId(clientId)
         .setGroupId(groupId)
         .setCallId(callId)

ratis-common/src/main/java/org/apache/ratis/trace/otel/OTelTraceUtils.java

@@ -32,6 +32,10 @@ public final class OTelTraceUtils {
   private OTelTraceUtils() {
   }
 
+  public static SpanContextProto injectContextToProto() {
+    return injectContextToProto(Context.current());
+  }
+
   public static SpanContextProto injectContextToProto(Context context) {
     final Map<String, String> carrier = new TreeMap<>();
     getTextMapPropagator().inject(context, carrier, (map, key, value) -> map.put(key, value));

ratis-examples/README.md

@@ -146,3 +146,53 @@ by using the `run_all_tests.sh` script found in [dev-support/vagrant/](../dev-su
 See the [dev-support/vagrant/README.md](../dev-support/vagrant/README.md) for more on dependencies and what is setup.
 This will allow one to try a fully setup three server Ratis cluster on a single VM image,
 preventing resource contention with your development host and allowing failure injection too.
+
+
+## Enable Tracing when testing examples
+Ratis uses OpenTelemetry to provide distributed tracing of requests across the cluster.
+Tracing is configured through `RaftProperties`; Ratis itself does not read JVM system
+properties for configuration.
+
+The example shell scripts pass `-Draft.otel.tracing.enabled=true` as a JVM option when the
+OpenTelemetry agent is found. `Runner` reads that property through `ExampleLauncher` and
+applies it to `RaftProperties` before starting an example (e.g. "filestore" or
+"arithmetic").
+
+For example, to enable tracing for the FileStore server, you can run the following command:
+
+```bash
+# build the assembly jar first
+mvn clean package -DskipTests
+
+# extract the assembly jar, mainly we need the dependencies for tracing
+pushd ratis-assembly/target
+file=$(ls -1t ratis-assembly-*.tar.gz | head -n1)
+tar -zxvf "$file"
+popd
+
+# run the server, the tracing will be enabled because it found the 
+# opentelemetry agent jar
+BIN=ratis-examples/src/main/bin
+PEERS=n0:127.0.0.1:6000,n1:127.0.0.1:6001,n2:127.0.0.1:6002
+
+ID=n0; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS}
+ID=n1; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS}
+ID=n2; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS}
+
+# some message should be shown
+[otel.javaagent 2026-04-15 15:55:11:320 -0700] [main] INFO io.opentelemetry.javaagent.tooling.VersionLogger - opentelemetry-javaagent - version: 2.26.1
+```
+
+If you want to configure the tracing further, you can set the following system properties
+
+```bash
+# below is an example to configure the OTLP exporter to send the traces to a local collector,
+# you can change the endpoint and protocol as needed. ExampleLauncher reads
+# -Draft.otel.tracing.enabled=true and applies it to RaftProperties.
+export OTEL_EXPORTER_OPTS="-Dotel.traces.exporter=otlp \
+-Dotel.exporter.otlp.protocol=grpc \
+-Dotel.exporter.otlp.endpoint=http://localhost:4317 \
+-Dotel.metrics.exporter=none \
+-Dotel.logs.exporter=none \
+-Draft.otel.tracing.enabled=true"
+```

ratis-examples/src/main/bin/client.sh

@@ -23,7 +23,8 @@ if [ "$#" -lt 2 ]; then
   exit 1
 fi
 
-source $DIR/common.sh
+OTEL_SERVICE_NAME=ratis.client
+source "${DIR}/common.sh"
 
 # One of the examples, e.g. "filestore" or "arithmetic"
 example="$1"
@@ -32,4 +33,4 @@ shift
 subcommand="$1"
 shift
 
-java ${LOGGER_OPTS} -jar $ARTIFACT "$example" "$subcommand" "$@"
+java ${OTEL_OPTS} ${LOGGER_OPTS} -jar $ARTIFACT "$example" "$subcommand" "$@"

ratis-examples/src/main/bin/common.sh

@@ -46,3 +46,43 @@ if [[ -d "${CONF_DIR}" ]]; then
 else
   LOGGER_OPTS="-Dlog4j.configuration=file:${DIR}/../resources/log4j.properties"
 fi
+
+
+# for opentelemetry: release tarball uses lib/trace next to examples/;
+# dev tree uses ratis-assembly/target/apache-ratis-*-bin/lib/trace after package.
+if [[ -n "${OTEL_JAR:-}" && -f "${OTEL_JAR}" ]]; then
+  : # use OTEL_JAR from environment
+else
+  otel_jar_candidates=()
+  for _jar in "${SCRIPT_DIR}"/../../lib/trace/opentelemetry-javaagent*.jar; do
+    [[ -f "${_jar}" ]] && otel_jar_candidates+=("${_jar}")
+  done
+  for _otel_trace_dir in "${SCRIPT_DIR}"/../../../../ratis-assembly/target/apache-ratis-*-bin/lib/trace; do
+    [[ -d "${_otel_trace_dir}" ]] || continue
+    for _jar in "${_otel_trace_dir}"/opentelemetry-javaagent*.jar; do
+      [[ -f "${_jar}" ]] && otel_jar_candidates+=("${_jar}")
+    done
+  done
+  if ((${#otel_jar_candidates[@]} > 0)); then
+    OTEL_JAR="$(printf '%s\n' "${otel_jar_candidates[@]}" | sort -V | tail -n 1)"
+  else
+    echo "Warning: OpenTelemetry agent jar not found; OTEL disabled"
+    OTEL_JAR=""
+  fi
+fi
+
+OTEL_SERVICE_NAME="${OTEL_SERVICE_NAME:-ratis.server}"
+OTEL_DEFAULT_EXPORTER_OPTS="-Dotel.traces.exporter=logging \
+-Dotel.metrics.exporter=none \
+-Dotel.logs.exporter=logging"
+# Unset or empty → default exporter flags (empty is not a supported override).
+OTEL_EXPORTER_OPTS="${OTEL_EXPORTER_OPTS:-${OTEL_DEFAULT_EXPORTER_OPTS}}"
+
+# Enable javaagent when OTEL_JAR resolves to a file (env or discovery above).
+if [[ -n "${OTEL_JAR}" && -f "${OTEL_JAR}" ]]; then
+  OTEL_OPTS="-javaagent:${OTEL_JAR} -Dotel.resource.attributes=service.name=${OTEL_SERVICE_NAME} ${OTEL_EXPORTER_OPTS} -Draft.otel.tracing.enabled=true"
+else
+  OTEL_OPTS=""
+fi
+
+echo "Using OTEL_OPTS: ${OTEL_OPTS}"

ratis-examples/src/main/bin/server.sh

@@ -16,5 +16,4 @@
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )"
 source $DIR/common.sh
-
-java ${LOGGER_OPTS} -jar $ARTIFACT "$@"
+java ${OTEL_OPTS} ${LOGGER_OPTS} -jar $ARTIFACT "$@"

ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/cli/Client.java

@@ -37,7 +37,7 @@ public abstract class Client extends SubCommandBase {
 
   @Override
   public void run() throws Exception {
-    RaftProperties raftProperties = new RaftProperties();
+    RaftProperties raftProperties = newRaftProperties();
 
     final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())),
             getPeers());

ratis-examples/src/main/java/org/apache/ratis/examples/arithmetic/cli/Server.java

@@ -56,7 +56,7 @@ public class Server extends SubCommandBase {
   @Override
   public void run() throws Exception {
     RaftPeerId peerId = RaftPeerId.valueOf(id);
-    RaftProperties properties = new RaftProperties();
+    RaftProperties properties = newRaftProperties();
 
     final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
     GrpcConfigKeys.Server.setPort(properties, port);

ratis-examples/src/main/java/org/apache/ratis/examples/common/ExampleLauncher.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.examples.common;
+
+import org.apache.ratis.conf.RaftProperties;
+import org.apache.ratis.trace.TraceConfigKeys;
+import org.apache.ratis.util.StringUtils;
+
+/**
+ * Applies example-only JVM system properties to {@link RaftProperties}.
+ * {@link Runner} calls {@link #init()} at startup; example subcommands obtain
+ * configured properties via {@link #newRaftProperties()}.
+ */
+public final class ExampleLauncher {
+
+  private static volatile boolean initialized;
+  private static Boolean tracingEnabled;
+
+  private ExampleLauncher() {
+  }
+
+  /** Reads example JVM system properties. Called once from {@link Runner#main}. */
+  public static void init() {
+    initFromSystemProperties();
+  }
+
+  /**
+   * Creates {@link RaftProperties} for examples, applying any tracing configuration
+   * read from the JVM system property {@value TraceConfigKeys#ENABLED_KEY}.
+   */
+  public static RaftProperties newRaftProperties() {
+    initFromSystemProperties();
+    final RaftProperties properties = new RaftProperties();
+    if (tracingEnabled != null) {
+      TraceConfigKeys.setEnabled(properties, tracingEnabled);
+    }
+    return properties;
+  }
+
+  private static void initFromSystemProperties() {
+    if (initialized) {
+      return;
+    }
+    synchronized (ExampleLauncher.class) {
+      if (initialized) {
+        return;
+      }
+      final String value = System.getProperty(TraceConfigKeys.ENABLED_KEY);
+      if (value != null) {
+        tracingEnabled = StringUtils.string2boolean(value.trim(), TraceConfigKeys.ENABLED_DEFAULT);
+      }
+      initialized = true;
+    }
+  }
+}

ratis-examples/src/main/java/org/apache/ratis/examples/common/Runner.java

@@ -40,6 +40,7 @@ public static void main(String[] args) throws Exception {
       System.err.println("No command type specified: ");
       return;
     }
+    ExampleLauncher.init();
     List<SubCommandBase> commands = initializeCommands(args[0]);
     Runner runner = new Runner();
 

ratis-examples/src/main/java/org/apache/ratis/examples/common/SubCommandBase.java

@@ -18,6 +18,7 @@
 package org.apache.ratis.examples.common;
 
 import com.beust.jcommander.Parameter;
+import org.apache.ratis.conf.RaftProperties;
 import org.apache.ratis.protocol.RaftPeer;
 import org.apache.ratis.protocol.RaftPeerId;
 import org.apache.ratis.protocol.RoutingTable;
@@ -73,6 +74,10 @@ public RaftPeer getPrimary() {
 
   public abstract void run() throws Exception;
 
+  protected RaftProperties newRaftProperties() {
+    return ExampleLauncher.newRaftProperties();
+  }
+
   public String getRaftGroupId() {
     return raftGroupId;
   }

ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Client.java

@@ -94,7 +94,7 @@ public int getNumFiles() {
   @Override
   public void run() throws Exception {
     int raftSegmentPreallocatedSize = 1024 * 1024 * 1024;
-    RaftProperties raftProperties = new RaftProperties();
+    RaftProperties raftProperties = newRaftProperties();
     RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC);
     GrpcConfigKeys.setMessageSizeMax(raftProperties,
         SizeInBytes.valueOf(raftSegmentPreallocatedSize));

ratis-examples/src/main/java/org/apache/ratis/examples/filestore/cli/Server.java

@@ -77,7 +77,7 @@ public void run() throws Exception {
     JvmMetrics.initJvmMetrics(TimeDuration.valueOf(10, TimeUnit.SECONDS));
 
     RaftPeerId peerId = RaftPeerId.valueOf(id);
-    RaftProperties properties = new RaftProperties();
+    RaftProperties properties = newRaftProperties();
 
     // Avoid leader change affect the performance
     RaftServerConfigKeys.Rpc.setTimeoutMin(properties, TimeDuration.valueOf(2, TimeUnit.SECONDS));