From 2a7cdf6bee2e49429b7c10acf25eee3006cfd294 Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Wed, 14 Oct 2026 10:16:39 -0400 Subject: [PATCH 1/6] Add OpenTelemetry v2 plugin --- CHANGELOG.md | 3 + .../temporal-opentelemetry-v2/build.gradle | 28 + ...penTelemetryActivityClientInterceptor.java | 20 + .../v2/OpenTelemetryClientInterceptor.java | 20 + .../OpenTelemetryNexusClientInterceptor.java | 19 + .../opentelemetry/v2/OpenTelemetryPlugin.java | 83 ++ ...penTelemetryScheduleClientInterceptor.java | 20 + .../v2/OpenTelemetryWorkerInterceptor.java | 35 + .../v2/ReplaySafeOpenTelemetry.java | 225 ++++ .../v2/internal/InterceptorTracer.java | 184 +++ ...lemetryActivityClientCallsInterceptor.java | 28 + ...emetryActivityInboundCallsInterceptor.java | 43 + .../OpenTelemetryContextPropagator.java | 47 + ...nTelemetryNexusClientCallsInterceptor.java | 31 + ...NexusOperationInboundCallsInterceptor.java | 58 + ...exusOperationOutboundCallsInterceptor.java | 12 + ...lemetryScheduleClientCallsInterceptor.java | 78 ++ ...lemetryWorkflowClientCallsInterceptor.java | 121 ++ ...emetryWorkflowInboundCallsInterceptor.java | 89 ++ ...metryWorkflowOutboundCallsInterceptor.java | 236 ++++ .../v2/internal/ReplaySafeIdGenerator.java | 93 ++ .../v2/internal/ReplaySafeSpan.java | 100 ++ .../v2/internal/ReplaySafeTracer.java | 130 ++ .../opentelemetry/v2/internal/SpanCodec.java | 136 +++ .../opentelemetry/v2/internal/TagKeys.java | 19 + .../v2/internal/TemporalContextStorage.java | 59 + .../v2/ContextStorageWrapperTest.java | 46 + .../opentelemetry/v2/InterceptorTest.java | 393 +++++++ .../v2/OpenTelemetryPluginGlobalTest.java | 44 + .../opentelemetry/v2/OtelTestBase.java | 148 +++ .../opentelemetry/v2/PropagationTest.java | 569 +++++++++ .../v2/ReplaySafeOpenTelemetryTest.java | 19 + .../opentelemetry/v2/TestWorkflows.java | 1045 +++++++++++++++++ .../temporal/opentelemetry/v2/TracerTest.java | 281 +++++ .../opentelemetry/v2/internal/CodecTest.java | 172 +++ settings.gradle | 2 + temporal-bom/build.gradle | 1 + 37 files changed, 4637 insertions(+) create mode 100644 contrib/temporal-opentelemetry-v2/build.gradle create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryActivityClientInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryClientInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryNexusClientInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryPlugin.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryScheduleClientInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryWorkerInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityClientCallsInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityInboundCallsInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryContextPropagator.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusClientCallsInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationInboundCallsInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationOutboundCallsInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowClientCallsInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowInboundCallsInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowOutboundCallsInterceptor.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeIdGenerator.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TagKeys.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TemporalContextStorage.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ContextStorageWrapperTest.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/InterceptorTest.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OpenTelemetryPluginGlobalTest.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/PropagationTest.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetryTest.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TestWorkflows.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TracerTest.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/CodecTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 344079ce48..4efa44121c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,9 @@ to docs, or any other relevant information. - Added experimental `ActivityClientPlugin` and `NexusClientPlugin`, which plugins on the service stubs use to configure `ActivityClient` and `NexusClient`. `SimplePlugin` implements both, and its builder gained `addScheduleClientInterceptors`, `addActivityClientInterceptors`, and `addNexusClientInterceptors`. +- Added the experimental `temporal-opentelemetry-v2` module. `OpenTelemetryPlugin` propagates OpenTelemetry context + through Temporal headers and can optionally create spans for Temporal operations. `ReplaySafeOpenTelemetry` makes + workflow spans replay safe with deterministic span and trace IDs. ### Changed - Moved the Spring AI integration to [temporalio/ai-integrations](https://github.com/temporalio/ai-integrations/tree/main/java/spring-ai). diff --git a/contrib/temporal-opentelemetry-v2/build.gradle b/contrib/temporal-opentelemetry-v2/build.gradle new file mode 100644 index 0000000000..3fb51d1651 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/build.gradle @@ -0,0 +1,28 @@ +description = '''Temporal Java SDK OpenTelemetry v2 Module''' + +ext { + otelVersion = '1.66.0' +} + +dependencies { + api platform("io.opentelemetry:opentelemetry-bom:$otelVersion") + + compileOnly project(':temporal-serviceclient') + compileOnly project(':temporal-sdk') + compileOnly "javax.annotation:javax.annotation-api:$annotationApiVersion" + + implementation "com.google.guava:guava:$guavaVersion" + + api "io.opentelemetry:opentelemetry-api" + api "io.opentelemetry:opentelemetry-sdk-trace" + api "io.opentelemetry:opentelemetry-sdk-metrics" + api "io.opentelemetry:opentelemetry-sdk-logs" + + testImplementation project(':temporal-sdk') + testImplementation project(':temporal-serviceclient') + testImplementation project(':temporal-testing') + testImplementation "io.opentelemetry:opentelemetry-sdk-testing" + testImplementation "junit:junit:${junitVersion}" + + testRuntimeOnly group: 'ch.qos.logback', name: 'logback-classic', version: "${logbackVersion}" +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryActivityClientInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryActivityClientInterceptor.java new file mode 100644 index 0000000000..3a06c5bde9 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryActivityClientInterceptor.java @@ -0,0 +1,20 @@ +package io.temporal.opentelemetry.v2; + +import io.temporal.common.interceptors.ActivityClientCallsInterceptor; +import io.temporal.common.interceptors.ActivityClientInterceptorBase; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryActivityClientCallsInterceptor; + +public class OpenTelemetryActivityClientInterceptor extends ActivityClientInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryActivityClientInterceptor(InterceptorTracer tracer) { + this.tracer = tracer; + } + + @Override + public ActivityClientCallsInterceptor activityClientCallsInterceptor( + ActivityClientCallsInterceptor next) { + return new OpenTelemetryActivityClientCallsInterceptor(tracer, next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryClientInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryClientInterceptor.java new file mode 100644 index 0000000000..fb7af88eab --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryClientInterceptor.java @@ -0,0 +1,20 @@ +package io.temporal.opentelemetry.v2; + +import io.temporal.common.interceptors.WorkflowClientCallsInterceptor; +import io.temporal.common.interceptors.WorkflowClientInterceptorBase; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryWorkflowClientCallsInterceptor; + +public class OpenTelemetryClientInterceptor extends WorkflowClientInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryClientInterceptor(InterceptorTracer tracer) { + this.tracer = tracer; + } + + @Override + public WorkflowClientCallsInterceptor workflowClientCallsInterceptor( + WorkflowClientCallsInterceptor next) { + return new OpenTelemetryWorkflowClientCallsInterceptor(tracer, next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryNexusClientInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryNexusClientInterceptor.java new file mode 100644 index 0000000000..46853d0bc3 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryNexusClientInterceptor.java @@ -0,0 +1,19 @@ +package io.temporal.opentelemetry.v2; + +import io.temporal.common.interceptors.NexusClientCallsInterceptor; +import io.temporal.common.interceptors.NexusClientInterceptorBase; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryNexusClientCallsInterceptor; + +public class OpenTelemetryNexusClientInterceptor extends NexusClientInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryNexusClientInterceptor(InterceptorTracer tracer) { + this.tracer = tracer; + } + + @Override + public NexusClientCallsInterceptor nexusClientCallsInterceptor(NexusClientCallsInterceptor next) { + return new OpenTelemetryNexusClientCallsInterceptor(tracer, next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryPlugin.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryPlugin.java new file mode 100644 index 0000000000..d57ee77efe --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryPlugin.java @@ -0,0 +1,83 @@ +package io.temporal.opentelemetry.v2; + +import io.opentelemetry.context.ContextStorage; +import io.temporal.common.Experimental; +import io.temporal.common.SimplePlugin; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryContextPropagator; +import io.temporal.opentelemetry.v2.internal.TemporalContextStorage; + +/** + * OpenTelemetry v2 plugin for Temporal clients and workers. + * + *

Set it on {@code WorkflowServiceStubsOptions} or {@code WorkflowClientOptions}. The SDK + * propagates plugins down that chain. + * + *

Register a {@link ReplaySafeOpenTelemetry} with {@code GlobalOpenTelemetry.set} before calling + * {@link Builder#build()}. + */ +@Experimental +public final class OpenTelemetryPlugin extends SimplePlugin { + public static final String NAME = "io.temporal.opentelemetry.v2"; + + private OpenTelemetryPlugin(InterceptorTracer tracer, String headerKey) { + super( + SimplePlugin.newBuilder(NAME) + .addClientInterceptors(new OpenTelemetryClientInterceptor(tracer)) + .addScheduleClientInterceptors(new OpenTelemetryScheduleClientInterceptor(tracer)) + .addActivityClientInterceptors(new OpenTelemetryActivityClientInterceptor(tracer)) + .addNexusClientInterceptors(new OpenTelemetryNexusClientInterceptor(tracer)) + .addWorkerInterceptors(new OpenTelemetryWorkerInterceptor(tracer)) + .addContextPropagators(new OpenTelemetryContextPropagator(headerKey))); + } + + public static Builder newBuilder() { + return new Builder(); + } + + /** Every option is optional; an unset one keeps its default. */ + public static final class Builder { + private String headerKey = "_tracer-data"; + private boolean addTemporalSpans; + + private Builder() {} + + /** + * The Temporal header key to serialize the span to. Defaults to {@code _tracer-data}, which + * Temporal uses; overriding it breaks trace continuity with workers using the standard key. + */ + public Builder setHeaderKey(String headerKey) { + this.headerKey = headerKey; + return this; + } + + /** + * Whether to create spans for Temporal operations such as StartWorkflow, RunWorkflow, and + * RunActivity. Defaults to false: trace context still propagates through Temporal headers, so + * spans created by application code remain connected. + */ + public Builder setAddTemporalSpans(boolean addTemporalSpans) { + this.addTemporalSpans = addTemporalSpans; + return this; + } + + public OpenTelemetryPlugin build() { + if (!ReplaySafeOpenTelemetry.isRegisteredGlobally()) { + throw new IllegalStateException( + "the global OpenTelemetry must be a ReplaySafeOpenTelemetry; build one with " + + "ReplaySafeOpenTelemetry.newBuilder() and register it with " + + "GlobalOpenTelemetry.set before building this plugin"); + } + // Initializing the storage applies every registered wrapper, and a wrapper registered after + // ours ends up outermost, so check that ours was applied rather than what is outermost. + ContextStorage.get(); + if (!TemporalContextStorage.isInstalled()) { + throw new IllegalStateException( + "the OpenTelemetry ContextStorage initialized before ReplaySafeOpenTelemetry could " + + "wrap it; build ReplaySafeOpenTelemetry before anything uses the OpenTelemetry " + + "Context"); + } + return new OpenTelemetryPlugin(new InterceptorTracer(headerKey, addTemporalSpans), headerKey); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryScheduleClientInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryScheduleClientInterceptor.java new file mode 100644 index 0000000000..01ab09bad2 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryScheduleClientInterceptor.java @@ -0,0 +1,20 @@ +package io.temporal.opentelemetry.v2; + +import io.temporal.common.interceptors.ScheduleClientCallsInterceptor; +import io.temporal.common.interceptors.ScheduleClientInterceptorBase; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryScheduleClientCallsInterceptor; + +public class OpenTelemetryScheduleClientInterceptor extends ScheduleClientInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryScheduleClientInterceptor(InterceptorTracer tracer) { + this.tracer = tracer; + } + + @Override + public ScheduleClientCallsInterceptor scheduleClientCallsInterceptor( + ScheduleClientCallsInterceptor next) { + return new OpenTelemetryScheduleClientCallsInterceptor(tracer, next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryWorkerInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryWorkerInterceptor.java new file mode 100644 index 0000000000..cf7d414f4a --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryWorkerInterceptor.java @@ -0,0 +1,35 @@ +package io.temporal.opentelemetry.v2; + +import io.nexusrpc.handler.OperationContext; +import io.temporal.common.interceptors.ActivityInboundCallsInterceptor; +import io.temporal.common.interceptors.NexusOperationInboundCallsInterceptor; +import io.temporal.common.interceptors.WorkerInterceptor; +import io.temporal.common.interceptors.WorkflowInboundCallsInterceptor; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryActivityInboundCallsInterceptor; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryNexusOperationInboundCallsInterceptor; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryWorkflowInboundCallsInterceptor; + +public class OpenTelemetryWorkerInterceptor implements WorkerInterceptor { + private final InterceptorTracer tracer; + + public OpenTelemetryWorkerInterceptor(InterceptorTracer tracer) { + this.tracer = tracer; + } + + @Override + public WorkflowInboundCallsInterceptor interceptWorkflow(WorkflowInboundCallsInterceptor next) { + return new OpenTelemetryWorkflowInboundCallsInterceptor(tracer, next); + } + + @Override + public ActivityInboundCallsInterceptor interceptActivity(ActivityInboundCallsInterceptor next) { + return new OpenTelemetryActivityInboundCallsInterceptor(tracer, next); + } + + @Override + public NexusOperationInboundCallsInterceptor interceptNexusOperation( + OperationContext context, NexusOperationInboundCallsInterceptor next) { + return new OpenTelemetryNexusOperationInboundCallsInterceptor(tracer, next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java new file mode 100644 index 0000000000..15646a212c --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java @@ -0,0 +1,225 @@ +package io.temporal.opentelemetry.v2; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; +import io.opentelemetry.api.logs.LoggerProvider; +import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerBuilder; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.SdkLoggerProviderBuilder; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.trace.IdGenerator; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.temporal.common.Experimental; +import io.temporal.opentelemetry.v2.internal.ReplaySafeIdGenerator; +import io.temporal.opentelemetry.v2.internal.ReplaySafeTracer; +import io.temporal.opentelemetry.v2.internal.TemporalContextStorage; +import java.io.Closeable; +import javax.annotation.Nonnull; + +/** + * The {@link OpenTelemetry} to use for OpenTelemetry integration with Temporal. Register it with + * {@code GlobalOpenTelemetry.set}; tracers obtained from it are replay safe inside workflows. + * + *

Known limitations: + * + *

+ */ +@Experimental +public final class ReplaySafeOpenTelemetry implements OpenTelemetry, Closeable { + private final ReplaySafeTracerProvider tracerProvider; + private final SdkMeterProvider meterProvider; // TODO - Make the meter provider replay safe + // TODO: Make the logger provider replay safe and add logger interceptor methods for Temporal and + // OpenTelemetry loggers. + private final SdkLoggerProvider loggerProvider; + private final ContextPropagators propagators; + + static { + // A wrapper, so Temporal's storage works alongside any storage the application already uses. + ContextStorage.addWrapper(TemporalContextStorage::new); + } + + private ReplaySafeOpenTelemetry(Builder builder) { + this.tracerProvider = + new ReplaySafeTracerProvider( + builder + .tracerProviderBuilder + .setIdGenerator(new ReplaySafeIdGenerator(builder.idGenerator)) + .build()); + this.meterProvider = builder.meterProviderBuilder.build(); + this.loggerProvider = builder.loggerProviderBuilder.build(); + this.propagators = builder.propagators; + } + + public static Builder newBuilder() { + return new Builder(); + } + + @Override + public TracerProvider getTracerProvider() { + return tracerProvider; + } + + @Override + public MeterProvider getMeterProvider() { + return meterProvider; + } + + @Override + public LoggerProvider getLogsBridge() { + return loggerProvider; + } + + @Override + public ContextPropagators getPropagators() { + return propagators; + } + + static boolean isRegisteredGlobally() { + return GlobalOpenTelemetry.getOrNoop().getTracerProvider() instanceof ReplaySafeTracerProvider; + } + + /** Shuts down every provider. Call after the clients and workers using them have stopped. */ + @Override + public void close() { + tracerProvider.close(); + meterProvider.close(); + loggerProvider.close(); + } + + /** Every provider is optional; an unset one is built from the SDK's default builder. */ + public static final class Builder { + private SdkTracerProviderBuilder tracerProviderBuilder = SdkTracerProvider.builder(); + private IdGenerator idGenerator = IdGenerator.random(); + private SdkMeterProviderBuilder meterProviderBuilder = SdkMeterProvider.builder(); + private SdkLoggerProviderBuilder loggerProviderBuilder = SdkLoggerProvider.builder(); + private ContextPropagators propagators = + ContextPropagators.create( + TextMapPropagator.composite( + W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance())); + + private Builder() {} + + /** + * Any {@link IdGenerator} set on {@code tracerProviderBuilder} is replaced; use {@link + * #setIdGenerator} instead. + */ + public Builder setTracerProviderBuilder(SdkTracerProviderBuilder tracerProviderBuilder) { + this.tracerProviderBuilder = tracerProviderBuilder; + return this; + } + + /** + * Generates span and trace IDs outside workflow code. Workflow spans always use replay safe + * IDs. Defaults to {@link IdGenerator#random()}. + */ + public Builder setIdGenerator(IdGenerator idGenerator) { + this.idGenerator = idGenerator; + return this; + } + + public Builder setMeterProviderBuilder(SdkMeterProviderBuilder meterProviderBuilder) { + this.meterProviderBuilder = meterProviderBuilder; + return this; + } + + public Builder setLoggerProviderBuilder(SdkLoggerProviderBuilder loggerProviderBuilder) { + this.loggerProviderBuilder = loggerProviderBuilder; + return this; + } + + /** + * The propagators returned by {@link ReplaySafeOpenTelemetry#getPropagators()}. Defaults to W3C + * trace context plus baggage, which is what {@link OpenTelemetryPlugin} serializes into + * Temporal headers when it is left to resolve its propagator from the global. + */ + public Builder setPropagators(ContextPropagators propagators) { + this.propagators = propagators; + return this; + } + + public ReplaySafeOpenTelemetry build() { + return new ReplaySafeOpenTelemetry(this); + } + } + + private static final class ReplaySafeTracerProvider implements TracerProvider, Closeable { + private final SdkTracerProvider delegate; + + private ReplaySafeTracerProvider(SdkTracerProvider delegate) { + this.delegate = delegate; + } + + @Override + public Tracer get(@Nonnull String instrumentationScopeName) { + return new ReplaySafeTracer(delegate.get(instrumentationScopeName), instrumentationScopeName); + } + + @Override + public Tracer get( + @Nonnull String instrumentationScopeName, @Nonnull String instrumentationScopeVersion) { + return new ReplaySafeTracer( + delegate.get(instrumentationScopeName, instrumentationScopeVersion), + instrumentationScopeName); + } + + @Override + public TracerBuilder tracerBuilder(@Nonnull String instrumentationScopeName) { + return new ReplaySafeTracerBuilder( + delegate.tracerBuilder(instrumentationScopeName), instrumentationScopeName); + } + + @Override + public void close() { + delegate.close(); + } + } + + private static final class ReplaySafeTracerBuilder implements TracerBuilder { + private final TracerBuilder delegate; + private final String instrumentationScopeName; + + private ReplaySafeTracerBuilder(TracerBuilder delegate, String instrumentationScopeName) { + this.delegate = delegate; + this.instrumentationScopeName = instrumentationScopeName; + } + + @Override + public TracerBuilder setSchemaUrl(@Nonnull String schemaUrl) { + delegate.setSchemaUrl(schemaUrl); + return this; + } + + @Override + public TracerBuilder setInstrumentationVersion(@Nonnull String instrumentationScopeVersion) { + delegate.setInstrumentationVersion(instrumentationScopeVersion); + return this; + } + + @Override + public Tracer build() { + return new ReplaySafeTracer(delegate.build(), instrumentationScopeName); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java new file mode 100644 index 0000000000..dad1a81d95 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java @@ -0,0 +1,184 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.temporal.common.interceptors.Header; +import io.temporal.failure.ApplicationErrorCategory; +import io.temporal.failure.ApplicationFailure; +import io.temporal.internal.sync.DestroyWorkflowThreadError; +import java.util.List; +import java.util.Map; + +/** Wraps intercepted Temporal calls in a span and propagates it through headers. */ +public final class InterceptorTracer { + private static final String INSTRUMENTATION_NAME = "temporal-sdk-java"; + + private final Tracer tracer; + private final SpanCodec codec; + private final boolean addTemporalSpans; + + public InterceptorTracer(String headerKey, boolean addTemporalSpans) { + this.tracer = GlobalOpenTelemetry.getTracer(INSTRUMENTATION_NAME); + this.codec = new SpanCodec(headerKey); + this.addTemporalSpans = addTemporalSpans; + } + + /** + * An intercepted call. {@code E} is the checked exception it declares, or {@link + * RuntimeException} when it declares none. + */ + @FunctionalInterface + interface Call { + R call() throws E; + } + + /** An intercepted call with no result. */ + @FunctionalInterface + interface VoidCall { + void call() throws E; + } + + R traceInbound( + String operation, String name, Attributes attributes, Header header, Call call) + throws E { + return traceInbound(operation, name, attributes, codec.read(header), call); + } + + void traceInbound( + String operation, String name, Attributes attributes, Header header, VoidCall call) + throws E { + traceInbound(operation, name, attributes, codec.read(header), asCall(call)); + } + + R traceNexusInbound( + String operation, + String name, + Attributes attributes, + Map nexusHeaders, + Call call) + throws E { + return traceInbound(operation, name, attributes, codec.read(nexusHeaders), call); + } + + private R traceInbound( + String operation, String name, Attributes attributes, Context parent, Call call) + throws E { + try (Scope ignoredParent = parent.makeCurrent()) { + if (!addTemporalSpans) { + return call.call(); + } + + return run(startSpan(operation, name, attributes, SpanKind.SERVER), call); + } + } + + R traceOutbound( + String operation, String name, Attributes attributes, Header header, Call call) + throws E { + return traceOutbound(operation, name, attributes, () -> codec.write(header), call); + } + + void injectOutboundHeader(Header header) { + codec.write(header); + } + + R traceOutbound( + String operation, String name, Attributes attributes, Call call) throws E { + return traceOutbound(operation, name, attributes, () -> {}, call); + } + + void traceOutbound( + String operation, String name, Attributes attributes, Header header, VoidCall call) + throws E { + traceOutbound(operation, name, attributes, () -> codec.write(header), asCall(call)); + } + + R traceOutbound( + String operation, String name, Attributes attributes, List
headers, Call call) + throws E { + return traceOutbound(operation, name, attributes, () -> headers.forEach(codec::write), call); + } + + R traceNexusOutbound( + String operation, + String name, + Attributes attributes, + Map nexusHeaders, + Call call) + throws E { + return traceOutbound(operation, name, attributes, () -> codec.write(nexusHeaders), call); + } + + private R traceOutbound( + String operation, String name, Attributes attributes, Runnable writeHeader, Call call) + throws E { + if (!addTemporalSpans) { + writeHeader.run(); + return call.call(); + } + + return run( + startSpan(operation, name, attributes, SpanKind.CLIENT), + () -> { + writeHeader.run(); + return call.call(); + }); + } + + private static Call asCall(VoidCall call) { + return () -> { + call.call(); + return null; + }; + } + + /** Runs {@code call} in {@code span}, recording any failure, then ends the span. */ + private static R run(Span span, Call call) throws E { + try (Scope ignored = span.makeCurrent()) { + return call.call(); + } catch (DestroyWorkflowThreadError unwind) { + // Eviction and continue-as-new unwind the thread; that is not a failure of the span. + throw unwind; + } catch (Throwable failure) { + span.recordException(failure); + if (!isBenign(failure)) { + span.setStatus(StatusCode.ERROR, failure.toString()); + } + throw failure; + } finally { + span.end(); + } + } + + private Span startSpan(String operation, String name, Attributes attributes, SpanKind kind) { + try (Scope ignored = + Context.current().with(ReplaySafeIdGenerator.INTERCEPTOR_SPAN, true).makeCurrent()) { + return tracer + .spanBuilder(spanName(operation, name)) + .setSpanKind(kind) + .setAllAttributes(attributes) + .startSpan(); + } + } + + static String spanName(String operation, String name) { + if (operation.isEmpty()) { + return name; + } + if (name.isEmpty()) { + return operation; + } + return operation + ":" + name; + } + + private static boolean isBenign(Throwable failure) { + return failure instanceof ApplicationFailure + && ((ApplicationFailure) failure).getCategory() == ApplicationErrorCategory.BENIGN; + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityClientCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityClientCallsInterceptor.java new file mode 100644 index 0000000000..fb901c2f59 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityClientCallsInterceptor.java @@ -0,0 +1,28 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.ACTIVITY_ID; + +import io.opentelemetry.api.common.Attributes; +import io.temporal.common.interceptors.ActivityClientCallsInterceptor; +import io.temporal.common.interceptors.ActivityClientCallsInterceptorBase; + +public class OpenTelemetryActivityClientCallsInterceptor + extends ActivityClientCallsInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryActivityClientCallsInterceptor( + InterceptorTracer tracer, ActivityClientCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public StartActivityOutput startActivity(StartActivityInput input) { + return tracer.traceOutbound( + "StartActivity", + input.getActivityType(), + Attributes.of(ACTIVITY_ID, input.getOptions().getId()), + input.getHeader(), + () -> super.startActivity(input)); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityInboundCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityInboundCallsInterceptor.java new file mode 100644 index 0000000000..6da63b04d6 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityInboundCallsInterceptor.java @@ -0,0 +1,43 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.opentelemetry.api.common.Attributes; +import io.temporal.activity.ActivityExecutionContext; +import io.temporal.activity.ActivityInfo; +import io.temporal.common.interceptors.ActivityInboundCallsInterceptor; +import io.temporal.common.interceptors.ActivityInboundCallsInterceptorBase; + +public class OpenTelemetryActivityInboundCallsInterceptor + extends ActivityInboundCallsInterceptorBase { + private final InterceptorTracer tracer; + // Activity code reaches its context through Activity.getExecutionContext(), but interceptors + // only see it in init. + private ActivityExecutionContext context; + + public OpenTelemetryActivityInboundCallsInterceptor( + InterceptorTracer tracer, ActivityInboundCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public void init(ActivityExecutionContext context) { + this.context = context; + super.init(context); + } + + @Override + public ActivityOutput execute(ActivityInput input) { + ActivityInfo info = context.getInfo(); + return tracer.traceInbound( + "RunActivity", + info.getActivityType(), + Attributes.of( + WORKFLOW_ID, info.getWorkflowId(), + RUN_ID, info.getWorkflowRunId(), + ACTIVITY_ID, info.getActivityId()), + input.getHeader(), + () -> super.execute(input)); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryContextPropagator.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryContextPropagator.java new file mode 100644 index 0000000000..b08c462420 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryContextPropagator.java @@ -0,0 +1,47 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.context.Context; +import io.temporal.api.common.v1.Payload; +import io.temporal.common.context.ContextPropagator; +import java.util.Collections; +import java.util.Map; + +/** + * Copies the current OpenTelemetry context between Temporal workflow threads. The root workflow + * thread starts from the start header, so the workflow constructor already sees the caller's span. + */ +public final class OpenTelemetryContextPropagator implements ContextPropagator { + private final SpanCodec codec; + + public OpenTelemetryContextPropagator(String headerKey) { + this.codec = new SpanCodec(headerKey); + } + + @Override + public String getName() { + return "io.temporal.opentelemetry.v2.workflow-context"; + } + + @Override + public Map serializeContext(Object context) { + if (!(context instanceof Context)) { + return Collections.emptyMap(); + } + return codec.serialize((Context) context); + } + + @Override + public Object deserializeContext(Map header) { + return codec.readFromRoot(header); + } + + @Override + public Object getCurrentContext() { + return Context.current(); + } + + @Override + public void setCurrentContext(Object context) { + TemporalContextStorage.setWorkflowContext((Context) context); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusClientCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusClientCallsInterceptor.java new file mode 100644 index 0000000000..5cd729a725 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusClientCallsInterceptor.java @@ -0,0 +1,31 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.opentelemetry.api.common.Attributes; +import io.temporal.common.interceptors.NexusClientCallsInterceptor; +import io.temporal.common.interceptors.NexusClientCallsInterceptorBase; + +public class OpenTelemetryNexusClientCallsInterceptor extends NexusClientCallsInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryNexusClientCallsInterceptor( + InterceptorTracer tracer, NexusClientCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public StartNexusOperationExecutionOutput startNexusOperationExecution( + StartNexusOperationExecutionInput input) { + return tracer.traceNexusOutbound( + "StartNexusOperation", + input.getService() + "/" + input.getOperation(), + Attributes.of( + NEXUS_ENDPOINT, input.getEndpoint(), + NEXUS_SERVICE, input.getService(), + NEXUS_OPERATION, input.getOperation()), + input.getHeaders(), + () -> super.startNexusOperationExecution(input)); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationInboundCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationInboundCallsInterceptor.java new file mode 100644 index 0000000000..c06203c3cc --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationInboundCallsInterceptor.java @@ -0,0 +1,58 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.nexusrpc.OperationException; +import io.nexusrpc.handler.OperationContext; +import io.opentelemetry.api.common.Attributes; +import io.temporal.common.interceptors.NexusOperationInboundCallsInterceptor; +import io.temporal.common.interceptors.NexusOperationInboundCallsInterceptorBase; +import io.temporal.common.interceptors.NexusOperationOutboundCallsInterceptor; + +public class OpenTelemetryNexusOperationInboundCallsInterceptor + extends NexusOperationInboundCallsInterceptorBase { + + private final InterceptorTracer tracer; + + public OpenTelemetryNexusOperationInboundCallsInterceptor( + InterceptorTracer tracer, NexusOperationInboundCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public void init(NexusOperationOutboundCallsInterceptor outboundCalls) { + super.init(new OpenTelemetryNexusOperationOutboundCallsInterceptor(outboundCalls)); + } + + @Override + public StartOperationOutput startOperation(StartOperationInput input) throws OperationException { + OperationContext context = input.getOperationContext(); + return tracer.traceNexusInbound( + "RunStartNexusOperationHandler", + spanName(context), + nexusTags(context), + context.getHeaders(), + () -> super.startOperation(input)); + } + + @Override + public CancelOperationOutput cancelOperation(CancelOperationInput input) { + OperationContext context = input.getOperationContext(); + return tracer.traceNexusInbound( + "RunCancelNexusOperationHandler", + spanName(context), + nexusTags(context), + context.getHeaders(), + () -> super.cancelOperation(input)); + } + + private static String spanName(OperationContext context) { + return context.getService() + "/" + context.getOperation(); + } + + private static Attributes nexusTags(OperationContext context) { + return Attributes.of( + NEXUS_SERVICE, context.getService(), NEXUS_OPERATION, context.getOperation()); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationOutboundCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationOutboundCallsInterceptor.java new file mode 100644 index 0000000000..c8f1a3e5c8 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationOutboundCallsInterceptor.java @@ -0,0 +1,12 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.temporal.common.interceptors.NexusOperationOutboundCallsInterceptor; +import io.temporal.common.interceptors.NexusOperationOutboundCallsInterceptorBase; + +public class OpenTelemetryNexusOperationOutboundCallsInterceptor + extends NexusOperationOutboundCallsInterceptorBase { + public OpenTelemetryNexusOperationOutboundCallsInterceptor( + NexusOperationOutboundCallsInterceptor next) { + super(next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java new file mode 100644 index 0000000000..8d497b17bd --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java @@ -0,0 +1,78 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.Attributes; +import io.temporal.client.schedules.Schedule; +import io.temporal.client.schedules.ScheduleActionStartWorkflow; +import io.temporal.client.schedules.ScheduleUpdate; +import io.temporal.client.schedules.ScheduleUpdateInput; +import io.temporal.common.interceptors.Header; +import io.temporal.common.interceptors.ScheduleClientCallsInterceptor; +import io.temporal.common.interceptors.ScheduleClientCallsInterceptorBase; +import java.util.HashMap; + +public class OpenTelemetryScheduleClientCallsInterceptor + extends ScheduleClientCallsInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryScheduleClientCallsInterceptor( + InterceptorTracer tracer, ScheduleClientCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public void createSchedule(CreateScheduleInput input) { + if (!(input.getSchedule().getAction() instanceof ScheduleActionStartWorkflow)) { + super.createSchedule(input); + return; + } + + tracer.traceOutbound( + "CreateSchedule", + input.getId(), + Attributes.empty(), + () -> { + super.createSchedule( + new CreateScheduleInput( + input.getId(), withTraceHeader(input.getSchedule()), input.getOptions())); + return null; + }); + } + + @Override + public void updateSchedule(UpdateScheduleInput input) { + tracer.traceOutbound( + "UpdateSchedule", + input.getDescription().getId(), + Attributes.empty(), + () -> { + super.updateSchedule( + new UpdateScheduleInput( + input.getDescription(), updateInput -> applyUpdate(input, updateInput))); + return null; + }); + } + + private ScheduleUpdate applyUpdate(UpdateScheduleInput input, ScheduleUpdateInput updateInput) { + ScheduleUpdate update = input.getUpdater().apply(updateInput); + if (update == null + || !(update.getSchedule().getAction() instanceof ScheduleActionStartWorkflow)) { + return update; + } + return new ScheduleUpdate( + withTraceHeader(update.getSchedule()), update.getTypedSearchAttributes()); + } + + /** + * Writes the current trace into a copy of the action header, so the caller's header, which may be + * immutable or reused, is never modified. + */ + private Schedule withTraceHeader(Schedule schedule) { + ScheduleActionStartWorkflow action = (ScheduleActionStartWorkflow) schedule.getAction(); + Header header = new Header(new HashMap<>(action.getHeader().getValues())); + tracer.injectOutboundHeader(header); + return Schedule.newBuilder(schedule) + .setAction(ScheduleActionStartWorkflow.newBuilder(action).setHeader(header).build()) + .build(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowClientCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowClientCallsInterceptor.java new file mode 100644 index 0000000000..d76609a452 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowClientCallsInterceptor.java @@ -0,0 +1,121 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.temporal.client.WorkflowUpdateHandle; +import io.temporal.common.interceptors.WorkflowClientCallsInterceptor; +import io.temporal.common.interceptors.WorkflowClientCallsInterceptorBase; +import java.util.Arrays; + +public class OpenTelemetryWorkflowClientCallsInterceptor + extends WorkflowClientCallsInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryWorkflowClientCallsInterceptor( + InterceptorTracer tracer, WorkflowClientCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public WorkflowStartOutput start(WorkflowStartInput input) { + return tracer.traceOutbound( + "StartWorkflow", + input.getWorkflowType(), + Attributes.of(WORKFLOW_ID, input.getWorkflowId()), + input.getHeader(), + () -> super.start(input)); + } + + @Override + public WorkflowSignalOutput signal(WorkflowSignalInput input) { + return tracer.traceOutbound( + "SignalWorkflow", + input.getSignalName(), + workflowExecutionTags(input.getWorkflowExecution()), + input.getHeader(), + () -> super.signal(input)); + } + + @Override + public WorkflowSignalWithStartOutput signalWithStart(WorkflowSignalWithStartInput input) { + WorkflowStartInput start = input.getWorkflowStartInput(); + return tracer.traceOutbound( + "SignalWithStartWorkflow", + start.getWorkflowType(), + Attributes.of(WORKFLOW_ID, start.getWorkflowId()), + start.getHeader(), + () -> super.signalWithStart(input)); + } + + @Override + public QueryOutput query(QueryInput input) { + return tracer.traceOutbound( + "QueryWorkflow", + input.getQueryType(), + workflowExecutionTags(input.getWorkflowExecution()), + input.getHeader(), + () -> super.query(input)); + } + + @Override + public WorkflowUpdateHandle startUpdate(StartUpdateInput input) { + AttributesBuilder attributes = workflowExecutionTags(input.getWorkflowExecution()).toBuilder(); + attributes.put(UPDATE_ID, input.getUpdateId()); + return tracer.traceOutbound( + "StartWorkflowUpdate", + input.getUpdateName(), + attributes.build(), + input.getHeader(), + () -> super.startUpdate(input)); + } + + @Override + public CancelOutput cancel(CancelInput input) { + return tracer.traceOutbound( + "CancelWorkflow", + "", + workflowExecutionTags(input.getWorkflowExecution()), + () -> super.cancel(input)); + } + + @Override + public TerminateOutput terminate(TerminateInput input) { + AttributesBuilder attributes = workflowExecutionTags(input.getWorkflowExecution()).toBuilder(); + if (input.getReason() != null) { + attributes.put(TERMINATE_REASON, input.getReason()); + } + return tracer.traceOutbound( + "TerminateWorkflow", "", attributes.build(), () -> super.terminate(input)); + } + + @Override + public DescribeWorkflowOutput describe(DescribeWorkflowInput input) { + return tracer.traceOutbound( + "DescribeWorkflow", + "", + workflowExecutionTags(input.getWorkflowExecution()), + () -> super.describe(input)); + } + + @Override + public WorkflowUpdateWithStartOutput updateWithStart( + WorkflowUpdateWithStartInput input) { + WorkflowStartInput start = input.getWorkflowStartInput(); + StartUpdateInput update = input.getStartUpdateInput(); + // The start header reaches the workflow and the update header reaches the update handler. + return tracer.traceOutbound( + "UpdateWithStartWorkflow", + update.getUpdateName(), + Attributes.of(WORKFLOW_ID, start.getWorkflowId(), UPDATE_ID, update.getUpdateId()), + Arrays.asList(start.getHeader(), update.getHeader()), + () -> super.updateWithStart(input)); + } + + private static Attributes workflowExecutionTags( + io.temporal.api.common.v1.WorkflowExecution execution) { + return Attributes.of(WORKFLOW_ID, execution.getWorkflowId(), RUN_ID, execution.getRunId()); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowInboundCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowInboundCallsInterceptor.java new file mode 100644 index 0000000000..c0d033bde7 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowInboundCallsInterceptor.java @@ -0,0 +1,89 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.temporal.common.interceptors.WorkflowInboundCallsInterceptor; +import io.temporal.common.interceptors.WorkflowInboundCallsInterceptorBase; +import io.temporal.common.interceptors.WorkflowOutboundCallsInterceptor; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInfo; + +public class OpenTelemetryWorkflowInboundCallsInterceptor + extends WorkflowInboundCallsInterceptorBase { + + private final InterceptorTracer tracer; + + public OpenTelemetryWorkflowInboundCallsInterceptor( + InterceptorTracer tracer, WorkflowInboundCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public void init(WorkflowOutboundCallsInterceptor outboundCalls) { + super.init(new OpenTelemetryWorkflowOutboundCallsInterceptor(tracer, outboundCalls)); + } + + @Override + public WorkflowOutput execute(WorkflowInput input) { + return tracer.traceInbound( + "RunWorkflow", + Workflow.getInfo().getWorkflowType(), + workflowTags(), + input.getHeader(), + () -> super.execute(input)); + } + + @Override + public void handleSignal(SignalInput input) { + tracer.traceInbound( + "HandleSignal", + input.getSignalName(), + workflowTags(), + input.getHeader(), + () -> super.handleSignal(input)); + } + + @Override + public QueryOutput handleQuery(QueryInput input) { + return tracer.traceInbound( + "HandleQuery", + input.getQueryName(), + workflowTags(), + input.getHeader(), + () -> super.handleQuery(input)); + } + + @Override + public void validateUpdate(UpdateInput input) { + tracer.traceInbound( + "ValidateUpdate", + input.getUpdateName(), + updateTags(), + input.getHeader(), + () -> super.validateUpdate(input)); + } + + @Override + public UpdateOutput executeUpdate(UpdateInput input) { + return tracer.traceInbound( + "HandleUpdate", + input.getUpdateName(), + updateTags(), + input.getHeader(), + () -> super.executeUpdate(input)); + } + + private static Attributes workflowTags() { + WorkflowInfo info = Workflow.getInfo(); + return Attributes.of(WORKFLOW_ID, info.getWorkflowId(), RUN_ID, info.getRunId()); + } + + private static Attributes updateTags() { + AttributesBuilder tags = workflowTags().toBuilder(); + Workflow.getCurrentUpdateInfo().ifPresent(update -> tags.put(UPDATE_ID, update.getUpdateId())); + return tags.build(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowOutboundCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowOutboundCallsInterceptor.java new file mode 100644 index 0000000000..9c0bad9692 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowOutboundCallsInterceptor.java @@ -0,0 +1,236 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.common.interceptors.WorkflowOutboundCallsInterceptor; +import io.temporal.common.interceptors.WorkflowOutboundCallsInterceptorBase; +import io.temporal.workflow.Functions; +import io.temporal.workflow.Promise; +import io.temporal.workflow.TimerOptions; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInfo; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.annotation.Nullable; + +public class OpenTelemetryWorkflowOutboundCallsInterceptor + extends WorkflowOutboundCallsInterceptorBase { + + private final InterceptorTracer tracer; + + public OpenTelemetryWorkflowOutboundCallsInterceptor( + InterceptorTracer tracer, WorkflowOutboundCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + /** + * Makes the context of the calling workflow code current while a callback runs. The SDK runs + * callbacks registered on an incomplete promise in a separate workflow thread, which would + * otherwise see the workflow's root context. + */ + private static class ContextPromise implements Promise { + private final Promise delegate; + + ContextPromise(Promise delegate) { + this.delegate = delegate; + } + + private static O wrap(Context context, Functions.Func fn) { + try (Scope ignored = context.makeCurrent()) { + return fn.apply(); + } + } + + @Override + public boolean isCompleted() { + return delegate.isCompleted(); + } + + @Override + public R get() { + return delegate.get(); + } + + @Override + public R cancellableGet() { + return delegate.cancellableGet(); + } + + @Override + public R get(long timeout, TimeUnit unit) throws TimeoutException { + return delegate.get(timeout, unit); + } + + @Override + public R cancellableGet(long timeout, TimeUnit unit) throws TimeoutException { + return delegate.cancellableGet(timeout, unit); + } + + @Override + public RuntimeException getFailure() { + return delegate.getFailure(); + } + + @Override + public Promise thenApply(Functions.Func1 fn) { + Context context = Context.current(); + return new ContextPromise<>(delegate.thenApply((r) -> wrap(context, () -> fn.apply(r)))); + } + + @Override + public Promise handle(Functions.Func2 fn) { + Context context = Context.current(); + return new ContextPromise<>(delegate.handle((r, e) -> wrap(context, () -> fn.apply(r, e)))); + } + + @Override + public Promise thenCompose(Functions.Func1> fn) { + Context context = Context.current(); + return new ContextPromise<>(delegate.thenCompose((r) -> wrap(context, () -> fn.apply(r)))); + } + + @Override + public Promise exceptionally(Functions.Func1 fn) { + Context context = Context.current(); + return new ContextPromise<>(delegate.exceptionally((t) -> wrap(context, () -> fn.apply(t)))); + } + } + + @Override + public ActivityOutput executeActivity(ActivityInput input) { + ActivityOutput output = + tracer.traceOutbound( + "StartActivity", + input.getActivityName(), + activityTags(input.getActivityId()), + input.getHeader(), + () -> super.executeActivity(input)); + return new ActivityOutput<>(output.getActivityId(), new ContextPromise<>(output.getResult())); + } + + @Override + public LocalActivityOutput executeLocalActivity(LocalActivityInput input) { + LocalActivityOutput output = + tracer.traceOutbound( + "StartActivity", + input.getActivityName(), + activityTags(input.getActivityId()), + input.getHeader(), + () -> super.executeLocalActivity(input)); + return new LocalActivityOutput<>(new ContextPromise<>(output.getResult())); + } + + @Override + public ChildWorkflowOutput executeChildWorkflow(ChildWorkflowInput input) { + ChildWorkflowOutput output = + tracer.traceOutbound( + "StartChildWorkflow", + input.getWorkflowType(), + childWorkflowTags(input), + input.getHeader(), + () -> super.executeChildWorkflow(input)); + return new ChildWorkflowOutput<>( + new ContextPromise<>(output.getResult()), + new ContextPromise<>(output.getWorkflowExecution())); + } + + @Override + public ExecuteNexusOperationOutput executeNexusOperation( + ExecuteNexusOperationInput input) { + ExecuteNexusOperationOutput output = + tracer.traceNexusOutbound( + "StartNexusOperation", + input.getService() + "/" + input.getOperation(), + nexusTags(input), + input.getHeaders(), + () -> super.executeNexusOperation(input)); + return new ExecuteNexusOperationOutput<>( + new ContextPromise<>(output.getResult()), + new ContextPromise<>(output.getOperationExecution())); + } + + @Override + public Promise newTimer(Duration duration) { + return new ContextPromise<>(super.newTimer(duration)); + } + + @Override + public Promise newTimer(Duration duration, TimerOptions options) { + return new ContextPromise<>(super.newTimer(duration, options)); + } + + @Override + public SignalExternalOutput signalExternalWorkflow(SignalExternalInput input) { + SignalExternalOutput output = + tracer.traceOutbound( + "SignalExternalWorkflow", + input.getSignalName(), + workflowExecutionTags(input.getExecution()), + input.getHeader(), + () -> super.signalExternalWorkflow(input)); + return new SignalExternalOutput(new ContextPromise<>(output.getResult())); + } + + @Override + public CancelWorkflowOutput cancelWorkflow(CancelWorkflowInput input) { + CancelWorkflowOutput output = + tracer.traceOutbound( + "CancelWorkflow", + "", + workflowExecutionTags(input.getExecution()), + () -> super.cancelWorkflow(input)); + return new CancelWorkflowOutput(new ContextPromise<>(output.getResult())); + } + + @Override + public void continueAsNew(ContinueAsNewInput input) { + String workflowType = input.getWorkflowType(); + if (workflowType == null) { + workflowType = Workflow.getInfo().getWorkflowType(); + } + tracer.traceOutbound( + "ContinueAsNew", + workflowType, + workflowTags(), + input.getHeader(), + () -> super.continueAsNew(input)); + } + + private static Attributes workflowTags() { + WorkflowInfo info = Workflow.getInfo(); + return Attributes.of(WORKFLOW_ID, info.getWorkflowId(), RUN_ID, info.getRunId()); + } + + private static Attributes activityTags(@Nullable String activityId) { + Attributes tags = workflowTags(); + if (activityId == null) { + return tags; + } + return tags.toBuilder().put(ACTIVITY_ID, activityId).build(); + } + + private static Attributes workflowExecutionTags(WorkflowExecution execution) { + return Attributes.of(WORKFLOW_ID, execution.getWorkflowId(), RUN_ID, execution.getRunId()); + } + + private static Attributes childWorkflowTags(ChildWorkflowInput input) { + return Attributes.of(WORKFLOW_ID, input.getWorkflowId()); + } + + private static Attributes nexusTags(ExecuteNexusOperationInput input) { + AttributesBuilder tags = + workflowTags().toBuilder() + .put(NEXUS_SERVICE, input.getService()) + .put(NEXUS_OPERATION, input.getOperation()) + .put(NEXUS_ENDPOINT, input.getEndpoint()); + + return tags.build(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeIdGenerator.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeIdGenerator.java new file mode 100644 index 0000000000..2f51a64638 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeIdGenerator.java @@ -0,0 +1,93 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.trace.SpanId; +import io.opentelemetry.api.trace.TraceId; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.sdk.trace.IdGenerator; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.unsafe.WorkflowUnsafe; +import java.util.Random; +import javax.annotation.Nullable; + +/** + * Generates span and trace IDs that are replay safe. + * + *

Mirrors OpenTelemetry's RandomIdGenerator, + * replacing its platform random source with a workflow random stream. + * + *

Note: {@link Random} has 48 bits of state, so IDs can collide across workflows once an + * installation has generated on the order of 2^24 trace IDs. The JDK has no deterministic random + * source with a wider seed that can also be reseeded. A random source with more state may be + * considered in the future. + */ +public final class ReplaySafeIdGenerator implements IdGenerator { + static final ContextKey INTERCEPTOR_SPAN = ContextKey.named("temporal-interceptor-span"); + private static final String INTERCEPTOR_STREAM = "io.temporal.opentelemetry.v2/interceptor"; + private static final String APPLICATION_STREAM = "io.temporal.opentelemetry.v2/application"; + + private static final long INVALID_ID = 0; + + private final IdGenerator delegate; + + /** + * @param delegate generates IDs wherever replay safety isn't needed + */ + public ReplaySafeIdGenerator(IdGenerator delegate) { + this.delegate = delegate; + } + + @Override + public String generateSpanId() { + Random stream = getStream(); + if (stream == null) { + return delegate.generateSpanId(); + } + + long id; + do { + id = stream.nextLong(); + } while (id == INVALID_ID); + return SpanId.fromLong(id); + } + + @Override + public String generateTraceId() { + Random stream = getStream(); + if (stream == null) { + return delegate.generateTraceId(); + } + + long idHi = stream.nextLong(); + long idLo; + do { + idLo = stream.nextLong(); + } while (idLo == INVALID_ID); + return TraceId.fromLongs(idHi, idLo); + } + + /** + * Interceptor spans and application spans draw from separate streams so their IDs never collide. + * Null means the delegate can be used. + */ + @Nullable + private static Random getStream() { + if (!WorkflowUnsafe.isWorkflowThread() || WorkflowUnsafe.isReadOnly()) { + return null; + } + + Context context = Context.current(); + if (context.get(INTERCEPTOR_SPAN) != null) { + return Workflow.getRandomStream(INTERCEPTOR_STREAM); + } + + String tracerName = context.get(ReplaySafeTracer.TRACER_NAME); + if (tracerName == null) { + throw new IllegalStateException( + "Workflow span started without a replay safe tracer. Ensure tracers used in workflows " + + "are obtained from ReplaySafeOpenTelemetry"); + } + return Workflow.getRandomStream(APPLICATION_STREAM + "/" + tracerName); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java new file mode 100644 index 0000000000..8d0ad4603b --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java @@ -0,0 +1,100 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.StatusCode; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.unsafe.WorkflowUnsafe; +import java.util.concurrent.TimeUnit; + +/** Wraps a span so that replayed code does not end it, which would export a duplicate. */ +public final class ReplaySafeSpan implements Span { + private final Span delegate; + private boolean ended; + + public ReplaySafeSpan(Span delegate) { + this.delegate = delegate; + } + + @Override + public void end() { + if (ended) { + return; + } + ended = true; + if (WorkflowUnsafe.isWorkflowThread() + && !WorkflowUnsafe.isReadOnly() + && WorkflowUnsafe.isReplaying()) { + return; + } + delegate.end(); + } + + @Override + public void end(long timestamp, TimeUnit unit) { + if (ended) { + return; + } + ended = true; + if (WorkflowUnsafe.isWorkflowThread() + && !WorkflowUnsafe.isReadOnly() + && WorkflowUnsafe.isReplaying()) { + return; + } + delegate.end(timestamp, unit); + } + + @Override + public Span setAttribute(AttributeKey key, T value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public Span addEvent(String name, Attributes attributes) { + if (WorkflowUnsafe.isWorkflowThread() + && !WorkflowUnsafe.isReadOnly() + && WorkflowUnsafe.isReplaying()) { + delegate.addEvent(name, attributes, Workflow.currentTimeMillis(), TimeUnit.MILLISECONDS); + } else { + delegate.addEvent(name, attributes); + } + return this; + } + + @Override + public Span addEvent(String name, Attributes attributes, long timestamp, TimeUnit unit) { + delegate.addEvent(name, attributes, timestamp, unit); + return this; + } + + @Override + public Span setStatus(StatusCode statusCode, String description) { + delegate.setStatus(statusCode, description); + return this; + } + + @Override + public Span recordException(Throwable exception, Attributes additionalAttributes) { + delegate.recordException(exception, additionalAttributes); + return this; + } + + @Override + public Span updateName(String name) { + delegate.updateName(name); + return this; + } + + @Override + public SpanContext getSpanContext() { + return delegate.getSpanContext(); + } + + @Override + public boolean isRecording() { + return !ended && delegate.isRecording(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java new file mode 100644 index 0000000000..678fd961ed --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java @@ -0,0 +1,130 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.context.Scope; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.unsafe.WorkflowUnsafe; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; + +/** + * Wraps a tracer so the spans it starts inside a workflow are replay safe. The tracer name is + * published on the current {@link Context} while the span starts, which is where {@link + * ReplaySafeIdGenerator} reads it. + */ +public final class ReplaySafeTracer implements Tracer { + /** The instrumentation name of the tracer starting the current span. */ + @Nonnull static final ContextKey TRACER_NAME = ContextKey.named("temporal-tracer-name"); + + private final Tracer delegate; + private final String name; + + public ReplaySafeTracer(Tracer delegate, String name) { + this.delegate = delegate; + this.name = name; + } + + @Override + public SpanBuilder spanBuilder(String spanName) { + return new NamedStreamSpanBuilder(delegate.spanBuilder(spanName), name); + } + + private static final class NamedStreamSpanBuilder implements SpanBuilder { + private final SpanBuilder delegate; + private final String tracerName; + private boolean startTimestampSet; + + NamedStreamSpanBuilder(SpanBuilder delegate, String tracerName) { + this.delegate = delegate; + this.tracerName = tracerName; + } + + @Override + public Span startSpan() { + if (WorkflowUnsafe.isWorkflowThread() + && !WorkflowUnsafe.isReadOnly() + && WorkflowUnsafe.isReplaying() + && !startTimestampSet) { + delegate.setStartTimestamp(Workflow.currentTimeMillis(), TimeUnit.MILLISECONDS); + } + try (Scope ignored = Context.current().with(TRACER_NAME, tracerName).makeCurrent()) { + return new ReplaySafeSpan(delegate.startSpan()); + } + } + + @Override + public SpanBuilder setParent(Context context) { + delegate.setParent(context); + return this; + } + + @Override + public SpanBuilder setNoParent() { + delegate.setNoParent(); + return this; + } + + @Override + public SpanBuilder addLink(SpanContext spanContext) { + delegate.addLink(spanContext); + return this; + } + + @Override + public SpanBuilder addLink(SpanContext spanContext, Attributes attributes) { + delegate.addLink(spanContext, attributes); + return this; + } + + @Override + public SpanBuilder setAttribute(String key, String value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public SpanBuilder setAttribute(String key, long value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public SpanBuilder setAttribute(String key, double value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public SpanBuilder setAttribute(String key, boolean value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public SpanBuilder setAttribute(AttributeKey key, T value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public SpanBuilder setSpanKind(SpanKind spanKind) { + delegate.setSpanKind(spanKind); + return this; + } + + @Override + public SpanBuilder setStartTimestamp(long startTimestamp, TimeUnit unit) { + startTimestampSet = true; + delegate.setStartTimestamp(startTimestamp, unit); + return this; + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java new file mode 100644 index 0000000000..750507f08a --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java @@ -0,0 +1,136 @@ +package io.temporal.opentelemetry.v2.internal; + +import com.google.common.reflect.TypeToken; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.propagation.TextMapGetter; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.opentelemetry.context.propagation.TextMapSetter; +import io.temporal.api.common.v1.Payload; +import io.temporal.common.converter.DefaultDataConverter; +import io.temporal.common.converter.StdConverterBackwardsCompatAdapter; +import io.temporal.common.interceptors.Header; +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; +import javax.annotation.Nullable; + +/** + * Serializes the current span and baggage into Temporal and Nexus headers and reads them back. + * + *

Temporal headers carry one {@link Properties} payload under the configured key. Nexus headers + * are flat and use HTTP header semantics, so they are read case-insensitively. + */ +final class SpanCodec { + private static final TextMapSetter> MAP_SETTER = Map::put; + + private static final TextMapGetter> MAP_GETTER = + new TextMapGetter>() { + @Override + public Iterable keys(Map carrier) { + return carrier.keySet(); + } + + @Override + @Nullable + public String get(Map carrier, String key) { + return carrier.get(key); + } + }; + private static final TextMapSetter PROPERTIES_SETTER = Properties::setProperty; + private static final TextMapGetter PROPERTIES_GETTER = + new TextMapGetter() { + @Override + public Iterable keys(Properties carrier) { + return carrier.stringPropertyNames(); + } + + @Override + @Nullable + public String get(Properties carrier, String key) { + return carrier.getProperty(key); + } + }; + private static final Type HASH_MAP_STRING_STRING_TYPE = + new TypeToken>() {}.getType(); + + private final TextMapPropagator propagator; + private final String headerKey; + + SpanCodec(String headerKey) { + this.propagator = GlobalOpenTelemetry.getPropagators().getTextMapPropagator(); + this.headerKey = headerKey; + } + + /** The current context extended with the span and baggage carried by {@code header}. */ + Context read(Header header) { + Payload payload = header.getValues().get(headerKey); + if (payload == null) { + return Context.current(); + } + return extract(decode(payload), PROPERTIES_GETTER); + } + + /** The root context extended with the span and baggage carried by {@code header}. */ + Context readFromRoot(Map header) { + Payload payload = header.get(headerKey); + if (payload == null) { + return Context.root(); + } + return propagator.extract(Context.root(), decode(payload), PROPERTIES_GETTER); + } + + /** The current context extended with the span and baggage carried by {@code nexusHeaders}. */ + Context read(Map nexusHeaders) { + // Nexus headers use HTTP header semantics, so the propagator must see them case-insensitively. + // See https://opentelemetry.io/docs/specs/otel/context/api-propagators/#get. + Map carrier = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + carrier.putAll(nexusHeaders); + return extract(carrier, MAP_GETTER); + } + + /** Writes the current span and baggage into {@code header}; leaves it untouched if empty. */ + void write(Header header) { + write(Context.current(), header.getValues()); + } + + Map serialize(Context context) { + Map header = new HashMap<>(); + write(context, header); + return header.isEmpty() ? Collections.emptyMap() : header; + } + + private void write(Context context, Map header) { + Properties carrier = new Properties(); + propagator.inject(context, carrier, PROPERTIES_SETTER); + if (!carrier.isEmpty()) { + header.put(headerKey, encode(carrier)); + } + } + + /** Writes the current span and baggage into {@code nexusHeaders}. */ + void write(Map nexusHeaders) { + propagator.inject(Context.current(), nexusHeaders, MAP_SETTER); + } + + private Context extract(C carrier, TextMapGetter getter) { + Context current = Context.current(); + return propagator.extract(current, carrier, getter); + } + + private static Payload encode(Properties carrier) { + return DefaultDataConverter.STANDARD_INSTANCE.toPayload(carrier).get(); + } + + static Properties decode(Payload payload) { + Map decoded = + StdConverterBackwardsCompatAdapter.fromPayload( + payload, HashMap.class, HASH_MAP_STRING_STRING_TYPE); + Properties properties = new Properties(); + properties.putAll(decoded); + return properties; + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TagKeys.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TagKeys.java new file mode 100644 index 0000000000..a1ce777011 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TagKeys.java @@ -0,0 +1,19 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.AttributeKey; + +final class TagKeys { + static final AttributeKey WORKFLOW_ID = AttributeKey.stringKey("temporalWorkflowID"); + static final AttributeKey RUN_ID = AttributeKey.stringKey("temporalRunID"); + static final AttributeKey ACTIVITY_ID = AttributeKey.stringKey("temporalActivityID"); + static final AttributeKey UPDATE_ID = AttributeKey.stringKey("temporalUpdateID"); + static final AttributeKey TERMINATE_REASON = + AttributeKey.stringKey("temporalTerminateReason"); + static final AttributeKey NEXUS_SERVICE = AttributeKey.stringKey("temporalNexusService"); + static final AttributeKey NEXUS_OPERATION = + AttributeKey.stringKey("temporalNexusOperation"); + static final AttributeKey NEXUS_ENDPOINT = + AttributeKey.stringKey("temporalNexusEndpoint"); + + private TagKeys() {} +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TemporalContextStorage.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TemporalContextStorage.java new file mode 100644 index 0000000000..c9ea1c0598 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TemporalContextStorage.java @@ -0,0 +1,59 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; +import io.temporal.workflow.WorkflowThreadLocal; +import io.temporal.workflow.unsafe.WorkflowUnsafe; + +/** Stores the current OpenTelemetry context in Temporal workflow threads. */ +public final class TemporalContextStorage implements ContextStorage { + private static final WorkflowThreadLocal WORKFLOW_CONTEXT = new WorkflowThreadLocal<>(); + private static volatile boolean installed; + + private final ContextStorage delegate; + + public TemporalContextStorage(ContextStorage delegate) { + this.delegate = delegate; + installed = true; + } + + /** True once OpenTelemetry has wrapped its context storage with this class. */ + public static boolean isInstalled() { + return installed; + } + + static void setWorkflowContext(Context context) { + if (WorkflowUnsafe.isWorkflowThread()) { + WORKFLOW_CONTEXT.set(context); + } + } + + @Override + public Scope attach(Context context) { + if (!WorkflowUnsafe.isWorkflowThread()) { + return delegate.attach(context); + } + + Context previous = WORKFLOW_CONTEXT.get(); + if (context == previous) { + return () -> {}; + } + WORKFLOW_CONTEXT.set(context); + return () -> { + if (WORKFLOW_CONTEXT.get() == context) { + WORKFLOW_CONTEXT.set(previous); + } + }; + } + + @Override + public Context current() { + return WorkflowUnsafe.isWorkflowThread() ? WORKFLOW_CONTEXT.get() : delegate.current(); + } + + @Override + public Context root() { + return delegate.root(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ContextStorageWrapperTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ContextStorageWrapperTest.java new file mode 100644 index 0000000000..5e461b8edd --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ContextStorageWrapperTest.java @@ -0,0 +1,46 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertTrue; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; +import org.junit.Test; + +public class ContextStorageWrapperTest { + @Test + public void pluginAcceptsAWrapperRegisteredAfterReplaySafeOpenTelemetry() { + try (ReplaySafeOpenTelemetry openTelemetry = ReplaySafeOpenTelemetry.newBuilder().build()) { + ContextStorage.addWrapper(OuterStorage::new); + GlobalOpenTelemetry.set(openTelemetry); + OpenTelemetryPlugin.newBuilder().build(); + } finally { + GlobalOpenTelemetry.resetForTest(); + } + assertTrue(ContextStorage.get() instanceof OuterStorage); + } + + private static final class OuterStorage implements ContextStorage { + private final ContextStorage delegate; + + OuterStorage(ContextStorage delegate) { + this.delegate = delegate; + } + + @Override + public Scope attach(Context context) { + return delegate.attach(context); + } + + @Override + public Context current() { + return delegate.current(); + } + + @Override + public Context root() { + return delegate.root(); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/InterceptorTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/InterceptorTest.java new file mode 100644 index 0000000000..ab09fb59a4 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/InterceptorTest.java @@ -0,0 +1,393 @@ +package io.temporal.opentelemetry.v2; + +import static io.temporal.opentelemetry.v2.TestWorkflows.OUTBOUND_ACTIVITY_ID; +import static io.temporal.opentelemetry.v2.TestWorkflows.OUTBOUND_LOCAL_ACTIVITY_ID; +import static io.temporal.opentelemetry.v2.TestWorkflows.TASK_TOKENS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.api.enums.v1.WorkflowIdConflictPolicy; +import io.temporal.client.UpdateOptions; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowFailedException; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.client.WorkflowTargetOptions; +import io.temporal.client.WorkflowUpdateStage; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncCompletionWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncCompletionWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.BenignErrorWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.BenignErrorWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ChildWorkflowWithSignal; +import io.temporal.opentelemetry.v2.TestWorkflows.ChildWorkflowWithSignalImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ErrorWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.ErrorWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.SpanKindWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.SpanKindWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TestActivitiesImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.UnservedWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.UnservedWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.UpdateTargetWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.WorkflowOutboundCall; +import io.temporal.opentelemetry.v2.TestWorkflows.WorkflowOutboundTagsWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.WorkflowOutboundTagsWorkflowImpl; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.junit.Rule; +import org.junit.Test; + +/** Tests interceptor behavior. */ +public class InterceptorTest extends OtelTestBase { + private static final String UNSERVED_TASK_QUEUE = "opentelemetry-v2-unserved"; + private static final String UPDATE_WITH_START_ID = "interceptor-update-with-start"; + private static final String TARGET_UPDATE_ID = "interceptor-update"; + private static final AttributeKey WORKFLOW_ID = + AttributeKey.stringKey("temporalWorkflowID"); + private static final AttributeKey RUN_ID = AttributeKey.stringKey("temporalRunID"); + private static final AttributeKey UPDATE_ID = AttributeKey.stringKey("temporalUpdateID"); + private static final AttributeKey ACTIVITY_ID = + AttributeKey.stringKey("temporalActivityID"); + + @Rule + public SDKTestWorkflowRule testWorkflowRule = + newRuleBuilder(true) + .setWorkflowTypes( + SpanKindWorkflowImpl.class, + BenignErrorWorkflowImpl.class, + ErrorWorkflowImpl.class, + AsyncCompletionWorkflowImpl.class, + TracerWorkflowImpl.class, + ChildWorkflowWithSignalImpl.class, + UnservedWorkflowImpl.class, + UpdateTargetWorkflowImpl.class, + WorkflowOutboundTagsWorkflowImpl.class) + .setActivityImplementations(new TestActivitiesImpl()) + .build(); + + @Test + public void spanKind() { + testWorkflowRule.newWorkflowStub(SpanKindWorkflow.class).run(); + + Map kinds = new HashMap<>(); + for (SpanData span : endedSpans()) { + kinds.put(span.getName(), span.getKind()); + } + assertEquals(SpanKind.SERVER, kinds.get("RunWorkflow:SpanKindWorkflow")); + assertEquals(SpanKind.CLIENT, kinds.get("StartActivity:NopActivity")); + assertEquals(SpanKind.SERVER, kinds.get("RunActivity:NopActivity")); + } + + @Test + public void historyReplayEmitsNoSpans() throws Exception { + SpanKindWorkflow workflow = testWorkflowRule.newWorkflowStub(SpanKindWorkflow.class); + workflow.run(); + String workflowId = WorkflowStub.fromTyped(workflow).getExecution().getWorkflowId(); + spanExporter.reset(); + + testWorkflowRule + .getWorker() + .replayWorkflowExecution(testWorkflowRule.getExecutionHistory(workflowId)); + + assertSpanTree(Collections.emptyList(), endedSpans()); + } + + @Test + public void workflowClientSignalIncludesTargetRunId() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run, false); + TracerWorkflow target = targetWorkflow(execution); + target.gate(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + assertWorkflowExecutionTags(requireSpanNamed(endedSpans(), "SignalWorkflow:gate"), execution); + } + + @Test + public void workflowClientQueryIncludesTargetRunId() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run, false); + TracerWorkflow target = targetWorkflow(execution); + + assertEquals("ok", target.query()); + target.gate(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + assertWorkflowExecutionTags(requireSpanNamed(endedSpans(), "QueryWorkflow:query"), execution); + } + + @Test + public void workflowClientUpdateIncludesTargetRunIdAndUpdateId() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run, false); + TracerWorkflow target = targetWorkflow(execution); + + WorkflowStub.fromTyped(target) + .startUpdate( + UpdateOptions.newBuilder(Void.class) + .setUpdateName("update") + .setUpdateId(TARGET_UPDATE_ID) + .setWaitForStage(WorkflowUpdateStage.COMPLETED) + .build()) + .getResult(); + target.gate(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + SpanData update = requireSpanNamed(endedSpans(), "StartWorkflowUpdate:update"); + assertWorkflowExecutionTags(update, execution); + assertEquals(TARGET_UPDATE_ID, update.getAttributes().get(UPDATE_ID)); + } + + @Test + public void workflowClientStartIncludesWorkflowId() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run, true); + + SpanData start = requireSpanNamed(endedSpans(), "StartWorkflow:TracerWorkflow"); + assertEquals(execution.getWorkflowId(), start.getAttributes().get(WORKFLOW_ID)); + assertNull(start.getAttributes().get(RUN_ID)); + } + + @Test + public void workflowClientUpdateWithStartIncludesWorkflowIdAndUpdateId() { + String workflowId = "interceptor-update-with-start-target"; + WorkflowStub target = + testWorkflowRule + .getWorkflowClient() + .newUntypedWorkflowStub( + "UpdateTargetWorkflow", + WorkflowOptions.newBuilder() + .setTaskQueue(testWorkflowRule.getTaskQueue()) + .setWorkflowId(workflowId) + .setWorkflowIdConflictPolicy( + WorkflowIdConflictPolicy.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING) + .build()); + + target + .startUpdateWithStart( + UpdateOptions.newBuilder(Void.class) + .setUpdateName("doUpdate") + .setUpdateId(UPDATE_WITH_START_ID) + .setWaitForStage(WorkflowUpdateStage.COMPLETED) + .build(), + new Object[0], + new Object[0]) + .getResult(); + target.signal("updateSignal"); + target.getResult(Void.class); + + SpanData updateWithStart = requireSpanNamed(endedSpans(), "UpdateWithStartWorkflow:doUpdate"); + assertEquals(workflowId, updateWithStart.getAttributes().get(WORKFLOW_ID)); + assertEquals(UPDATE_WITH_START_ID, updateWithStart.getAttributes().get(UPDATE_ID)); + } + + @Test + public void workflowOutboundActivityStartIncludesSuppliedActivityId() { + runWorkflowOutboundCall(WorkflowOutboundCall.ACTIVITY); + + SpanData start = requireSpanNamed(endedSpans(), "StartActivity:Activity"); + SpanData run = requireSpanNamed(endedSpans(), "RunActivity:Activity"); + assertEquals(OUTBOUND_ACTIVITY_ID, start.getAttributes().get(ACTIVITY_ID)); + assertEquals(OUTBOUND_ACTIVITY_ID, run.getAttributes().get(ACTIVITY_ID)); + } + + @Test + public void workflowOutboundLocalActivityStartIncludesSuppliedActivityId() { + runWorkflowOutboundCall(WorkflowOutboundCall.LOCAL_ACTIVITY); + + SpanData start = requireSpanNamed(endedSpans(), "StartActivity:LocalActivity"); + SpanData run = requireSpanNamed(endedSpans(), "RunActivity:LocalActivity"); + assertEquals(OUTBOUND_LOCAL_ACTIVITY_ID, start.getAttributes().get(ACTIVITY_ID)); + assertEquals(OUTBOUND_LOCAL_ACTIVITY_ID, run.getAttributes().get(ACTIVITY_ID)); + } + + @Test + public void workflowOutboundActivityStartOmitsGeneratedActivityId() { + testWorkflowRule.newWorkflowStub(SpanKindWorkflow.class).run(); + + SpanData start = requireSpanNamed(endedSpans(), "StartActivity:NopActivity"); + assertNull(start.getAttributes().get(ACTIVITY_ID)); + } + + @Test + public void workflowOutboundChildStartIncludesWorkflowId() { + runWorkflowOutboundCall(WorkflowOutboundCall.CHILD_WORKFLOW); + + SpanData start = requireSpanNamed(endedSpans(), "StartChildWorkflow:ChildWorkflowWithSignal"); + SpanData run = requireSpanNamed(endedSpans(), "RunWorkflow:ChildWorkflowWithSignal"); + assertEquals(run.getAttributes().get(WORKFLOW_ID), start.getAttributes().get(WORKFLOW_ID)); + assertNull(start.getAttributes().get(RUN_ID)); + } + + @Test + public void workflowOutboundChildSignalIncludesTargetExecution() { + runWorkflowOutboundCall(WorkflowOutboundCall.CHILD_WORKFLOW); + + SpanData run = requireSpanNamed(endedSpans(), "RunWorkflow:ChildWorkflowWithSignal"); + SpanData signal = requireSpanNamed(endedSpans(), "SignalExternalWorkflow:childSignal"); + assertWorkflowExecutionTags(signal, workflowExecution(run)); + } + + @Test + public void workflowOutboundExternalSignalIncludesTargetExecution() { + ChildWorkflowWithSignal target = + testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + ChildWorkflowWithSignal.class, + WorkflowOptions.newBuilder() + .setTaskQueue(testWorkflowRule.getTaskQueue()) + .setWorkflowId("external-signal-target-" + UUID.randomUUID()) + .build()); + WorkflowExecution execution = WorkflowClient.start(target::run); + + WorkflowOutboundTagsWorkflow workflow = + testWorkflowRule.newWorkflowStub(WorkflowOutboundTagsWorkflow.class); + workflow.run(WorkflowOutboundCall.EXTERNAL_SIGNAL, execution); + + assertWorkflowExecutionTags( + requireSpanNamed(endedSpans(), "SignalExternalWorkflow:childSignal"), execution); + } + + @Test + public void workflowOutboundCancelIncludesTargetExecution() { + UnservedWorkflow target = + testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + UnservedWorkflow.class, + WorkflowOptions.newBuilder() + .setTaskQueue(UNSERVED_TASK_QUEUE) + .setWorkflowId("external-cancel-target-" + UUID.randomUUID()) + .build()); + WorkflowExecution execution = WorkflowClient.start(target::run); + + WorkflowOutboundTagsWorkflow workflow = + testWorkflowRule.newWorkflowStub(WorkflowOutboundTagsWorkflow.class); + workflow.run(WorkflowOutboundCall.EXTERNAL_CANCEL, execution); + + assertWorkflowExecutionTags(requireSpanNamed(endedSpans(), "CancelWorkflow"), execution); + } + + private TracerWorkflow targetWorkflow(WorkflowExecution execution) { + return testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + TracerWorkflow.class, + WorkflowTargetOptions.newBuilder().setWorkflowExecution(execution).build()); + } + + private void runWorkflowOutboundCall(WorkflowOutboundCall call) { + WorkflowOutboundTagsWorkflow workflow = + testWorkflowRule.newWorkflowStub(WorkflowOutboundTagsWorkflow.class); + workflow.run(call, WorkflowExecution.getDefaultInstance()); + } + + private static WorkflowExecution workflowExecution(SpanData span) { + return WorkflowExecution.newBuilder() + .setWorkflowId(span.getAttributes().get(WORKFLOW_ID)) + .setRunId(span.getAttributes().get(RUN_ID)) + .build(); + } + + private static void assertWorkflowExecutionTags(SpanData span, WorkflowExecution execution) { + assertEquals(execution.getWorkflowId(), span.getAttributes().get(WORKFLOW_ID)); + assertEquals(execution.getRunId(), span.getAttributes().get(RUN_ID)); + } + + @Test + public void benignErrorLeavesSpanStatusUnset() { + assertThrows( + WorkflowFailedException.class, + () -> testWorkflowRule.newWorkflowStub(BenignErrorWorkflow.class).run()); + + assertSpanTree( + Arrays.asList("StartWorkflow:BenignErrorWorkflow", " RunWorkflow:BenignErrorWorkflow"), + endedSpans()); + SpanData run = requireSpanNamed(endedSpans(), "RunWorkflow:BenignErrorWorkflow"); + assertEquals(StatusCode.UNSET, run.getStatus().getStatusCode()); + } + + @Test + public void errorSetsSpanStatusError() { + assertThrows( + WorkflowFailedException.class, + () -> testWorkflowRule.newWorkflowStub(ErrorWorkflow.class).run()); + + assertSpanTree( + Arrays.asList("StartWorkflow:ErrorWorkflow", " RunWorkflow:ErrorWorkflow"), endedSpans()); + SpanData run = requireSpanNamed(endedSpans(), "RunWorkflow:ErrorWorkflow"); + assertEquals(StatusCode.ERROR, run.getStatus().getStatusCode()); + } + + @Test + public void continueAsNewLeavesWorkflowSpanUnset() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowClient.start(workflow::run, false); + workflow.gate(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + SpanData run = requireSpanNamed(endedSpans(), "RunWorkflow:TracerWorkflow"); + assertEquals(StatusCode.UNSET, run.getStatus().getStatusCode()); + assertTrue(run.getEvents().toString(), run.getEvents().isEmpty()); + } + + @Test + public void pendingActivityLeavesActivitySpanUnset() { + AsyncCompletionWorkflow workflow = + testWorkflowRule.newWorkflowStub(AsyncCompletionWorkflow.class); + WorkflowClient.start(workflow::run); + byte[] taskToken = takeTaskToken(); + testWorkflowRule.getWorkflowClient().newActivityCompletionClient().complete(taskToken, null); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + SpanData activity = requireSpanNamed(endedSpans(), "RunActivity:AsyncCompletionActivity"); + assertEquals(StatusCode.UNSET, activity.getStatus().getStatusCode()); + assertTrue(activity.getEvents().toString(), activity.getEvents().isEmpty()); + } + + @Test + public void evictedWorkflowLeavesRunWorkflowSpanOpen() { + ChildWorkflowWithSignal workflow = + testWorkflowRule.newWorkflowStub(ChildWorkflowWithSignal.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run); + testWorkflowRule.waitForTheEndOfWFT(execution.getWorkflowId()); + testWorkflowRule.invalidateWorkflowCache(); + + assertEquals(0, countSpansNamed("RunWorkflow:ChildWorkflowWithSignal")); + + workflow.childSignal(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + assertEquals(1, countSpansNamed("RunWorkflow:ChildWorkflowWithSignal")); + } + + private static long countSpansNamed(String name) { + return endedSpans().stream().filter(span -> span.getName().equals(name)).count(); + } + + private static byte[] takeTaskToken() { + try { + byte[] taskToken = TASK_TOKENS.poll(30, TimeUnit.SECONDS); + assertNotNull("timed out waiting for activity task token", taskToken); + return taskToken; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OpenTelemetryPluginGlobalTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OpenTelemetryPluginGlobalTest.java new file mode 100644 index 0000000000..384f4c9c35 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OpenTelemetryPluginGlobalTest.java @@ -0,0 +1,44 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.ContextStorage; +import io.temporal.opentelemetry.v2.internal.TemporalContextStorage; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class OpenTelemetryPluginGlobalTest { + @Before + @After + public void resetGlobalOpenTelemetry() { + GlobalOpenTelemetry.resetForTest(); + } + + @Test + public void buildRejectsAnUnregisteredGlobal() { + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> OpenTelemetryPlugin.newBuilder().build()); + assertTrue(e.getMessage(), e.getMessage().contains("ReplaySafeOpenTelemetry")); + } + + @Test + public void buildWithAnUnregisteredGlobalStillAllowsRegistration() { + assertThrows(IllegalStateException.class, () -> OpenTelemetryPlugin.newBuilder().build()); + try (ReplaySafeOpenTelemetry openTelemetry = ReplaySafeOpenTelemetry.newBuilder().build()) { + GlobalOpenTelemetry.set(openTelemetry); + OpenTelemetryPlugin.newBuilder().build(); + } + } + + @Test + public void buildAcceptsAReplaySafeGlobal() { + try (ReplaySafeOpenTelemetry openTelemetry = ReplaySafeOpenTelemetry.newBuilder().build()) { + GlobalOpenTelemetry.set(openTelemetry); + OpenTelemetryPlugin.newBuilder().build(); + } + assertTrue(ContextStorage.get() instanceof TemporalContextStorage); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java new file mode 100644 index 0000000000..b2b924b604 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java @@ -0,0 +1,148 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +public abstract class OtelTestBase { + static final InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + private static ReplaySafeOpenTelemetry openTelemetry; + + @BeforeClass + public static void registerGlobalOpenTelemetry() { + openTelemetry = + ReplaySafeOpenTelemetry.newBuilder() + .setTracerProviderBuilder( + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter))) + .build(); + GlobalOpenTelemetry.set(openTelemetry); + } + + @AfterClass + public static void resetGlobalOpenTelemetry() { + GlobalOpenTelemetry.resetForTest(); + openTelemetry.close(); + } + + @Before + public void clearSpans() { + spanExporter.reset(); + } + + /** + * A rule whose service stubs carry the plugin, so it propagates to every client and the worker. + */ + static SDKTestWorkflowRule.Builder newRuleBuilder(boolean addTemporalSpans) { + return SDKTestWorkflowRule.newBuilder() + .setWorkflowServiceStubsOptions( + WorkflowServiceStubsOptions.newBuilder() + .setPlugins( + OpenTelemetryPlugin.newBuilder().setAddTemporalSpans(addTemporalSpans).build()) + .build()); + } + + static List endedSpans() { + return spanExporter.getFinishedSpanItems(); + } + + static SpanData requireSpanNamed(List spans, String name) { + for (SpanData span : spans) { + if (span.getName().equals(name)) { + return span; + } + } + fail(name + " span not found in " + spanTree(spans)); + return null; + } + + static String requireSpanAttribute(SpanData span, AttributeKey key) { + String value = span.getAttributes().get(key); + assertNotNull(key.getKey() + " attribute not found on " + span.getName(), value); + return value; + } + + static void requireUniqueSpanIds(List spans) { + Map namesById = new HashMap<>(); + for (SpanData span : spans) { + String previous = namesById.put(span.getSpanId(), span.getName()); + if (previous != null) { + fail("span " + span.getName() + " shares an ID with span " + previous); + } + } + } + + static void assertSpanTree(List expected, List spans) { + assertEquals(String.join("\n", expected), String.join("\n", spanTree(spans))); + } + + /** Returns the spans as an indented tree in end order. */ + static List spanTree(List spans) { + Map> childrenByParent = new HashMap<>(); + for (int child = 0; child < spans.size(); child++) { + childrenByParent + .computeIfAbsent(closestParentIndex(spans, spans.get(child)), k -> new ArrayList<>()) + .add(child); + } + List tree = new ArrayList<>(); + appendChildren(spans, childrenByParent, -1, 0, tree); + return tree; + } + + /** + * Resets can emit the same span ID and start time more than once; the nearest end time identifies + * the matching parent. + */ + private static int closestParentIndex(List spans, SpanData child) { + String parentId = child.getParentSpanContext().getSpanId(); + int closest = -1; + long closestDistance = Long.MAX_VALUE; + for (int i = 0; i < spans.size(); i++) { + if (!spans.get(i).getSpanId().equals(parentId)) { + continue; + } + long distance = Math.abs(child.getEndEpochNanos() - spans.get(i).getEndEpochNanos()); + if (distance < closestDistance) { + closest = i; + closestDistance = distance; + } + } + return closest; + } + + private static void appendChildren( + List spans, + Map> childrenByParent, + int parent, + int depth, + List tree) { + for (int child : childrenByParent.getOrDefault(parent, new ArrayList<>())) { + tree.add(repeat(" ", depth) + spans.get(child).getName()); + appendChildren(spans, childrenByParent, child, depth + 1, tree); + } + } + + private static String repeat(String s, int times) { + StringBuilder out = new StringBuilder(); + for (int i = 0; i < times; i++) { + out.append(s); + } + return out.toString(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/PropagationTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/PropagationTest.java new file mode 100644 index 0000000000..9502e51e40 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/PropagationTest.java @@ -0,0 +1,569 @@ +package io.temporal.opentelemetry.v2; + +import static io.temporal.opentelemetry.v2.TestWorkflows.NEXUS_CANCEL_OPERATION_NAME; +import static io.temporal.opentelemetry.v2.TestWorkflows.NEXUS_OPERATION_NAME; +import static io.temporal.opentelemetry.v2.TestWorkflows.NEXUS_SERVICE_NAME; +import static io.temporal.opentelemetry.v2.TestWorkflows.TASK_TOKENS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assume.assumeTrue; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.temporal.api.enums.v1.WorkflowIdConflictPolicy; +import io.temporal.client.StartActivityOptions; +import io.temporal.client.StartNexusOperationOptions; +import io.temporal.client.UpdateOptions; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.client.WorkflowUpdateStage; +import io.temporal.client.schedules.Schedule; +import io.temporal.client.schedules.ScheduleActionStartWorkflow; +import io.temporal.client.schedules.ScheduleClient; +import io.temporal.client.schedules.ScheduleClientOptions; +import io.temporal.client.schedules.ScheduleHandle; +import io.temporal.client.schedules.ScheduleOptions; +import io.temporal.client.schedules.ScheduleSpec; +import io.temporal.client.schedules.ScheduleUpdate; +import io.temporal.common.interceptors.Header; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncCompletionWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncCompletionWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncLambdaWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncLambdaWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ChildWorkflowWithSignalImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ComprehensiveNexusServiceImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ComprehensiveWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.ComprehensiveWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ConstructorSpanWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.ConstructorSpanWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ContinueAsNewToDifferentWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.ContinueAsNewToDifferentWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.DifferentWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ExternalWorkflowWithSignal; +import io.temporal.opentelemetry.v2.TestWorkflows.ExternalWorkflowWithSignalImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.NexusCancelHandlerWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.NexusHandlerWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.SchedulePropagationReceiver; +import io.temporal.opentelemetry.v2.TestWorkflows.SchedulePropagationReceiverImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.SchedulePropagationWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.SignalWithStartTargetImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.StandaloneWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.StandaloneWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TestActivities; +import io.temporal.opentelemetry.v2.TestWorkflows.TestActivitiesImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.UnservedWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.UpdateTargetWorkflowImpl; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Covers one scenario touching every traced operation, asserted as a span tree with Temporal spans + * on and off. + * + *

Needs a real server: client-started Nexus operations, standalone activities, and schedules are + * not implemented by the in-memory test server. CI runs it in the dev-server job. + */ +@RunWith(Parameterized.class) +public class PropagationTest extends OtelTestBase { + private static final String COMPREHENSIVE_UPDATE_ID = "comprehensive-update"; + private static final String UPDATE_WITH_START_UPDATE_ID = "comprehensive-update-with-start"; + private static final String UNSERVED_TASK_QUEUE = "opentelemetry-v2-unserved"; + private static final String TERMINATE_REASON = "otel-terminate-reason"; + + @Parameters(name = "addTemporalSpans={0}") + public static List addTemporalSpans() { + return Arrays.asList(true, false); + } + + private final boolean addTemporalSpans; + + private final String runId = UUID.randomUUID().toString(); + private final String scheduleId = "otel-schedule-" + runId; + private final String reusedScheduleSeedId = "otel-reused-schedule-seed-" + runId; + private final String reusedScheduleId = "otel-reused-schedule-" + runId; + private final String reusedScheduleWorkflowId = "otel-reused-schedule-workflow-" + runId; + private final String reusedScheduleReceiverId = "otel-reused-schedule-receiver-" + runId; + private final String externalWorkflowId = "externalWorkflowWithSignal-" + runId; + private final String cancelTargetId = "externalCancelTarget-" + runId; + private final String comprehensiveWorkflowId = "comprehensive-outbound-" + runId; + private final String updateWithStartWorkflowId = "otel-update-with-start-" + runId; + + @Rule public SDKTestWorkflowRule testWorkflowRule; + + public PropagationTest(boolean addTemporalSpans) { + this.addTemporalSpans = addTemporalSpans; + this.testWorkflowRule = + newRuleBuilder(addTemporalSpans) + .setWorkflowTypes( + ComprehensiveWorkflowImpl.class, + ContinueAsNewToDifferentWorkflowImpl.class, + DifferentWorkflowImpl.class, + ChildWorkflowWithSignalImpl.class, + ExternalWorkflowWithSignalImpl.class, + NexusHandlerWorkflowImpl.class, + NexusCancelHandlerWorkflowImpl.class, + StandaloneWorkflowImpl.class, + SchedulePropagationWorkflowImpl.class, + SchedulePropagationReceiverImpl.class, + SignalWithStartTargetImpl.class, + UpdateTargetWorkflowImpl.class, + AsyncCompletionWorkflowImpl.class, + UnservedWorkflowImpl.class, + AsyncLambdaWorkflowImpl.class, + ConstructorSpanWorkflowImpl.class) + .setActivityImplementations(new TestActivitiesImpl()) + .setNexusServiceImplementation(new ComprehensiveNexusServiceImpl()) + .build(); + } + + @Before + public void requireRealServer() { + assumeTrue( + "Test Server doesn't support client Nexus operations, standalone activities, or schedules", + SDKTestWorkflowRule.useExternalService); + } + + @Test + public void comprehensive() { + List spans = runScenario(); + assertSpanTree(addTemporalSpans ? fullTree() : noTemporalSpansTree(), spans); + requireUniqueSpanIds(spans); + } + + /** Drives every traced operation under one client span and returns the ended spans. */ + private List runScenario() { + WorkflowClient client = testWorkflowRule.getWorkflowClient(); + String taskQueue = testWorkflowRule.getTaskQueue(); + String nexusEndpoint = testWorkflowRule.getNexusEndpoint().getSpec().getName(); + + // All client calls share this parent span. + Span clientSpan = + GlobalOpenTelemetry.getTracer("client").spanBuilder("client-span").startSpan(); + try (Scope ignored = clientSpan.makeCurrent()) { + // Start the external signal target first. + ExternalWorkflowWithSignal external = + client.newWorkflowStub( + ExternalWorkflowWithSignal.class, options(taskQueue, externalWorkflowId)); + WorkflowClient.start(external::run); + + WorkflowStub cancelCallbackTarget = + client.newUntypedWorkflowStub( + "UnservedWorkflow", options(UNSERVED_TASK_QUEUE, cancelTargetId)); + cancelCallbackTarget.start(); + + client + .newWorkflowStub( + AsyncLambdaWorkflow.class, + options(taskQueue, "otel-async-lambda-" + UUID.randomUUID())) + .run(); + client + .newWorkflowStub( + ConstructorSpanWorkflow.class, + options(taskQueue, "otel-constructor-span-" + UUID.randomUUID())) + .run(); + + client + .newWorkflowStub( + ContinueAsNewToDifferentWorkflow.class, + options(taskQueue, "otel-continue-as-new-" + UUID.randomUUID())) + .run(); + + ComprehensiveWorkflow comprehensive = + client.newWorkflowStub( + ComprehensiveWorkflow.class, options(taskQueue, comprehensiveWorkflowId)); + WorkflowClient.start( + comprehensive::run, false, nexusEndpoint, externalWorkflowId, cancelTargetId); + WorkflowStub comprehensiveStub = WorkflowStub.fromTyped(comprehensive); + + comprehensiveStub + .startUpdate( + UpdateOptions.newBuilder(Void.class) + .setUpdateName("testUpdate") + .setUpdateId(COMPREHENSIVE_UPDATE_ID) + .setWaitForStage(WorkflowUpdateStage.COMPLETED) + .build()) + .getResult(); + assertEquals("ok", comprehensive.getStatus()); + comprehensive.proceed(); + comprehensiveStub.getResult(Void.class); + assertEquals("ok", comprehensive.getStatus()); + + testWorkflowRule + .getActivityClient() + .execute( + TestActivities.class, + TestActivities::standaloneActivity, + StartActivityOptions.newBuilder() + .setId("otel-standalone-activity-" + UUID.randomUUID()) + .setTaskQueue(taskQueue) + .setStartToCloseTimeout(Duration.ofSeconds(10)) + .build()); + + client + .newWorkflowStub( + StandaloneWorkflow.class, + options(taskQueue, "otel-standalone-workflow-" + UUID.randomUUID())) + .run(null); + + AsyncCompletionWorkflow asyncCompletion = + client.newWorkflowStub( + AsyncCompletionWorkflow.class, + options(taskQueue, "otel-async-completion-" + UUID.randomUUID())); + WorkflowClient.start(asyncCompletion::run); + byte[] taskToken = takeTaskToken(); + client.newActivityCompletionClient().complete(taskToken, null); + WorkflowStub.fromTyped(asyncCompletion).getResult(Void.class); + + WorkflowStub cancelTarget = + client.newUntypedWorkflowStub( + "UnservedWorkflow", + options(UNSERVED_TASK_QUEUE, "otel-cancel-target-" + UUID.randomUUID())); + cancelTarget.start(); + cancelTarget.cancel(); + + WorkflowStub terminateTarget = + client.newUntypedWorkflowStub( + "UnservedWorkflow", + options(UNSERVED_TASK_QUEUE, "otel-terminate-target-" + UUID.randomUUID())); + terminateTarget.start(); + terminateTarget.terminate(TERMINATE_REASON); + + comprehensiveStub.describe(); + + // A protobuf-backed header is immutable, so create and update must not write into it. + ScheduleActionStartWorkflow scheduleAction = + ScheduleActionStartWorkflow.newBuilder() + .setWorkflowType("StandaloneWorkflow") + .setHeader(new Header(io.temporal.api.common.v1.Header.getDefaultInstance())) + .setOptions(options(taskQueue, "otel-schedule-workflow-" + UUID.randomUUID())) + .build(); + ScheduleClient scheduleClient = + ScheduleClient.newInstance( + testWorkflowRule.getWorkflowServiceStubs(), + ScheduleClientOptions.newBuilder() + .setNamespace(client.getOptions().getNamespace()) + .build()); + ScheduleHandle schedule = + scheduleClient.createSchedule( + scheduleId, + Schedule.newBuilder() + .setAction(scheduleAction) + .setSpec(ScheduleSpec.newBuilder().build()) + .build(), + ScheduleOptions.newBuilder().build()); + try { + String updatedScheduleWorkflowId = "otel-schedule-updated-workflow-" + UUID.randomUUID(); + try (Scope rootScope = Context.root().makeCurrent()) { + schedule.update( + input -> + new ScheduleUpdate( + Schedule.newBuilder(input.getDescription().getSchedule()) + .setAction( + ScheduleActionStartWorkflow.newBuilder(scheduleAction) + .setOptions(options(taskQueue, updatedScheduleWorkflowId)) + .setArguments(externalWorkflowId) + .build()) + .build())); + } + schedule.trigger(); + String scheduledWorkflowId = WorkflowStub.fromTyped(external).getResult(String.class); + client.newUntypedWorkflowStub(scheduledWorkflowId).getResult(Void.class); + + // Reusing this action verifies a root-context create cannot retain the first create's + // trace. + SchedulePropagationReceiver reusedScheduleReceiver = + client.newWorkflowStub( + SchedulePropagationReceiver.class, options(taskQueue, reusedScheduleReceiverId)); + WorkflowClient.start(reusedScheduleReceiver::run); + ScheduleActionStartWorkflow reusedScheduleAction = + ScheduleActionStartWorkflow.newBuilder() + .setWorkflowType("SchedulePropagationWorkflow") + .setOptions(options(taskQueue, reusedScheduleWorkflowId)) + .setArguments(reusedScheduleReceiverId) + .build(); + ScheduleHandle reusedScheduleSeed = + scheduleClient.createSchedule( + reusedScheduleSeedId, + Schedule.newBuilder() + .setAction(reusedScheduleAction) + .setSpec(ScheduleSpec.newBuilder().build()) + .build(), + ScheduleOptions.newBuilder().build()); + ScheduleHandle reusedSchedule = null; + try { + try (Scope rootScope = Context.root().makeCurrent()) { + reusedSchedule = + scheduleClient.createSchedule( + reusedScheduleId, + Schedule.newBuilder() + .setAction(reusedScheduleAction) + .setSpec(ScheduleSpec.newBuilder().build()) + .build(), + ScheduleOptions.newBuilder().build()); + } + reusedSchedule.trigger(); + String reusedScheduledWorkflowId = + WorkflowStub.fromTyped(reusedScheduleReceiver).getResult(String.class); + client.newUntypedWorkflowStub(reusedScheduledWorkflowId).getResult(Void.class); + } finally { + reusedScheduleSeed.delete(); + if (reusedSchedule != null) { + reusedSchedule.delete(); + } + } + + WorkflowStub signalWithStart = + client.newUntypedWorkflowStub( + "SignalWithStartTarget", + options(taskQueue, "otel-signal-with-start-" + UUID.randomUUID())); + signalWithStart.signalWithStart("startSignal", new Object[0], new Object[0]); + signalWithStart.getResult(Void.class); + + WorkflowStub updateWithStart = + client.newUntypedWorkflowStub( + "UpdateTargetWorkflow", + options(taskQueue, updateWithStartWorkflowId).toBuilder() + .setWorkflowIdConflictPolicy( + WorkflowIdConflictPolicy.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING) + .build()); + updateWithStart + .startUpdateWithStart( + UpdateOptions.newBuilder(Void.class) + .setUpdateName("doUpdate") + .setUpdateId(UPDATE_WITH_START_UPDATE_ID) + .setWaitForStage(WorkflowUpdateStage.COMPLETED) + .build(), + new Object[0], + new Object[0]) + .getResult(); + updateWithStart.signal("updateSignal"); + updateWithStart.getResult(Void.class); + + testWorkflowRule + .getNexusClient() + .newUntypedNexusServiceClient(nexusEndpoint, NEXUS_SERVICE_NAME) + .execute( + NEXUS_OPERATION_NAME, + Void.class, + StartNexusOperationOptions.newBuilder() + .setId("otel-nexus-operation-" + UUID.randomUUID()) + .setScheduleToCloseTimeout(Duration.ofSeconds(10)) + .build(), + ""); + } finally { + schedule.delete(); + } + } finally { + clientSpan.end(); + } + return endedSpans(); + } + + private static WorkflowOptions options(String taskQueue, String workflowId) { + return WorkflowOptions.newBuilder().setTaskQueue(taskQueue).setWorkflowId(workflowId).build(); + } + + private static byte[] takeTaskToken() { + try { + byte[] taskToken = TASK_TOKENS.poll(30, TimeUnit.SECONDS); + assertNotNull("timed out waiting for activity task token", taskToken); + return taskToken; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + + private static String nexusOp(String operation) { + return NEXUS_SERVICE_NAME + "/" + operation; + } + + /** The span tree with Temporal spans enabled. */ + private List fullTree() { + return Arrays.asList( + // The scheduled workflow finishes before the client span, so its root appears first. + "UpdateSchedule:" + scheduleId, + " RunWorkflow:StandaloneWorkflow", + " standalone-workflow-span", + " SignalExternalWorkflow:scheduleStarted", + " HandleSignal:scheduleStarted", + "CreateSchedule:" + reusedScheduleId, + " RunWorkflow:SchedulePropagationWorkflow", + " schedule-propagation-workflow-span", + " SignalExternalWorkflow:scheduleStarted", + " HandleSignal:scheduleStarted", + "client-span", + " StartWorkflow:ExternalWorkflowWithSignal", + " RunWorkflow:ExternalWorkflowWithSignal", + " external-workflow-with-signal-span", + " StartWorkflow:UnservedWorkflow", + " StartWorkflow:AsyncLambdaWorkflow", + " RunWorkflow:AsyncLambdaWorkflow", + " parent", + " child", + " StartActivity:NopActivity", + " RunActivity:NopActivity", + " StartWorkflow:ConstructorSpanWorkflow", + " constructor", + " RunWorkflow:ConstructorSpanWorkflow", + " StartWorkflow:ContinueAsNewToDifferentWorkflow", + " RunWorkflow:ContinueAsNewToDifferentWorkflow", + " ContinueAsNew:DifferentWorkflow", + " RunWorkflow:DifferentWorkflow", + " StartWorkflow:ComprehensiveWorkflow", + " RunWorkflow:ComprehensiveWorkflow", + " StartActivity:Activity", + " RunActivity:Activity", + " activity-span", + " StartActivity:LocalActivity", + " RunActivity:LocalActivity", + " local-activity-span", + " timer-registration-span", + " timer-callback-span", + " StartChildWorkflow:ChildWorkflowWithSignal", + " RunWorkflow:ChildWorkflowWithSignal", + " child-workflow-with-signal-span", + " promise-registration-span", + " promise-callback-span", + // There is no SignalChildWorkflow outbound method; child signals go through + // signalExternalWorkflow, so this span is named for that method. + " SignalExternalWorkflow:childSignal", + " HandleSignal:childSignal", + " SignalExternalWorkflow:externalSignal", + " HandleSignal:externalSignal", + " cancel-registration-span", + " CancelWorkflow", + " cancel-callback-span", + " StartNexusOperation:" + nexusOp(NEXUS_OPERATION_NAME), + " RunStartNexusOperationHandler:" + nexusOp(NEXUS_OPERATION_NAME), + " StartWorkflow:NexusHandlerWorkflow", + " RunWorkflow:NexusHandlerWorkflow", + " workflow-with-nexus-handler-span", + " StartNexusOperation:" + nexusOp(NEXUS_CANCEL_OPERATION_NAME), + " RunStartNexusOperationHandler:" + nexusOp(NEXUS_CANCEL_OPERATION_NAME), + " StartWorkflow:NexusCancelHandlerWorkflow", + " RunWorkflow:NexusCancelHandlerWorkflow", + " nexus-cancel-handler-span", + " RunCancelNexusOperationHandler:" + nexusOp(NEXUS_CANCEL_OPERATION_NAME), + " CancelWorkflow", + // Continue-as-new links the outbound, continued-run, and user spans. + " ContinueAsNew:ComprehensiveWorkflow", + " RunWorkflow:ComprehensiveWorkflow", + " comprehensive-outbound-workflow-span", + " comprehensive-outbound-workflow-span", + // Update user spans follow their current inbound operation. + " StartWorkflowUpdate:testUpdate", + " ValidateUpdate:testUpdate", + " validate-update-span", + " validate-update-span-child", + " HandleUpdate:testUpdate", + " update-handler-span", + " update-handler-child-span", + // Query handler spans parent under the query that ran them. + " QueryWorkflow:getStatus", + " HandleQuery:getStatus", + " query-handler-span", + " query-handler-child-span", + " SignalWorkflow:proceed", + " HandleSignal:proceed", + // The query after completion replays the run, but its handler spans are still exported. + " QueryWorkflow:getStatus", + " HandleQuery:getStatus", + " query-handler-span", + " query-handler-child-span", + // Headers link standalone StartActivity and RunActivity spans. + " StartActivity:StandaloneActivity", + " RunActivity:StandaloneActivity", + " StartWorkflow:StandaloneWorkflow", + " RunWorkflow:StandaloneWorkflow", + " StartWorkflow:AsyncCompletionWorkflow", + " RunWorkflow:AsyncCompletionWorkflow", + " StartActivity:AsyncCompletionActivity", + " RunActivity:AsyncCompletionActivity", + // Cancel, terminate, and describe have no propagation carrier but are still traced. + " StartWorkflow:UnservedWorkflow", + " CancelWorkflow", + " StartWorkflow:UnservedWorkflow", + " TerminateWorkflow", + " DescribeWorkflow", + " CreateSchedule:" + scheduleId, + " StartWorkflow:SchedulePropagationReceiver", + " RunWorkflow:SchedulePropagationReceiver", + " CreateSchedule:" + reusedScheduleSeedId, + // Signal-with-start links client, worker, signal, and user spans. + " SignalWithStartWorkflow:SignalWithStartTarget", + " HandleSignal:startSignal", + " RunWorkflow:SignalWithStartTarget", + " signal-with-start-target-span", + // Update-with-start links validation, execution, worker, and user spans. + " UpdateWithStartWorkflow:doUpdate", + " ValidateUpdate:doUpdate", + " HandleUpdate:doUpdate", + " update start", + " RunWorkflow:UpdateTargetWorkflow", + " update-target-workflow-span", + " SignalWorkflow:updateSignal", + " HandleSignal:updateSignal", + " StartNexusOperation:" + nexusOp(NEXUS_OPERATION_NAME), + " RunStartNexusOperationHandler:" + nexusOp(NEXUS_OPERATION_NAME), + " StartWorkflow:NexusHandlerWorkflow", + " RunWorkflow:NexusHandlerWorkflow", + " workflow-with-nexus-handler-span"); + } + + /** + * {@link #fullTree()} without any Temporal spans. Only user spans remain, and each reattaches to + * the nearest surviving ancestor. + */ + private static List noTemporalSpansTree() { + return Arrays.asList( + "standalone-workflow-span", + "schedule-propagation-workflow-span", + "client-span", + " parent", + " child", + " constructor", + " validate-update-span", + " validate-update-span-child", + " update-handler-span", + " update-handler-child-span", + " query-handler-span", + " query-handler-child-span", + " activity-span", + " local-activity-span", + " timer-registration-span", + " timer-callback-span", + // The child span ends after its signal, which follows the promise spans. + " promise-registration-span", + " promise-callback-span", + " child-workflow-with-signal-span", + " cancel-registration-span", + " cancel-callback-span", + " workflow-with-nexus-handler-span", + " nexus-cancel-handler-span", + // Continue-as-new emits the user span once per run. + " comprehensive-outbound-workflow-span", + " comprehensive-outbound-workflow-span", + " query-handler-span", + " query-handler-child-span", + " external-workflow-with-signal-span", + " signal-with-start-target-span", + " update start", + " update-target-workflow-span", + " workflow-with-nexus-handler-span"); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetryTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetryTest.java new file mode 100644 index 0000000000..722af201bc --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetryTest.java @@ -0,0 +1,19 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertThrows; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.Context; +import org.junit.Test; + +public class ReplaySafeOpenTelemetryTest { + @Test + public void pluginRejectsContextUsedBeforeReplaySafeOpenTelemetry() { + Context.current(); + + try (ReplaySafeOpenTelemetry openTelemetry = ReplaySafeOpenTelemetry.newBuilder().build()) { + GlobalOpenTelemetry.set(openTelemetry); + assertThrows(IllegalStateException.class, () -> OpenTelemetryPlugin.newBuilder().build()); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TestWorkflows.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TestWorkflows.java new file mode 100644 index 0000000000..c052ea36ca --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TestWorkflows.java @@ -0,0 +1,1045 @@ +package io.temporal.opentelemetry.v2; + +import io.nexusrpc.Operation; +import io.nexusrpc.Service; +import io.nexusrpc.handler.OperationHandler; +import io.nexusrpc.handler.OperationImpl; +import io.nexusrpc.handler.ServiceImpl; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import io.temporal.activity.Activity; +import io.temporal.activity.ActivityInterface; +import io.temporal.activity.ActivityOptions; +import io.temporal.activity.LocalActivityOptions; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.client.WorkflowOptions; +import io.temporal.failure.ApplicationErrorCategory; +import io.temporal.failure.ApplicationFailure; +import io.temporal.failure.CanceledFailure; +import io.temporal.failure.NexusOperationFailure; +import io.temporal.nexus.Nexus; +import io.temporal.nexus.WorkflowRunOperation; +import io.temporal.workflow.ActivityInvocationOptions; +import io.temporal.workflow.Async; +import io.temporal.workflow.CancellationScope; +import io.temporal.workflow.NexusOperationHandle; +import io.temporal.workflow.NexusOperationOptions; +import io.temporal.workflow.NexusServiceOptions; +import io.temporal.workflow.Promise; +import io.temporal.workflow.QueryMethod; +import io.temporal.workflow.SignalMethod; +import io.temporal.workflow.UpdateMethod; +import io.temporal.workflow.UpdateValidatorMethod; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInterface; +import io.temporal.workflow.WorkflowMethod; +import java.time.Duration; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** Workflow and activity fixtures used by the OpenTelemetry v2 tests. */ +public final class TestWorkflows { + private TestWorkflows() {} + + static final String TRACER_TEST_QUERY_NAME = "query"; + static final String TRACER_TEST_UPDATE_NAME = "update"; + static final String TRACER_TEST_SIGNAL_NAME = "gate"; + static final String NEXUS_SERVICE_NAME = "ComprehensiveNexusService"; + static final String NEXUS_OPERATION_NAME = "nexusHandlerWorkflow"; + static final String NEXUS_CANCEL_OPERATION_NAME = "nexusCancelHandlerWorkflow"; + + /** Stands in for the external system an async activity hands its task token to. */ + static final BlockingQueue TASK_TOKENS = new LinkedBlockingQueue<>(); + + static Span startSpan(String tracerName, String spanName) { + return GlobalOpenTelemetry.getTracer(tracerName).spanBuilder(spanName).startSpan(); + } + + static void spanAround(String tracerName, String spanName, Runnable body) { + Span span = startSpan(tracerName, spanName); + try (Scope ignored = span.makeCurrent()) { + body.run(); + } finally { + span.end(); + } + } + + // --------------------------------------------------------------------------------------------- + // Activities + // --------------------------------------------------------------------------------------------- + + @ActivityInterface + public interface TestActivities { + void activity(); + + void localActivity(); + + void asyncCompletionActivity(); + + void standaloneActivity(); + + void nopActivity(); + } + + public static class TestActivitiesImpl implements TestActivities { + @Override + public void activity() { + spanAround("activity", "activity-span", () -> {}); + } + + @Override + public void localActivity() { + spanAround("localActivity", "local-activity-span", () -> {}); + } + + @Override + public void asyncCompletionActivity() { + TASK_TOKENS.add(Activity.getExecutionContext().getTaskToken()); + Activity.getExecutionContext().doNotCompleteOnReturn(); + } + + @Override + public void standaloneActivity() {} + + @Override + public void nopActivity() {} + } + + // --------------------------------------------------------------------------------------------- + // Interceptor tests + // --------------------------------------------------------------------------------------------- + + @WorkflowInterface + public interface SpanKindWorkflow { + @WorkflowMethod + void run(); + } + + public static class SpanKindWorkflowImpl implements SpanKindWorkflow { + @Override + public void run() { + activities().nopActivity(); + } + } + + @WorkflowInterface + public interface AsyncLambdaWorkflow { + @WorkflowMethod + void run(); + } + + public static class AsyncLambdaWorkflowImpl implements AsyncLambdaWorkflow { + @Override + public void run() { + Span parent = startSpan("asyncLambda", "parent"); + try (Scope ignored = parent.makeCurrent()) { + Async.function( + () -> { + Span child = startSpan("asyncLambda", "child"); + try (Scope ignoredChild = child.makeCurrent()) { + activities().nopActivity(); + } finally { + child.end(); + } + return null; + }) + .get(); + } finally { + parent.end(); + } + } + } + + @WorkflowInterface + public interface ConstructorSpanWorkflow { + @WorkflowMethod + void run(); + } + + public static class ConstructorSpanWorkflowImpl implements ConstructorSpanWorkflow { + public ConstructorSpanWorkflowImpl() { + startSpan("constructorSpan", "constructor").end(); + } + + @Override + public void run() {} + } + + @WorkflowInterface + public interface PromiseCallbackWorkflow { + @WorkflowMethod + void run(); + } + + public static class PromiseCallbackWorkflowImpl implements PromiseCallbackWorkflow { + @Override + public void run() { + Span application = startSpan("promiseCallback", "application"); + try (Scope ignored = application.makeCurrent()) { + Promise continuation = + Async.procedure(activities()::asyncCompletionActivity) + .thenApply(result -> result) + .thenApply( + ignoredResult -> { + Span callback = startSpan("promiseCallback", "callback"); + try (Scope ignoredCallback = callback.makeCurrent()) { + activities().nopActivity(); + } finally { + callback.end(); + } + return null; + }); + continuation.get(); + } finally { + application.end(); + } + } + } + + @WorkflowInterface + public interface BenignErrorWorkflow { + @WorkflowMethod + void run(); + } + + public static class BenignErrorWorkflowImpl implements BenignErrorWorkflow { + @Override + public void run() { + throw ApplicationFailure.newBuilder() + .setMessage("expected error") + .setType("BenignError") + .setCategory(ApplicationErrorCategory.BENIGN) + .build(); + } + } + + @WorkflowInterface + public interface ErrorWorkflow { + @WorkflowMethod + void run(); + } + + public static class ErrorWorkflowImpl implements ErrorWorkflow { + @Override + public void run() { + throw ApplicationFailure.newFailure("unexpected error", "UnexpectedError"); + } + } + + // --------------------------------------------------------------------------------------------- + // Tracer tests + // --------------------------------------------------------------------------------------------- + + @WorkflowInterface + public interface TracerWorkflow { + @WorkflowMethod + void run(boolean end); + + @QueryMethod(name = TRACER_TEST_QUERY_NAME) + String query(); + + @UpdateValidatorMethod(updateName = TRACER_TEST_UPDATE_NAME) + void validateUpdate(); + + @UpdateMethod(name = TRACER_TEST_UPDATE_NAME) + void update(); + + @SignalMethod(name = TRACER_TEST_SIGNAL_NAME) + void gate(); + } + + public static class TracerWorkflowImpl implements TracerWorkflow { + private boolean gated; + + @Override + public void run(boolean end) { + if (!end) { + Workflow.await(() -> gated); + } + + // Both spans end before continue-as-new, so the next run's context starts from the root. + Span beginProcessing = startSpan("processorTracer", "process start"); + try (Scope ignored = beginProcessing.makeCurrent()) { + startSpan("recorderTracer", "record results").end(); + } finally { + beginProcessing.end(); + } + + if (!end) { + Workflow.continueAsNew(true); + } + } + + @Override + public String query() { + startSpan("queryTracer", "query start").end(); + return "ok"; + } + + @Override + public void update() { + startSpan("updateTracer", "update start").end(); + } + + @Override + public void validateUpdate() { + startSpan("validatorTracer", "validate start").end(); + } + + @Override + public void gate() { + gated = true; + } + } + + public static final String OUTBOUND_ACTIVITY_ID = "outbound-activity"; + public static final String OUTBOUND_LOCAL_ACTIVITY_ID = "outbound-local-activity"; + + public enum WorkflowOutboundCall { + ACTIVITY, + LOCAL_ACTIVITY, + CHILD_WORKFLOW, + EXTERNAL_SIGNAL, + EXTERNAL_CANCEL + } + + @WorkflowInterface + public interface WorkflowOutboundTagsWorkflow { + @WorkflowMethod + void run(WorkflowOutboundCall call, WorkflowExecution targetExecution); + } + + public static class WorkflowOutboundTagsWorkflowImpl implements WorkflowOutboundTagsWorkflow { + @Override + public void run(WorkflowOutboundCall call, WorkflowExecution targetExecution) { + switch (call) { + case ACTIVITY: + Workflow.executeActivity( + activities()::activity, + ActivityInvocationOptions.newBuilder().setActivityId(OUTBOUND_ACTIVITY_ID).build()); + break; + case LOCAL_ACTIVITY: + TestActivities localActivities = + Workflow.newLocalActivityStub( + TestActivities.class, + LocalActivityOptions.newBuilder() + .setStartToCloseTimeout(Duration.ofSeconds(10)) + .build()); + Workflow.executeActivity( + localActivities::localActivity, + ActivityInvocationOptions.newBuilder() + .setActivityId(OUTBOUND_LOCAL_ACTIVITY_ID) + .build()); + break; + case CHILD_WORKFLOW: + ChildWorkflowWithSignal child = + Workflow.newChildWorkflowStub(ChildWorkflowWithSignal.class); + Promise childResult = Async.procedure(child::run); + Workflow.getWorkflowExecution(child).get(); + child.childSignal(); + childResult.get(); + break; + case EXTERNAL_SIGNAL: + Workflow.newUntypedExternalWorkflowStub(targetExecution).signal("childSignal"); + break; + case EXTERNAL_CANCEL: + Workflow.newUntypedExternalWorkflowStub(targetExecution).cancel(); + break; + } + } + } + + @WorkflowInterface + public interface ChainedContinueAsNewWorkflow { + @WorkflowMethod + void run(boolean finalRun); + } + + /** Continues as new inside its user span, so the next run parents under that span. */ + public static class ChainedContinueAsNewWorkflowImpl implements ChainedContinueAsNewWorkflow { + @Override + public void run(boolean finalRun) { + spanAround( + "chainedContinueAsNewWorkflow", + "chained-span", + () -> { + if (!finalRun) { + Workflow.continueAsNew(true); + } + }); + } + } + + @WorkflowInterface + public interface ContinueAsNewToDifferentWorkflow { + @WorkflowMethod + void run(); + } + + public static class ContinueAsNewToDifferentWorkflowImpl + implements ContinueAsNewToDifferentWorkflow { + @Override + public void run() { + Workflow.continueAsNew(DifferentWorkflow.class.getSimpleName(), null); + } + } + + @WorkflowInterface + public interface DifferentWorkflow { + @WorkflowMethod + void run(); + } + + public static class DifferentWorkflowImpl implements DifferentWorkflow { + @Override + public void run() {} + } + + @WorkflowInterface + public interface TracerResetWorkflow { + @WorkflowMethod + void run(); + } + + public static class TracerResetWorkflowImpl implements TracerResetWorkflow { + @Override + public void run() { + // Ended but kept current, so later spans parent to it. + Span beginProcessing = startSpan("processorTracer", "process start"); + try (Scope ignored = beginProcessing.makeCurrent()) { + beginProcessing.end(); + Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()) + .nopActivity(); + startSpan("recorderTracer", "record results").end(); + } + } + } + + @WorkflowInterface + public interface TracerResetLateSourceWorkflow { + @WorkflowMethod + void run(); + } + + /** Obtains the second tracer only after the reset point. */ + public static class TracerResetLateSourceWorkflowImpl implements TracerResetLateSourceWorkflow { + @Override + public void run() { + Span beginProcessing = startSpan("processorTracer", "process start"); + try (Scope ignored = beginProcessing.makeCurrent()) { + beginProcessing.end(); + Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()) + .nopActivity(); + Tracer recorder = GlobalOpenTelemetry.getTracer("recorderTracer"); + recorder.spanBuilder("record results").startSpan().end(); + } + } + } + + @WorkflowInterface + public interface TracerResetDuringSpan { + @WorkflowMethod + void run(); + } + + /** Both spans stay open across the reset point. */ + public static class TracerResetDuringSpanImpl implements TracerResetDuringSpan { + @Override + public void run() { + Span beginProcessing = startSpan("processorTracer", "process start"); + try (Scope ignored = beginProcessing.makeCurrent()) { + Span recordingResults = startSpan("recorderTracer", "record results"); + Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()) + .nopActivity(); + beginProcessing.end(); + recordingResults.end(); + } + } + } + + @WorkflowInterface + public interface TracerWorkflowTaskRetry { + @WorkflowMethod + void run(); + } + + public static class TracerWorkflowTaskRetryImpl implements TracerWorkflowTaskRetry { + @Override + public void run() { + startSpan("test", "workflow-task-retry-span").end(); + throw new RuntimeException("intentional workflow task failure"); + } + } + + @WorkflowInterface + public interface TracerSpanTimestampWorkflow { + @WorkflowMethod + void run(); + + @QueryMethod + String query(); + } + + public static class TracerSpanTimestampWorkflowImpl implements TracerSpanTimestampWorkflow { + @Override + public void run() { + Span span = + GlobalOpenTelemetry.getTracer("timestampTracer") + .spanBuilder("explicit timestamp") + .setStartTimestamp(123456789L, TimeUnit.MILLISECONDS) + .startSpan(); + try { + Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()) + .nopActivity(); + } finally { + span.end(); + } + } + + @Override + public String query() { + return "ok"; + } + } + + @WorkflowInterface + public interface TracerSpanEventWorkflow { + @WorkflowMethod + void run(); + + @QueryMethod + String query(); + } + + public static class TracerSpanEventWorkflowImpl implements TracerSpanEventWorkflow { + @Override + public void run() { + Span span = startSpan("eventTracer", "span with event"); + span.addEvent("event"); + span.end(); + } + + @Override + public String query() { + return "ok"; + } + } + + @WorkflowInterface + public interface TracerSpanAcrossEvictionWorkflow { + @WorkflowMethod + void run(); + + @SignalMethod + void finish(); + } + + public static class TracerSpanAcrossEvictionWorkflowImpl + implements TracerSpanAcrossEvictionWorkflow { + private boolean finished; + + @Override + public void run() { + spanAround("evictionTracer", "span across eviction", () -> Workflow.await(() -> finished)); + } + + @Override + public void finish() { + finished = true; + } + } + + @WorkflowInterface + public interface TracerSpanEndedTwiceWorkflow { + @WorkflowMethod + void run(); + + @SignalMethod + void finish(); + } + + public static class TracerSpanEndedTwiceWorkflowImpl implements TracerSpanEndedTwiceWorkflow { + private boolean finished; + + @Override + public void run() { + Span span = startSpan("endedTwiceTracer", "span ended twice"); + span.end(); + Workflow.await(() -> finished); + span.end(); + } + + @Override + public void finish() { + finished = true; + } + } + + // --------------------------------------------------------------------------------------------- + // Comprehensive scenario + // --------------------------------------------------------------------------------------------- + + @WorkflowInterface + public interface ExternalWorkflowWithSignal { + @WorkflowMethod + String run(); + + @SignalMethod + void externalSignal(); + + @SignalMethod + void scheduleStarted(String workflowId); + } + + public static class ExternalWorkflowWithSignalImpl implements ExternalWorkflowWithSignal { + private boolean externalSignaled; + private String scheduledWorkflowId; + + @Override + public String run() { + spanAround( + "externalWorkflowWithSignal", + "external-workflow-with-signal-span", + () -> Workflow.await(() -> externalSignaled && scheduledWorkflowId != null)); + return scheduledWorkflowId; + } + + @Override + public void externalSignal() { + externalSignaled = true; + } + + @Override + public void scheduleStarted(String workflowId) { + scheduledWorkflowId = workflowId; + } + } + + @WorkflowInterface + public interface ChildWorkflowWithSignal { + @WorkflowMethod + void run(); + + @SignalMethod + void childSignal(); + } + + public static class ChildWorkflowWithSignalImpl implements ChildWorkflowWithSignal { + private boolean signaled; + + @Override + public void run() { + spanAround( + "childWorkflowWithSignal", + "child-workflow-with-signal-span", + () -> Workflow.await(() -> signaled)); + } + + @Override + public void childSignal() { + signaled = true; + } + } + + @WorkflowInterface + public interface NexusHandlerWorkflow { + @WorkflowMethod + Void run(String input); + } + + public static class NexusHandlerWorkflowImpl implements NexusHandlerWorkflow { + @Override + public Void run(String input) { + spanAround("workflowWithNexusHandler", "workflow-with-nexus-handler-span", () -> {}); + return null; + } + } + + @WorkflowInterface + public interface NexusCancelHandlerWorkflow { + @WorkflowMethod + Void run(String input); + } + + /** Waits until the Nexus caller cancels it. */ + public static class NexusCancelHandlerWorkflowImpl implements NexusCancelHandlerWorkflow { + @Override + public Void run(String input) { + spanAround( + "nexusCancelHandlerWorkflow", + "nexus-cancel-handler-span", + () -> Workflow.await(() -> false)); + return null; + } + } + + @Service(name = NEXUS_SERVICE_NAME) + public interface ComprehensiveNexusService { + @Operation(name = NEXUS_OPERATION_NAME) + Void nexusHandlerWorkflow(String input); + + @Operation(name = NEXUS_CANCEL_OPERATION_NAME) + Void nexusCancelHandlerWorkflow(String input); + } + + @ServiceImpl(service = ComprehensiveNexusService.class) + public static class ComprehensiveNexusServiceImpl { + @OperationImpl + public OperationHandler nexusHandlerWorkflow() { + return WorkflowRunOperation.fromWorkflowMethod( + (context, details, input) -> + Nexus.getOperationContext() + .getWorkflowClient() + .newWorkflowStub( + NexusHandlerWorkflow.class, + WorkflowOptions.newBuilder() + .setWorkflowId("nexus-handler-" + details.getRequestId()) + .build()) + ::run); + } + + @OperationImpl + public OperationHandler nexusCancelHandlerWorkflow() { + return WorkflowRunOperation.fromWorkflowMethod( + (context, details, input) -> + Nexus.getOperationContext() + .getWorkflowClient() + .newWorkflowStub( + NexusCancelHandlerWorkflow.class, + WorkflowOptions.newBuilder() + .setWorkflowId("nexus-cancel-handler-" + details.getRequestId()) + .build()) + ::run); + } + } + + @WorkflowInterface + public interface ComprehensiveWorkflow { + @WorkflowMethod + void run( + boolean finalRun, String nexusEndpoint, String externalWorkflowId, String cancelTargetId); + + @QueryMethod(name = "getStatus") + String getStatus(); + + @UpdateMethod(name = "testUpdate") + void testUpdate(); + + @UpdateValidatorMethod(updateName = "testUpdate") + void validateTestUpdate(); + + @SignalMethod + void proceed(); + } + + public static class ComprehensiveWorkflowImpl implements ComprehensiveWorkflow { + private boolean proceed; + + @Override + public void run( + boolean finalRun, String nexusEndpoint, String externalWorkflowId, String cancelTargetId) { + // The returned context is discarded, so the span is not made current. The outbound calls + // below parent to the RunWorkflow span and this span is their sibling. + Span span = startSpan("comprehensiveWorkflow", "comprehensive-outbound-workflow-span"); + try { + if (finalRun) { + return; + } + + Workflow.await(() -> proceed); + + activities().activity(); + Workflow.newLocalActivityStub( + TestActivities.class, + LocalActivityOptions.newBuilder() + .setStartToCloseTimeout(Duration.ofSeconds(10)) + .build()) + .localActivity(); + + Span timerRegistration = startSpan("comprehensiveWorkflow", "timer-registration-span"); + Promise timerContinuation; + try (Scope ignoredRegistration = timerRegistration.makeCurrent()) { + timerContinuation = + Workflow.newTimer(Duration.ofMillis(1)) + .thenApply( + ignoredResult -> { + startSpan("comprehensiveWorkflow", "timer-callback-span").end(); + return null; + }); + } finally { + timerRegistration.end(); + } + timerContinuation.get(); + + ChildWorkflowWithSignal child = + Workflow.newChildWorkflowStub(ChildWorkflowWithSignal.class); + Promise childResult = Async.procedure(child::run); + Span registration = startSpan("comprehensiveWorkflow", "promise-registration-span"); + Promise childContinuation; + try (Scope ignoredRegistration = registration.makeCurrent()) { + childContinuation = + childResult.thenApply( + ignoredResult -> { + startSpan("comprehensiveWorkflow", "promise-callback-span").end(); + return null; + }); + } finally { + registration.end(); + } + Workflow.getWorkflowExecution(child).get(); + child.childSignal(); + childContinuation.get(); + + Workflow.newExternalWorkflowStub(ExternalWorkflowWithSignal.class, externalWorkflowId) + .externalSignal(); + + Span cancelRegistration = startSpan("comprehensiveWorkflow", "cancel-registration-span"); + Promise cancelCallback; + try (Scope ignoredRegistration = cancelRegistration.makeCurrent()) { + cancelCallback = + Async.procedure( + () -> Workflow.newUntypedExternalWorkflowStub(cancelTargetId).cancel()) + .thenApply( + ignoredResult -> { + startSpan("comprehensiveWorkflow", "cancel-callback-span").end(); + return null; + }); + } finally { + cancelRegistration.end(); + } + cancelCallback.get(); + + ComprehensiveNexusService nexus = + Workflow.newNexusServiceStub( + ComprehensiveNexusService.class, + NexusServiceOptions.newBuilder() + .setEndpoint(nexusEndpoint) + .setOperationOptions( + NexusOperationOptions.newBuilder() + .setScheduleToCloseTimeout(Duration.ofSeconds(10)) + .build()) + .build()); + nexus.nexusHandlerWorkflow(""); + + try { + Workflow.newCancellationScope( + () -> { + NexusOperationHandle handle = + Workflow.startNexusOperation(nexus::nexusCancelHandlerWorkflow, ""); + handle.getExecution().get(); + CancellationScope.current().cancel(); + handle.getResult().get(); + }) + .run(); + } catch (NexusOperationFailure failure) { + // Cancellation is expected. + if (!(failure.getCause() instanceof CanceledFailure)) { + throw failure; + } + } + + Workflow.continueAsNew(true, nexusEndpoint, externalWorkflowId, cancelTargetId); + } finally { + span.end(); + } + } + + @Override + public String getStatus() { + spanAround( + "comprehensiveWorkflow", + "query-handler-span", + () -> startSpan("comprehensiveWorkflow", "query-handler-child-span").end()); + return "ok"; + } + + @Override + public void testUpdate() { + spanAround( + "comprehensiveWorkflow", + "update-handler-span", + () -> startSpan("comprehensiveWorkflow", "update-handler-child-span").end()); + } + + @Override + public void validateTestUpdate() { + spanAround( + "comprehensiveWorkflow", + "validate-update-span", + () -> startSpan("comprehensiveWorkflow", "validate-update-span-child").end()); + } + + @Override + public void proceed() { + proceed = true; + } + } + + @WorkflowInterface + public interface StandaloneWorkflow { + @WorkflowMethod + void run(String signalReceiverWorkflowId); + } + + public static class StandaloneWorkflowImpl implements StandaloneWorkflow { + @Override + public void run(String signalReceiverWorkflowId) { + if (signalReceiverWorkflowId != null) { + spanAround( + "standaloneWorkflow", + "standalone-workflow-span", + () -> + Workflow.newExternalWorkflowStub( + ExternalWorkflowWithSignal.class, signalReceiverWorkflowId) + .scheduleStarted(Workflow.getInfo().getWorkflowId())); + } + } + } + + @WorkflowInterface + public interface SchedulePropagationWorkflow { + @WorkflowMethod + void run(String signalReceiverWorkflowId); + } + + public static class SchedulePropagationWorkflowImpl implements SchedulePropagationWorkflow { + @Override + public void run(String signalReceiverWorkflowId) { + spanAround( + "schedulePropagationWorkflow", + "schedule-propagation-workflow-span", + () -> + Workflow.newExternalWorkflowStub( + SchedulePropagationReceiver.class, signalReceiverWorkflowId) + .scheduleStarted(Workflow.getInfo().getWorkflowId())); + } + } + + @WorkflowInterface + public interface SchedulePropagationReceiver { + @WorkflowMethod + String run(); + + @SignalMethod + void scheduleStarted(String workflowId); + } + + public static class SchedulePropagationReceiverImpl implements SchedulePropagationReceiver { + private String scheduledWorkflowId; + + @Override + public String run() { + Workflow.await(() -> scheduledWorkflowId != null); + return scheduledWorkflowId; + } + + @Override + public void scheduleStarted(String workflowId) { + scheduledWorkflowId = workflowId; + } + } + + @WorkflowInterface + public interface UnservedWorkflow { + @WorkflowMethod + void run(); + } + + public static class UnservedWorkflowImpl implements UnservedWorkflow { + @Override + public void run() {} + } + + @WorkflowInterface + public interface SignalWithStartTarget { + @WorkflowMethod + void run(); + + @SignalMethod + void startSignal(); + } + + public static class SignalWithStartTargetImpl implements SignalWithStartTarget { + private boolean signaled; + + @Override + public void run() { + spanAround( + "signalWithStartTarget", + "signal-with-start-target-span", + () -> Workflow.await(() -> signaled)); + } + + @Override + public void startSignal() { + signaled = true; + } + } + + @WorkflowInterface + public interface UpdateTargetWorkflow { + @WorkflowMethod + void run(); + + @UpdateMethod(name = "doUpdate") + void doUpdate(); + + @SignalMethod + void updateSignal(); + } + + public static class UpdateTargetWorkflowImpl implements UpdateTargetWorkflow { + private boolean signaled; + + @Override + public void run() { + spanAround( + "updateTargetWorkflow", + "update-target-workflow-span", + () -> Workflow.await(() -> signaled)); + } + + @Override + public void doUpdate() { + startSpan("updateTracer", "update start").end(); + } + + @Override + public void updateSignal() { + signaled = true; + } + } + + @WorkflowInterface + public interface AsyncCompletionWorkflow { + @WorkflowMethod + void run(); + } + + public static class AsyncCompletionWorkflowImpl implements AsyncCompletionWorkflow { + @Override + public void run() { + Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(15)).build()) + .asyncCompletionActivity(); + } + } + + private static TestActivities activities() { + return Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TracerTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TracerTest.java new file mode 100644 index 0000000000..5c151be4e0 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TracerTest.java @@ -0,0 +1,281 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +import io.opentelemetry.sdk.trace.data.SpanData; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.api.enums.v1.EventType; +import io.temporal.api.history.v1.HistoryEvent; +import io.temporal.api.workflowservice.v1.ResetWorkflowExecutionRequest; +import io.temporal.api.workflowservice.v1.ResetWorkflowExecutionResponse; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowFailedException; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.client.WorkflowTargetOptions; +import io.temporal.common.RetryOptions; +import io.temporal.failure.TimeoutFailure; +import io.temporal.opentelemetry.v2.TestWorkflows.ChainedContinueAsNewWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.ChainedContinueAsNewWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TestActivitiesImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerResetDuringSpanImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerResetLateSourceWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerResetWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanAcrossEvictionWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanAcrossEvictionWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanEndedTwiceWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanEndedTwiceWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanEventWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanEventWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanTimestampWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanTimestampWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflowTaskRetry; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflowTaskRetryImpl; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.junit.Rule; +import org.junit.Test; + +/** + * Verifies that spans started by workflow code through the replay-safe global preserve parentage, + * with Temporal spans disabled. + */ +public class TracerTest extends OtelTestBase { + @Rule + public SDKTestWorkflowRule testWorkflowRule = + newRuleBuilder(false) + .setWorkflowTypes( + TracerWorkflowImpl.class, + ChainedContinueAsNewWorkflowImpl.class, + TracerResetWorkflowImpl.class, + TracerResetLateSourceWorkflowImpl.class, + TracerResetDuringSpanImpl.class, + TracerWorkflowTaskRetryImpl.class, + TracerSpanTimestampWorkflowImpl.class, + TracerSpanEventWorkflowImpl.class, + TracerSpanAcrossEvictionWorkflowImpl.class, + TracerSpanEndedTwiceWorkflowImpl.class) + .setActivityImplementations(new TestActivitiesImpl()) + .build(); + + @Test + public void tracerWorkflow() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowClient.start(workflow::run, false); + + workflow.update(); + workflow.query(); + workflow.gate(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + workflow.query(); + + List spans = endedSpans(); + assertSpanTree( + Arrays.asList( + "validate start", + "update start", + "query start", + "process start", + " record results", + "process start", // ContinueAsNew + " record results", + // The query after completion replays the run with no span current, so its span is a + // root like the first query's. + "query start"), + spans); + requireUniqueSpanIds(spans); + } + + @Test + public void continueAsNewUnderUserSpan() { + testWorkflowRule.newWorkflowStub(ChainedContinueAsNewWorkflow.class).run(false); + + List spans = endedSpans(); + assertSpanTree(Arrays.asList("chained-span", " chained-span"), spans); + requireUniqueSpanIds(spans); + } + + @Test + public void workflowTaskRetryReusesSpanId() { + TracerWorkflowTaskRetry workflow = + testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + TracerWorkflowTaskRetry.class, + WorkflowOptions.newBuilder() + .setTaskQueue(testWorkflowRule.getTaskQueue()) + .setWorkflowRunTimeout(Duration.ofSeconds(1)) + .setRetryOptions(RetryOptions.newBuilder().setMaximumAttempts(1).build()) + .build()); + try { + workflow.run(); + throw new AssertionError("expected the workflow run to time out"); + } catch (WorkflowFailedException e) { + assertTrue(e.getCause().toString(), e.getCause() instanceof TimeoutFailure); + } + + List spans = endedSpans(); + assertTrue("expected the span from more than one task attempt", spans.size() > 1); + Set spanIds = new HashSet<>(); + for (SpanData span : spans) { + spanIds.add(span.getSpanId()); + } + assertEquals(spanTree(spans).toString(), 1, spanIds.size()); + } + + @Test + public void explicitSpanStartTimestampSurvivesReplay() { + TracerSpanTimestampWorkflow workflow = + testWorkflowRule.newWorkflowStub(TracerSpanTimestampWorkflow.class); + workflow.run(); + + assertEquals(1, endedSpans().size()); + SpanData span = requireSpanNamed(endedSpans(), "explicit timestamp"); + assertEquals(TimeUnit.MILLISECONDS.toNanos(123456789L), span.getStartEpochNanos()); + + // Querying the completed workflow replays its history. + workflow.query(); + + assertEquals(1, endedSpans().size()); + span = requireSpanNamed(endedSpans(), "explicit timestamp"); + assertEquals(TimeUnit.MILLISECONDS.toNanos(123456789L), span.getStartEpochNanos()); + } + + @Test + public void spanEventTimestampSurvivesReplay() { + TracerSpanEventWorkflow workflow = + testWorkflowRule.newWorkflowStub(TracerSpanEventWorkflow.class); + workflow.run(); + + List spans = endedSpans(); + assertEquals(1, spans.size()); + assertEquals(1, spans.get(0).getEvents().size()); + long eventNanos = spans.get(0).getEvents().get(0).getEpochNanos(); + + // Querying the completed workflow replays its history. + workflow.query(); + + spans = endedSpans(); + assertEquals(1, spans.size()); + assertEquals(1, spans.get(0).getEvents().size()); + assertEquals(eventNanos, spans.get(0).getEvents().get(0).getEpochNanos()); + } + + @Test + public void evictionDoesNotEndOpenSpan() { + TracerSpanAcrossEvictionWorkflow workflow = + testWorkflowRule.newWorkflowStub(TracerSpanAcrossEvictionWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run); + testWorkflowRule.waitForTheEndOfWFT(execution.getWorkflowId()); + testWorkflowRule.invalidateWorkflowCache(); + + assertEquals(0, endedSpans().size()); + + workflow.finish(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + List spans = endedSpans(); + assertEquals(1, spans.size()); + assertEquals("span across eviction", spans.get(0).getName()); + } + + @Test + public void secondEndAfterEvictionDoesNotExportAgain() { + TracerSpanEndedTwiceWorkflow workflow = + testWorkflowRule.newWorkflowStub(TracerSpanEndedTwiceWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run); + testWorkflowRule.waitForTheEndOfWFT(execution.getWorkflowId()); + testWorkflowRule.invalidateWorkflowCache(); + + workflow.finish(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + List spans = endedSpans(); + assertEquals(1, spans.size()); + assertEquals("span ended twice", spans.get(0).getName()); + } + + @Test + public void resetWithTracerCreatedBeforeResetPoint() { + List spans = runAndReset(TracerResetWorkflowImpl.class); + assertSpanTree(Arrays.asList("process start", " record results", " record results"), spans); + requireUniqueSpanIds(spans); + } + + @Test + public void resetWithTracerCreatedAfterResetPoint() { + List spans = runAndReset(TracerResetLateSourceWorkflowImpl.class); + assertSpanTree(Arrays.asList("process start", " record results", " record results"), spans); + requireUniqueSpanIds(spans); + } + + @Test + public void resetWithSpanCrossingResetPoint() { + List spans = runAndReset(TracerResetDuringSpanImpl.class); + assertSpanTree( + Arrays.asList("process start", " record results", "process start", " record results"), + spans); + // These spans reuse their IDs because they were created before the reset point. + assertEquals(spans.get(0).getSpanContext(), spans.get(2).getSpanContext()); + assertEquals(spans.get(1).getSpanContext(), spans.get(3).getSpanContext()); + // The new spans end after the old ones. + assertTrue(spans.get(2).getEndEpochNanos() > spans.get(0).getEndEpochNanos()); + assertTrue(spans.get(3).getEndEpochNanos() > spans.get(1).getEndEpochNanos()); + } + + /** + * Runs a gated workflow to completion, resets it to the task that handled the gate signal so the + * work after the signal is redone, and returns every span from both runs. + */ + private List runAndReset(Class workflowImpl) { + assumeTrue( + "Test Server doesn't support reset workflow", SDKTestWorkflowRule.useExternalService); + WorkflowClient client = testWorkflowRule.getWorkflowClient(); + WorkflowStub stub = + client.newUntypedWorkflowStub( + workflowImpl.getInterfaces()[0].getSimpleName(), + WorkflowOptions.newBuilder().setTaskQueue(testWorkflowRule.getTaskQueue()).build()); + stub.start(); + WorkflowExecution execution = stub.getExecution(); + stub.getResult(Void.class); + + ResetWorkflowExecutionResponse response = + client + .getWorkflowServiceStubs() + .blockingStub() + .resetWorkflowExecution( + ResetWorkflowExecutionRequest.newBuilder() + .setNamespace(client.getOptions().getNamespace()) + .setWorkflowExecution(execution) + .setWorkflowTaskFinishEventId(secondWorkflowTaskCompletedEventId(execution)) + .setReason("Integration test") + .setRequestId(UUID.randomUUID().toString()) + .build()); + client + .newUntypedWorkflowStub( + WorkflowTargetOptions.newBuilder() + .setWorkflowId(execution.getWorkflowId()) + .setRunId(response.getRunId()) + .build()) + .getResult(Void.class); + + return endedSpans(); + } + + private long secondWorkflowTaskCompletedEventId(WorkflowExecution execution) { + List completed = + testWorkflowRule.getHistoryEvents( + execution.getWorkflowId(), EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED); + return completed.get(1).getEventId(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/CodecTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/CodecTest.java new file mode 100644 index 0000000000..45462f7e23 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/CodecTest.java @@ -0,0 +1,172 @@ +package io.temporal.opentelemetry.v2.internal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.google.protobuf.ByteString; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.baggage.Baggage; +import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanId; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceId; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.temporal.api.common.v1.Payload; +import io.temporal.common.converter.DataConverter; +import io.temporal.common.converter.DefaultDataConverter; +import io.temporal.common.converter.GlobalDataConverter; +import io.temporal.common.converter.JacksonJsonPayloadConverter; +import io.temporal.common.interceptors.Header; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** Unit tests for {@link SpanCodec} write/read on Temporal headers and Nexus maps. */ +public class CodecTest { + private static final String HEADER_KEY = "_tracer-data"; + private static final SpanContext SPAN = + SpanContext.create( + TraceId.fromLongs(0, 1), + SpanId.fromLong(2), + TraceFlags.getSampled(), + TraceState.getDefault()); + private static final Baggage BAGGAGE = Baggage.builder().put("key", "value").build(); + + private SpanCodec codec; + + @Before + public void installPropagators() { + GlobalOpenTelemetry.resetForTest(); + GlobalOpenTelemetry.set( + OpenTelemetry.propagating( + ContextPropagators.create( + TextMapPropagator.composite( + W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance())))); + codec = new SpanCodec(HEADER_KEY); + } + + @After + public void resetGlobalOpenTelemetry() { + GlobalOpenTelemetry.resetForTest(); + } + + @Test + public void writesAndReadsTemporalHeaders() { + Header header = new Header(new HashMap<>()); + try (Scope ignored = current().makeCurrent()) { + codec.write(header); + } + assertPropagated(codec.read(header)); + } + + @Test + public void serializesSuppliedContext() { + Map header; + try (Scope ignored = Context.root().makeCurrent()) { + header = codec.serialize(current()); + } + assertPropagated(codec.readFromRoot(header)); + } + + @Test + public void writesAndReadsNexusHeaders() { + Map headers = new HashMap<>(); + try (Scope ignored = current().makeCurrent()) { + codec.write(headers); + } + assertPropagated(codec.read(headers)); + } + + @Test + public void leavesTemporalHeaderAloneWhenNothingToInject() { + Header header = new Header(new HashMap<>()); + try (Scope ignored = Context.root().makeCurrent()) { + codec.write(header); + } + assertNull(header.getValues().get(HEADER_KEY)); + } + + @Test + public void missingTemporalHeaderKeepsCurrentContext() { + try (Scope ignored = current().makeCurrent()) { + assertPropagated(codec.read(new Header(new HashMap<>()))); + } + } + + @Test + public void readsNexusHeadersCaseInsensitively() { + Map written = new HashMap<>(); + try (Scope ignored = current().makeCurrent()) { + codec.write(written); + } + Map upper = new HashMap<>(); + written.forEach((key, value) -> upper.put(key.toUpperCase(), value)); + assertPropagated(codec.read(upper)); + } + + @Test + public void preservesNexusHeaderKeyCase() { + Map headers = new HashMap<>(); + try (Scope ignored = current().makeCurrent()) { + codec.write(headers); + } + assertTrue(headers.containsKey("traceparent")); + assertTrue(headers.containsKey("baggage")); + } + + @Test + public void decodesPropertiesPayload() { + Properties carrier = new Properties(); + carrier.setProperty("traceparent", "00-trace-id-span-id-01"); + Payload payload = DefaultDataConverter.STANDARD_INSTANCE.toPayload(carrier).get(); + assertEquals(carrier, SpanCodec.decode(payload)); + } + + @Test + public void decodesLegacyMapPayloadViaGlobalConverter() { + Map carrier = new HashMap<>(); + carrier.put("traceparent", "00-trace-id-span-id-01"); + Payload payload = + Payload.newBuilder() + .putMetadata("encoding", ByteString.copyFromUtf8("legacy/json")) + .build(); + + DataConverter original = GlobalDataConverter.get(); + GlobalDataConverter.register( + new DefaultDataConverter(new JacksonJsonPayloadConverter()) { + @Override + public T fromPayload(Payload ignored, Class valueClass, Type type) { + return valueClass.cast(carrier); + } + }); + try { + assertEquals(carrier, SpanCodec.decode(payload)); + } finally { + GlobalDataConverter.register(original); + } + } + + private static Context current() { + return Context.root().with(Span.wrap(SPAN)).with(BAGGAGE); + } + + private static void assertPropagated(Context context) { + SpanContext span = Span.fromContext(context).getSpanContext(); + assertEquals(SPAN.getTraceId(), span.getTraceId()); + assertEquals(SPAN.getSpanId(), span.getSpanId()); + assertEquals("value", Baggage.fromContext(context).getEntryValue("key")); + } +} diff --git a/settings.gradle b/settings.gradle index a2ee73e91a..808f83481d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -20,6 +20,8 @@ include 'temporal-opentracing' project(':temporal-opentracing').projectDir = file('contrib/temporal-opentracing') include 'temporal-opentelemetry' project(':temporal-opentelemetry').projectDir = file('contrib/temporal-opentelemetry') +include 'temporal-opentelemetry-v2' +project(':temporal-opentelemetry-v2').projectDir = file('contrib/temporal-opentelemetry-v2') include 'temporal-kotlin' // Retained for the one-time relocation POM; implementation lives in ai-integrations. include 'temporal-spring-ai' diff --git a/temporal-bom/build.gradle b/temporal-bom/build.gradle index 0bbae3e4bd..7c7847fa2a 100644 --- a/temporal-bom/build.gradle +++ b/temporal-bom/build.gradle @@ -8,6 +8,7 @@ dependencies { constraints { api project(':temporal-kotlin') api project(':temporal-opentelemetry') + api project(':temporal-opentelemetry-v2') api project(':temporal-opentracing') api project(':temporal-aws-lambda') api project(':temporal-gcp-cloud-run-opentelemetry') From 0abccd71ede9aaa74f60879c8844d6ca18840616 Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Wed, 14 Oct 2026 10:16:39 -0400 Subject: [PATCH 2/6] Make the OpenTelemetry v2 meter provider replay safe --- CHANGELOG.md | 3 +- .../v2/ReplaySafeOpenTelemetry.java | 65 +- .../v2/internal/OpenTelemetrySuppression.java | 18 + .../v2/internal/ReplaySafeMeter.java | 691 ++++++++++++++++++ .../v2/internal/ReplaySafeSpan.java | 13 +- .../v2/internal/ReplaySafeTracer.java | 6 +- .../opentelemetry/v2/MetricsTest.java | 122 ++++ .../opentelemetry/v2/OtelTestBase.java | 15 + 8 files changed, 913 insertions(+), 20 deletions(-) create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetrySuppression.java create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 4efa44121c..c3b3e276c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,8 @@ to docs, or any other relevant information. `addScheduleClientInterceptors`, `addActivityClientInterceptors`, and `addNexusClientInterceptors`. - Added the experimental `temporal-opentelemetry-v2` module. `OpenTelemetryPlugin` propagates OpenTelemetry context through Temporal headers and can optionally create spans for Temporal operations. `ReplaySafeOpenTelemetry` makes - workflow spans replay safe with deterministic span and trace IDs. + workflow spans replay safe with deterministic span and trace IDs, and drops metric recordings made by replaying + workflow code. ### Changed - Moved the Spring AI integration to [temporalio/ai-integrations](https://github.com/temporalio/ai-integrations/tree/main/java/spring-ai). diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java index 15646a212c..1bccd9be0b 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java @@ -4,6 +4,8 @@ import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; import io.opentelemetry.api.logs.LoggerProvider; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterBuilder; import io.opentelemetry.api.metrics.MeterProvider; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.api.trace.TracerBuilder; @@ -21,6 +23,7 @@ import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.temporal.common.Experimental; import io.temporal.opentelemetry.v2.internal.ReplaySafeIdGenerator; +import io.temporal.opentelemetry.v2.internal.ReplaySafeMeter; import io.temporal.opentelemetry.v2.internal.ReplaySafeTracer; import io.temporal.opentelemetry.v2.internal.TemporalContextStorage; import java.io.Closeable; @@ -28,7 +31,8 @@ /** * The {@link OpenTelemetry} to use for OpenTelemetry integration with Temporal. Register it with - * {@code GlobalOpenTelemetry.set}; tracers obtained from it are replay safe inside workflows. + * {@code GlobalOpenTelemetry.set}; tracers and meters obtained from it are replay safe inside + * workflows. * *

Known limitations: * @@ -36,6 +40,10 @@ *

  • {@code SpanProcessor.onStart} runs for spans started during replay, but {@code onEnd} does * not run for spans ended during replay, so do not rely on the two being called the same * number of times. + *
  • Observable instruments, from {@code buildWithCallback}, {@code buildObserver}, or {@code + * batchCallback}, throw {@link IllegalStateException} when created from workflow code, + * because their callbacks would stay registered across replays. Query handlers are not + * checked, so do not create observable instruments in them either. *
  • {@code Span.recordException} takes no timestamp, so an exception recorded during replay is * timestamped with the replay time rather than the workflow time. *
  • Continuations keep the context current when they were registered only on promises returned @@ -43,13 +51,14 @@ * Promise.allOf} and {@code Promise.anyOf}, run with the context current when the promise * completes. *
  • Continuations on the promise returned by cancelling an external workflow run outside a - * workflow thread, so spans they start get random IDs and are exported again on every replay. + * workflow thread, so spans they start get random IDs, and their spans and metric recordings + * are exported again on every replay. * */ @Experimental public final class ReplaySafeOpenTelemetry implements OpenTelemetry, Closeable { private final ReplaySafeTracerProvider tracerProvider; - private final SdkMeterProvider meterProvider; // TODO - Make the meter provider replay safe + private final ReplaySafeMeterProvider meterProvider; // TODO: Make the logger provider replay safe and add logger interceptor methods for Temporal and // OpenTelemetry loggers. private final SdkLoggerProvider loggerProvider; @@ -67,7 +76,7 @@ private ReplaySafeOpenTelemetry(Builder builder) { .tracerProviderBuilder .setIdGenerator(new ReplaySafeIdGenerator(builder.idGenerator)) .build()); - this.meterProvider = builder.meterProviderBuilder.build(); + this.meterProvider = new ReplaySafeMeterProvider(builder.meterProviderBuilder.build()); this.loggerProvider = builder.loggerProviderBuilder.build(); this.propagators = builder.propagators; } @@ -196,6 +205,54 @@ public void close() { } } + private static final class ReplaySafeMeterProvider implements MeterProvider, Closeable { + private final SdkMeterProvider delegate; + + private ReplaySafeMeterProvider(SdkMeterProvider delegate) { + this.delegate = delegate; + } + + @Override + public Meter get(@Nonnull String instrumentationScopeName) { + return new ReplaySafeMeter(delegate.get(instrumentationScopeName)); + } + + @Override + public MeterBuilder meterBuilder(@Nonnull String instrumentationScopeName) { + return new ReplaySafeMeterBuilder(delegate.meterBuilder(instrumentationScopeName)); + } + + @Override + public void close() { + delegate.close(); + } + } + + private static final class ReplaySafeMeterBuilder implements MeterBuilder { + private final MeterBuilder delegate; + + private ReplaySafeMeterBuilder(MeterBuilder delegate) { + this.delegate = delegate; + } + + @Override + public MeterBuilder setSchemaUrl(@Nonnull String schemaUrl) { + delegate.setSchemaUrl(schemaUrl); + return this; + } + + @Override + public MeterBuilder setInstrumentationVersion(@Nonnull String instrumentationScopeVersion) { + delegate.setInstrumentationVersion(instrumentationScopeVersion); + return this; + } + + @Override + public Meter build() { + return new ReplaySafeMeter(delegate.build()); + } + } + private static final class ReplaySafeTracerBuilder implements TracerBuilder { private final TracerBuilder delegate; private final String instrumentationScopeName; diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetrySuppression.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetrySuppression.java new file mode 100644 index 0000000000..881639ebc6 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetrySuppression.java @@ -0,0 +1,18 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.temporal.workflow.unsafe.WorkflowUnsafe; + +/** Where replayed workflow code must not export telemetry again. */ +public final class OpenTelemetrySuppression { + private OpenTelemetrySuppression() {} + + /** + * True on a workflow thread that is re-executing history. Read-only code, such as query handlers, + * update validators, and await conditions, is never suppressed. It does not run a deterministic + * number of times, so its telemetry is emitted every time, with IDs that do not come from a + * random stream. + */ + public static boolean shouldSuppress() { + return !WorkflowUnsafe.isReadOnly() && WorkflowUnsafe.isReplaying(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java new file mode 100644 index 0000000000..9d63c94801 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java @@ -0,0 +1,691 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.DoubleCounter; +import io.opentelemetry.api.metrics.DoubleCounterBuilder; +import io.opentelemetry.api.metrics.DoubleGauge; +import io.opentelemetry.api.metrics.DoubleGaugeBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; +import io.opentelemetry.api.metrics.DoubleUpDownCounter; +import io.opentelemetry.api.metrics.DoubleUpDownCounterBuilder; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongCounterBuilder; +import io.opentelemetry.api.metrics.LongGauge; +import io.opentelemetry.api.metrics.LongGaugeBuilder; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.api.metrics.LongHistogramBuilder; +import io.opentelemetry.api.metrics.LongUpDownCounter; +import io.opentelemetry.api.metrics.LongUpDownCounterBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.ObservableDoubleCounter; +import io.opentelemetry.api.metrics.ObservableDoubleGauge; +import io.opentelemetry.api.metrics.ObservableDoubleMeasurement; +import io.opentelemetry.api.metrics.ObservableDoubleUpDownCounter; +import io.opentelemetry.api.metrics.ObservableLongCounter; +import io.opentelemetry.api.metrics.ObservableLongGauge; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; +import io.opentelemetry.api.metrics.ObservableMeasurement; +import io.opentelemetry.context.Context; +import io.temporal.workflow.unsafe.WorkflowUnsafe; +import java.util.List; +import java.util.function.Consumer; + +/** + * Wraps a meter so the synchronous instruments it builds drop recordings made by replaying workflow + * code, which would otherwise be recorded again on every replay. + * + *

    Observable instruments cannot be created from workflow code. Their callbacks stay registered + * until the meter provider closes, so every replay would register another callback, and each + * callback would read workflow state from the reader thread without workflow synchronization. + */ +public final class ReplaySafeMeter implements Meter { + private final Meter delegate; + + public ReplaySafeMeter(Meter delegate) { + this.delegate = delegate; + } + + @Override + public LongCounterBuilder counterBuilder(String name) { + return new ReplaySafeLongCounterBuilder(delegate.counterBuilder(name)); + } + + @Override + public LongUpDownCounterBuilder upDownCounterBuilder(String name) { + return new ReplaySafeLongUpDownCounterBuilder(delegate.upDownCounterBuilder(name)); + } + + @Override + public DoubleHistogramBuilder histogramBuilder(String name) { + return new ReplaySafeDoubleHistogramBuilder(delegate.histogramBuilder(name)); + } + + @Override + public DoubleGaugeBuilder gaugeBuilder(String name) { + return new ReplaySafeDoubleGaugeBuilder(delegate.gaugeBuilder(name)); + } + + @Override + public BatchCallback batchCallback( + Runnable callback, + ObservableMeasurement observableMeasurement, + ObservableMeasurement... additionalMeasurements) { + rejectWorkflowThread(); + return delegate.batchCallback(callback, observableMeasurement, additionalMeasurements); + } + + private static void rejectWorkflowThread() { + if (WorkflowUnsafe.isWorkflowThread()) { + throw new IllegalStateException( + "Observable instruments cannot be created from workflow code."); + } + } + + private static final class ReplaySafeLongCounterBuilder implements LongCounterBuilder { + private final LongCounterBuilder delegate; + + ReplaySafeLongCounterBuilder(LongCounterBuilder delegate) { + this.delegate = delegate; + } + + @Override + public LongCounterBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public LongCounterBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public DoubleCounterBuilder ofDoubles() { + return new ReplaySafeDoubleCounterBuilder(delegate.ofDoubles()); + } + + @Override + public LongCounter build() { + return new ReplaySafeLongCounter(delegate.build()); + } + + @Override + public ObservableLongCounter buildWithCallback(Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableLongMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + } + + private static final class ReplaySafeDoubleCounterBuilder implements DoubleCounterBuilder { + private final DoubleCounterBuilder delegate; + + ReplaySafeDoubleCounterBuilder(DoubleCounterBuilder delegate) { + this.delegate = delegate; + } + + @Override + public DoubleCounterBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public DoubleCounterBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public DoubleCounter build() { + return new ReplaySafeDoubleCounter(delegate.build()); + } + + @Override + public ObservableDoubleCounter buildWithCallback( + Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableDoubleMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + } + + private static final class ReplaySafeLongUpDownCounterBuilder + implements LongUpDownCounterBuilder { + private final LongUpDownCounterBuilder delegate; + + ReplaySafeLongUpDownCounterBuilder(LongUpDownCounterBuilder delegate) { + this.delegate = delegate; + } + + @Override + public LongUpDownCounterBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public LongUpDownCounterBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public DoubleUpDownCounterBuilder ofDoubles() { + return new ReplaySafeDoubleUpDownCounterBuilder(delegate.ofDoubles()); + } + + @Override + public LongUpDownCounter build() { + return new ReplaySafeLongUpDownCounter(delegate.build()); + } + + @Override + public ObservableLongUpDownCounter buildWithCallback( + Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableLongMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + } + + private static final class ReplaySafeDoubleUpDownCounterBuilder + implements DoubleUpDownCounterBuilder { + private final DoubleUpDownCounterBuilder delegate; + + ReplaySafeDoubleUpDownCounterBuilder(DoubleUpDownCounterBuilder delegate) { + this.delegate = delegate; + } + + @Override + public DoubleUpDownCounterBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public DoubleUpDownCounterBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public DoubleUpDownCounter build() { + return new ReplaySafeDoubleUpDownCounter(delegate.build()); + } + + @Override + public ObservableDoubleUpDownCounter buildWithCallback( + Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableDoubleMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + } + + private static final class ReplaySafeDoubleHistogramBuilder implements DoubleHistogramBuilder { + private final DoubleHistogramBuilder delegate; + + ReplaySafeDoubleHistogramBuilder(DoubleHistogramBuilder delegate) { + this.delegate = delegate; + } + + @Override + public DoubleHistogramBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public DoubleHistogramBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public DoubleHistogramBuilder setExplicitBucketBoundariesAdvice(List bucketBoundaries) { + delegate.setExplicitBucketBoundariesAdvice(bucketBoundaries); + return this; + } + + @Override + public LongHistogramBuilder ofLongs() { + return new ReplaySafeLongHistogramBuilder(delegate.ofLongs()); + } + + @Override + public DoubleHistogram build() { + return new ReplaySafeDoubleHistogram(delegate.build()); + } + } + + private static final class ReplaySafeLongHistogramBuilder implements LongHistogramBuilder { + private final LongHistogramBuilder delegate; + + ReplaySafeLongHistogramBuilder(LongHistogramBuilder delegate) { + this.delegate = delegate; + } + + @Override + public LongHistogramBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public LongHistogramBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public LongHistogramBuilder setExplicitBucketBoundariesAdvice(List bucketBoundaries) { + delegate.setExplicitBucketBoundariesAdvice(bucketBoundaries); + return this; + } + + @Override + public LongHistogram build() { + return new ReplaySafeLongHistogram(delegate.build()); + } + } + + private static final class ReplaySafeDoubleGaugeBuilder implements DoubleGaugeBuilder { + private final DoubleGaugeBuilder delegate; + + ReplaySafeDoubleGaugeBuilder(DoubleGaugeBuilder delegate) { + this.delegate = delegate; + } + + @Override + public DoubleGaugeBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public DoubleGaugeBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public LongGaugeBuilder ofLongs() { + return new ReplaySafeLongGaugeBuilder(delegate.ofLongs()); + } + + @Override + public ObservableDoubleGauge buildWithCallback(Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableDoubleMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + + @Override + public DoubleGauge build() { + return new ReplaySafeDoubleGauge(delegate.build()); + } + } + + private static final class ReplaySafeLongGaugeBuilder implements LongGaugeBuilder { + private final LongGaugeBuilder delegate; + + ReplaySafeLongGaugeBuilder(LongGaugeBuilder delegate) { + this.delegate = delegate; + } + + @Override + public LongGaugeBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public LongGaugeBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public ObservableLongGauge buildWithCallback(Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableLongMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + + @Override + public LongGauge build() { + return new ReplaySafeLongGauge(delegate.build()); + } + } + + private static final class ReplaySafeLongCounter implements LongCounter { + private final LongCounter delegate; + + ReplaySafeLongCounter(LongCounter delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + } + + @Override + public void add(long value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value); + } + + @Override + public void add(long value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes); + } + + @Override + public void add(long value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes, context); + } + } + + private static final class ReplaySafeDoubleCounter implements DoubleCounter { + private final DoubleCounter delegate; + + ReplaySafeDoubleCounter(DoubleCounter delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + } + + @Override + public void add(double value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value); + } + + @Override + public void add(double value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes); + } + + @Override + public void add(double value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes, context); + } + } + + private static final class ReplaySafeLongUpDownCounter implements LongUpDownCounter { + private final LongUpDownCounter delegate; + + ReplaySafeLongUpDownCounter(LongUpDownCounter delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + } + + @Override + public void add(long value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value); + } + + @Override + public void add(long value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes); + } + + @Override + public void add(long value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes, context); + } + } + + private static final class ReplaySafeDoubleUpDownCounter implements DoubleUpDownCounter { + private final DoubleUpDownCounter delegate; + + ReplaySafeDoubleUpDownCounter(DoubleUpDownCounter delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + } + + @Override + public void add(double value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value); + } + + @Override + public void add(double value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes); + } + + @Override + public void add(double value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes, context); + } + } + + private static final class ReplaySafeDoubleHistogram implements DoubleHistogram { + private final DoubleHistogram delegate; + + ReplaySafeDoubleHistogram(DoubleHistogram delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + } + + @Override + public void record(double value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value); + } + + @Override + public void record(double value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value, attributes); + } + + @Override + public void record(double value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value, attributes, context); + } + } + + private static final class ReplaySafeLongHistogram implements LongHistogram { + private final LongHistogram delegate; + + ReplaySafeLongHistogram(LongHistogram delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + } + + @Override + public void record(long value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value); + } + + @Override + public void record(long value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value, attributes); + } + + @Override + public void record(long value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value, attributes, context); + } + } + + private static final class ReplaySafeDoubleGauge implements DoubleGauge { + private final DoubleGauge delegate; + + ReplaySafeDoubleGauge(DoubleGauge delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + } + + @Override + public void set(double value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value); + } + + @Override + public void set(double value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value, attributes); + } + + @Override + public void set(double value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value, attributes, context); + } + } + + private static final class ReplaySafeLongGauge implements LongGauge { + private final LongGauge delegate; + + ReplaySafeLongGauge(LongGauge delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + } + + @Override + public void set(long value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value); + } + + @Override + public void set(long value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value, attributes); + } + + @Override + public void set(long value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value, attributes, context); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java index 8d0ad4603b..52a26d36c3 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java @@ -6,7 +6,6 @@ import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.StatusCode; import io.temporal.workflow.Workflow; -import io.temporal.workflow.unsafe.WorkflowUnsafe; import java.util.concurrent.TimeUnit; /** Wraps a span so that replayed code does not end it, which would export a duplicate. */ @@ -24,9 +23,7 @@ public void end() { return; } ended = true; - if (WorkflowUnsafe.isWorkflowThread() - && !WorkflowUnsafe.isReadOnly() - && WorkflowUnsafe.isReplaying()) { + if (OpenTelemetrySuppression.shouldSuppress()) { return; } delegate.end(); @@ -38,9 +35,7 @@ public void end(long timestamp, TimeUnit unit) { return; } ended = true; - if (WorkflowUnsafe.isWorkflowThread() - && !WorkflowUnsafe.isReadOnly() - && WorkflowUnsafe.isReplaying()) { + if (OpenTelemetrySuppression.shouldSuppress()) { return; } delegate.end(timestamp, unit); @@ -54,9 +49,7 @@ public Span setAttribute(AttributeKey key, T value) { @Override public Span addEvent(String name, Attributes attributes) { - if (WorkflowUnsafe.isWorkflowThread() - && !WorkflowUnsafe.isReadOnly() - && WorkflowUnsafe.isReplaying()) { + if (OpenTelemetrySuppression.shouldSuppress()) { delegate.addEvent(name, attributes, Workflow.currentTimeMillis(), TimeUnit.MILLISECONDS); } else { delegate.addEvent(name, attributes); diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java index 678fd961ed..9bd2e7fe7a 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java @@ -11,7 +11,6 @@ import io.opentelemetry.context.ContextKey; import io.opentelemetry.context.Scope; import io.temporal.workflow.Workflow; -import io.temporal.workflow.unsafe.WorkflowUnsafe; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; @@ -49,10 +48,7 @@ private static final class NamedStreamSpanBuilder implements SpanBuilder { @Override public Span startSpan() { - if (WorkflowUnsafe.isWorkflowThread() - && !WorkflowUnsafe.isReadOnly() - && WorkflowUnsafe.isReplaying() - && !startTimestampSet) { + if (OpenTelemetrySuppression.shouldSuppress() && !startTimestampSet) { delegate.setStartTimestamp(Workflow.currentTimeMillis(), TimeUnit.MILLISECONDS); } try (Scope ignored = Context.current().with(TRACER_NAME, tracerName).makeCurrent()) { diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java new file mode 100644 index 0000000000..3838260248 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java @@ -0,0 +1,122 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableSet; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.sdk.metrics.data.LongPointData; +import io.temporal.activity.ActivityInterface; +import io.temporal.activity.ActivityMethod; +import io.temporal.activity.ActivityOptions; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowStub; +import io.temporal.testing.WorkflowReplayer; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import io.temporal.workflow.SignalMethod; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInterface; +import io.temporal.workflow.WorkflowMethod; +import io.temporal.workflow.unsafe.WorkflowUnsafe; +import java.time.Duration; +import java.util.Collection; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.junit.Rule; +import org.junit.Test; + +/** + * Verifies that workflow and activity metrics record during live execution and that replaying a + * workflow run does not emit duplicate metrics. + */ +public class MetricsTest extends OtelTestBase { + private static final String METER_NAME = "custom-metrics"; + private static final String WORKFLOW_COUNTER = "custom_workflow_counter"; + private static final String ACTIVITY_COUNTER = "custom_activity_counter"; + private static final Set liveEnabled = ConcurrentHashMap.newKeySet(); + private static final Set replayEnabled = ConcurrentHashMap.newKeySet(); + + @ActivityInterface + public interface TestActivity { + @ActivityMethod + void doActivity(); + } + + public static class TestActivityImpl implements TestActivity { + @Override + public void doActivity() { + GlobalOpenTelemetry.getMeter(METER_NAME).counterBuilder(ACTIVITY_COUNTER).build().add(1); + } + } + + @WorkflowInterface + public interface TestWorkflow { + @WorkflowMethod + void run(); + + @SignalMethod + void proceed(); + } + + public static class TestWorkflowImpl implements TestWorkflow { + private boolean proceed; + + @Override + public void run() { + LongCounter counter = + GlobalOpenTelemetry.getMeter(METER_NAME).counterBuilder(WORKFLOW_COUNTER).build(); + (WorkflowUnsafe.isReplaying() ? replayEnabled : liveEnabled).add(counter.isEnabled()); + counter.add(1); + + TestActivity activity = + Workflow.newActivityStub( + TestActivity.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(5)).build()); + activity.doActivity(); + + Workflow.await(() -> proceed); + } + + @Override + public void proceed() { + proceed = true; + } + } + + @Rule + public SDKTestWorkflowRule testWorkflowRule = + newRuleBuilder(false) + .setWorkflowTypes(TestWorkflowImpl.class) + .setActivityImplementations(new TestActivityImpl()) + .build(); + + @Test + public void liveExecutionRecordsAndReplayDoesNotDuplicate() throws Exception { + liveEnabled.clear(); + replayEnabled.clear(); + TestWorkflow workflow = testWorkflowRule.newWorkflowStub(TestWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run); + + workflow.proceed(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + assertEquals(1, singleLongValue(WORKFLOW_COUNTER)); + assertEquals(1, singleLongValue(ACTIVITY_COUNTER)); + + // Replay the full workflow history and verify workflow metrics are suppressed on replay + WorkflowReplayer.replayWorkflowExecution( + testWorkflowRule.getExecutionHistory(execution.getWorkflowId()), TestWorkflowImpl.class); + + assertEquals(1, singleLongValue(WORKFLOW_COUNTER)); + assertEquals(1, singleLongValue(ACTIVITY_COUNTER)); + assertEquals(ImmutableSet.of(true), liveEnabled); + assertEquals(ImmutableSet.of(false), replayEnabled); + } + + private static long singleLongValue(String name) { + Collection points = requireMetricNamed(name).getLongSumData().getPoints(); + assertEquals(name + " points: " + points, 1, points.size()); + return points.iterator().next().getValue(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java index b2b924b604..8c826ff0b6 100644 --- a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java @@ -6,6 +6,9 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.data.SpanData; @@ -22,6 +25,7 @@ public abstract class OtelTestBase { static final InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + static final InMemoryMetricReader metricReader = InMemoryMetricReader.create(); private static ReplaySafeOpenTelemetry openTelemetry; @BeforeClass @@ -31,6 +35,7 @@ public static void registerGlobalOpenTelemetry() { .setTracerProviderBuilder( SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(spanExporter))) + .setMeterProviderBuilder(SdkMeterProvider.builder().registerMetricReader(metricReader)) .build(); GlobalOpenTelemetry.set(openTelemetry); } @@ -72,6 +77,16 @@ static SpanData requireSpanNamed(List spans, String name) { return null; } + static MetricData requireMetricNamed(String name) { + for (MetricData metric : metricReader.collectAllMetrics()) { + if (metric.getName().equals(name)) { + return metric; + } + } + fail(name + " metric not found"); + return null; + } + static String requireSpanAttribute(SpanData span, AttributeKey key) { String value = span.getAttributes().get(key); assertNotNull(key.getKey() + " attribute not found on " + span.getName(), value); From ecbce8bf31620bce9cd368ff7e0fe87d72693864 Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Wed, 14 Oct 2026 10:16:39 -0400 Subject: [PATCH 3/6] Make the OpenTelemetry v2 logger provider replay safe --- CHANGELOG.md | 4 +- contrib/temporal-opentelemetry-v2/README.md | 164 +++++++++++++++++ .../temporal-opentelemetry-v2/build.gradle | 5 +- .../v2/ReplaySafeOpenTelemetry.java | 65 ++++++- .../v2/internal/ReplaySafeLogger.java | 170 ++++++++++++++++++ .../temporal/opentelemetry/v2/LoggerTest.java | 116 ++++++++++++ .../opentelemetry/v2/OtelTestBase.java | 17 +- .../src/test/resources/logback-test.xml | 17 ++ 8 files changed, 545 insertions(+), 13 deletions(-) create mode 100644 contrib/temporal-opentelemetry-v2/README.md create mode 100644 contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/LoggerTest.java create mode 100644 contrib/temporal-opentelemetry-v2/src/test/resources/logback-test.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index c3b3e276c7..25c9dc11f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,8 +58,8 @@ to docs, or any other relevant information. `addScheduleClientInterceptors`, `addActivityClientInterceptors`, and `addNexusClientInterceptors`. - Added the experimental `temporal-opentelemetry-v2` module. `OpenTelemetryPlugin` propagates OpenTelemetry context through Temporal headers and can optionally create spans for Temporal operations. `ReplaySafeOpenTelemetry` makes - workflow spans replay safe with deterministic span and trace IDs, and drops metric recordings made by replaying - workflow code. + workflow spans replay safe with deterministic span and trace IDs, and drops metric recordings and log records made + by replaying workflow code. ### Changed - Moved the Spring AI integration to [temporalio/ai-integrations](https://github.com/temporalio/ai-integrations/tree/main/java/spring-ai). diff --git a/contrib/temporal-opentelemetry-v2/README.md b/contrib/temporal-opentelemetry-v2/README.md new file mode 100644 index 0000000000..76acde1c11 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/README.md @@ -0,0 +1,164 @@ +# OpenTelemetry v2 integration for the Temporal Java SDK + +Module `io.temporal:temporal-opentelemetry-v2` provides replay-safe +OpenTelemetry tracing, metrics, and logs for Temporal. + +> [!WARNING] +> This module is experimental, and its API may change. + +## Setup + +Use the same version as the rest of your Temporal Java SDK dependencies: + +```groovy +implementation 'io.temporal:temporal-opentelemetry-v2:' +// Add the exporters you use, such as this OTLP exporter. +implementation 'io.opentelemetry:opentelemetry-exporter-otlp' +``` + +Create a replay-safe OpenTelemetry instance, register it as the global, and +attach the plugin to your service stubs: + +```java +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.temporal.client.WorkflowClient; +import io.temporal.opentelemetry.v2.OpenTelemetryPlugin; +import io.temporal.opentelemetry.v2.ReplaySafeOpenTelemetry; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; +import io.temporal.worker.WorkerFactory; + +ReplaySafeOpenTelemetry openTelemetry = + ReplaySafeOpenTelemetry.newBuilder() + .setTracerProviderBuilder( + SdkTracerProvider.builder() + .addSpanProcessor( + BatchSpanProcessor.builder(OtlpGrpcSpanExporter.builder().build()).build())) + .setMeterProviderBuilder( + SdkMeterProvider.builder() + .registerMetricReader( + PeriodicMetricReader.builder(OtlpGrpcMetricExporter.builder().build()) + .build())) + .setLoggerProviderBuilder( + SdkLoggerProvider.builder() + .addLogRecordProcessor( + BatchLogRecordProcessor.builder(OtlpGrpcLogRecordExporter.builder().build()) + .build())) + .build(); +GlobalOpenTelemetry.set(openTelemetry); + +WorkflowServiceStubs service = + WorkflowServiceStubs.newServiceStubs( + WorkflowServiceStubsOptions.newBuilder() + .setPlugins(OpenTelemetryPlugin.newBuilder().build()) + .build()); + +WorkflowClient client = WorkflowClient.newInstance(service); +WorkerFactory factory = WorkerFactory.newInstance(client); +``` + +Plugins configured on `WorkflowServiceStubsOptions` propagate to clients and +workers created from those stubs. It can also be configured directly on +`WorkflowClientOptions`. + +Build `ReplaySafeOpenTelemetry` before anything else uses the OpenTelemetry +`Context`, and register it as the global before building the plugin. Otherwise, +`OpenTelemetryPlugin.Builder.build()` throws `IllegalStateException`. + +Call `ReplaySafeOpenTelemetry.close()` after your workers and clients have +stopped to export buffered telemetry and shut down its providers. + +## Tracing + +The plugin propagates application trace context through Temporal headers. +Application spans remain connected across clients, workflows, activities, and +Nexus operations. + +Set `OpenTelemetryPlugin.Builder.setAddTemporalSpans(true)` to emit spans for +operations such as `StartWorkflow`, `RunWorkflow`, `RunActivity`, and +`ContinueAsNew`. + +Create spans in workflow, client, and activity code with the standard OpenTelemetry API: + +```java +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Scope; + +Span span = + GlobalOpenTelemetry.getTracer("my-workflows") + .spanBuilder("my-span") + .startSpan(); +try (Scope ignored = span.makeCurrent()) { + activity.doWork(); +} finally { + span.end(); +} +``` + +## Metrics + +Create metrics in workflow, client, and activity code with the standard +OpenTelemetry API. Workflow code supports only synchronous instruments: + +```java +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.metrics.LongCounter; + +LongCounter counter = + GlobalOpenTelemetry.getMeter("my-workflows") + .counterBuilder("workflow.items.processed") + .build(); +counter.add(1); +``` + +## Logs + +Create logs in workflow, client, and activity code with SLF4J, and send them to +OpenTelemetry by installing a logging appender, such as the +[OpenTelemetry Logback appender](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/logback/logback-appender-1.0/library), +with the replay-safe instance. The appender drops records from replaying +workflow code only when it is installed with `ReplaySafeOpenTelemetry`, not a +plain `OpenTelemetrySdk`: + +```java +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; +import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor; +import io.temporal.opentelemetry.v2.ReplaySafeOpenTelemetry; + +ReplaySafeOpenTelemetry openTelemetry = + ReplaySafeOpenTelemetry.newBuilder() + .setLoggerProviderBuilder( + SdkLoggerProvider.builder() + .addLogRecordProcessor( + BatchLogRecordProcessor.builder(OtlpGrpcLogRecordExporter.builder().build()) + .build())) + .build(); +OpenTelemetryAppender.install(openTelemetry); +``` + +Then log with SLF4J: + +```java +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +Logger logger = LoggerFactory.getLogger(MyWorkflowImpl.class); +logger.info("workflow step completed"); +``` + +Configure the appender as its setup guide describes. Replay is checked on the +thread that emits the record, so behind an asynchronous appender, such as +Logback's `AsyncAppender`, log from workflow code with `Workflow.getLogger()`, +which skips logging during replay. diff --git a/contrib/temporal-opentelemetry-v2/build.gradle b/contrib/temporal-opentelemetry-v2/build.gradle index 3fb51d1651..bc4000f89c 100644 --- a/contrib/temporal-opentelemetry-v2/build.gradle +++ b/contrib/temporal-opentelemetry-v2/build.gradle @@ -2,6 +2,7 @@ description = '''Temporal Java SDK OpenTelemetry v2 Module''' ext { otelVersion = '1.66.0' + otelInstrumentationVersion = '2.32.0-alpha' } dependencies { @@ -22,7 +23,7 @@ dependencies { testImplementation project(':temporal-serviceclient') testImplementation project(':temporal-testing') testImplementation "io.opentelemetry:opentelemetry-sdk-testing" + testImplementation "io.opentelemetry.instrumentation:opentelemetry-logback-appender-1.0:$otelInstrumentationVersion" testImplementation "junit:junit:${junitVersion}" - - testRuntimeOnly group: 'ch.qos.logback', name: 'logback-classic', version: "${logbackVersion}" + testImplementation group: 'ch.qos.logback', name: 'logback-classic', version: "${logbackVersion}" } diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java index 1bccd9be0b..953feb9896 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java @@ -3,6 +3,8 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; +import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.api.logs.LoggerBuilder; import io.opentelemetry.api.logs.LoggerProvider; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.MeterBuilder; @@ -23,6 +25,7 @@ import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; import io.temporal.common.Experimental; import io.temporal.opentelemetry.v2.internal.ReplaySafeIdGenerator; +import io.temporal.opentelemetry.v2.internal.ReplaySafeLogger; import io.temporal.opentelemetry.v2.internal.ReplaySafeMeter; import io.temporal.opentelemetry.v2.internal.ReplaySafeTracer; import io.temporal.opentelemetry.v2.internal.TemporalContextStorage; @@ -31,8 +34,8 @@ /** * The {@link OpenTelemetry} to use for OpenTelemetry integration with Temporal. Register it with - * {@code GlobalOpenTelemetry.set}; tracers and meters obtained from it are replay safe inside - * workflows. + * {@code GlobalOpenTelemetry.set}; tracers, meters, and loggers obtained from it are replay safe + * inside workflows. * *

    Known limitations: * @@ -51,17 +54,15 @@ * Promise.allOf} and {@code Promise.anyOf}, run with the context current when the promise * completes. *

  • Continuations on the promise returned by cancelling an external workflow run outside a - * workflow thread, so spans they start get random IDs, and their spans and metric recordings - * are exported again on every replay. + * workflow thread, so spans they start get random IDs, and their spans, metric recordings, + * and log records are exported again on every replay. * */ @Experimental public final class ReplaySafeOpenTelemetry implements OpenTelemetry, Closeable { private final ReplaySafeTracerProvider tracerProvider; private final ReplaySafeMeterProvider meterProvider; - // TODO: Make the logger provider replay safe and add logger interceptor methods for Temporal and - // OpenTelemetry loggers. - private final SdkLoggerProvider loggerProvider; + private final ReplaySafeLoggerProvider loggerProvider; private final ContextPropagators propagators; static { @@ -77,7 +78,7 @@ private ReplaySafeOpenTelemetry(Builder builder) { .setIdGenerator(new ReplaySafeIdGenerator(builder.idGenerator)) .build()); this.meterProvider = new ReplaySafeMeterProvider(builder.meterProviderBuilder.build()); - this.loggerProvider = builder.loggerProviderBuilder.build(); + this.loggerProvider = new ReplaySafeLoggerProvider(builder.loggerProviderBuilder.build()); this.propagators = builder.propagators; } @@ -253,6 +254,54 @@ public Meter build() { } } + private static final class ReplaySafeLoggerProvider implements LoggerProvider, Closeable { + private final SdkLoggerProvider delegate; + + private ReplaySafeLoggerProvider(SdkLoggerProvider delegate) { + this.delegate = delegate; + } + + @Override + public Logger get(@Nonnull String instrumentationScopeName) { + return new ReplaySafeLogger(delegate.get(instrumentationScopeName)); + } + + @Override + public LoggerBuilder loggerBuilder(@Nonnull String instrumentationScopeName) { + return new ReplaySafeLoggerBuilder(delegate.loggerBuilder(instrumentationScopeName)); + } + + @Override + public void close() { + delegate.close(); + } + } + + private static final class ReplaySafeLoggerBuilder implements LoggerBuilder { + private final LoggerBuilder delegate; + + private ReplaySafeLoggerBuilder(LoggerBuilder delegate) { + this.delegate = delegate; + } + + @Override + public LoggerBuilder setSchemaUrl(@Nonnull String schemaUrl) { + delegate.setSchemaUrl(schemaUrl); + return this; + } + + @Override + public LoggerBuilder setInstrumentationVersion(@Nonnull String instrumentationScopeVersion) { + delegate.setInstrumentationVersion(instrumentationScopeVersion); + return this; + } + + @Override + public Logger build() { + return new ReplaySafeLogger(delegate.build()); + } + } + private static final class ReplaySafeTracerBuilder implements TracerBuilder { private final TracerBuilder delegate; private final String instrumentationScopeName; diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java new file mode 100644 index 0000000000..68c8297b60 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java @@ -0,0 +1,170 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.Value; +import io.opentelemetry.api.logs.LogRecordBuilder; +import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.api.logs.Severity; +import io.opentelemetry.context.Context; +import java.time.Instant; +import java.util.concurrent.TimeUnit; + +/** + * Wraps a logger so the records it builds are not emitted by replaying workflow code, which would + * otherwise emit them again on every replay. Replay is checked on the thread that calls {@link + * LogRecordBuilder#emit()}, so records emitted on another thread, such as an asynchronous + * appender's worker, are not suppressed. + */ +public final class ReplaySafeLogger implements Logger { + private final Logger delegate; + + public ReplaySafeLogger(Logger delegate) { + this.delegate = delegate; + } + + @Override + public LogRecordBuilder logRecordBuilder() { + return new ReplaySafeLogRecordBuilder(delegate.logRecordBuilder()); + } + + @Override + public boolean isEnabled(Severity severity, Context context) { + return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(severity, context); + } + + @Override + public boolean isEnabled(Severity severity) { + return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(severity); + } + + private static final class ReplaySafeLogRecordBuilder implements LogRecordBuilder { + private final LogRecordBuilder delegate; + + ReplaySafeLogRecordBuilder(LogRecordBuilder delegate) { + this.delegate = delegate; + } + + @Override + public void emit() { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.emit(); + } + + @Override + public LogRecordBuilder setTimestamp(long timestamp, TimeUnit unit) { + delegate.setTimestamp(timestamp, unit); + return this; + } + + @Override + public LogRecordBuilder setTimestamp(Instant instant) { + delegate.setTimestamp(instant); + return this; + } + + @Override + public LogRecordBuilder setObservedTimestamp(long timestamp, TimeUnit unit) { + delegate.setObservedTimestamp(timestamp, unit); + return this; + } + + @Override + public LogRecordBuilder setObservedTimestamp(Instant instant) { + delegate.setObservedTimestamp(instant); + return this; + } + + @Override + public LogRecordBuilder setContext(Context context) { + delegate.setContext(context); + return this; + } + + @Override + public LogRecordBuilder setSeverity(Severity severity) { + delegate.setSeverity(severity); + return this; + } + + @Override + public LogRecordBuilder setSeverityText(String severityText) { + delegate.setSeverityText(severityText); + return this; + } + + @Override + public LogRecordBuilder setBody(String body) { + delegate.setBody(body); + return this; + } + + @Override + public LogRecordBuilder setBody(Value body) { + delegate.setBody(body); + return this; + } + + @Override + public LogRecordBuilder setAllAttributes(Attributes attributes) { + delegate.setAllAttributes(attributes); + return this; + } + + @Override + public LogRecordBuilder setAttribute(AttributeKey key, T value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, String value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, long value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, double value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, boolean value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, int value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, Value value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setEventName(String eventName) { + delegate.setEventName(eventName); + return this; + } + + @Override + public LogRecordBuilder setException(Throwable throwable) { + delegate.setException(throwable); + return this; + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/LoggerTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/LoggerTest.java new file mode 100644 index 0000000000..24e6992613 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/LoggerTest.java @@ -0,0 +1,116 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import io.opentelemetry.api.logs.Severity; +import io.opentelemetry.sdk.logs.data.LogRecordData; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowStub; +import io.temporal.testing.WorkflowReplayer; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import io.temporal.workflow.QueryMethod; +import io.temporal.workflow.SignalMethod; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInterface; +import io.temporal.workflow.WorkflowMethod; +import java.util.ArrayList; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Verifies that log records emit during live execution and that replaying a workflow run does not + * emit duplicate records. + */ +public class LoggerTest extends OtelTestBase { + private static final Logger logger = LoggerFactory.getLogger(LoggerTest.class); + private static final String RUN_BODY = "custom workflow run"; + private static final String QUERY_BODY = "custom workflow query"; + + @WorkflowInterface + public interface TestWorkflow { + @WorkflowMethod + void run(); + + @QueryMethod + String queryAndEmit(); + + @SignalMethod + void proceed(); + } + + public static class TestWorkflowImpl implements TestWorkflow { + private boolean proceed; + + @Override + public void run() { + emit(RUN_BODY); + Workflow.await(() -> proceed); + } + + @Override + public String queryAndEmit() { + emit(QUERY_BODY); + return "ok"; + } + + @Override + public void proceed() { + proceed = true; + } + + private static void emit(String body) { + logger.info(body); + } + } + + @Rule + public SDKTestWorkflowRule testWorkflowRule = + newRuleBuilder(false).setWorkflowTypes(TestWorkflowImpl.class).build(); + + @Test + public void recordsEmitOutsideWorkflows() { + logger.warn("outside"); + + List logs = emittedLogs(); + assertEquals(1, logs.size()); + LogRecordData log = logs.get(0); + assertEquals(Severity.WARN, log.getSeverity()); + assertEquals("outside", log.getBodyValue().asString()); + } + + /** + * The live run and the live query handler each emit once; replaying the finished run drops its + * record. + */ + @Test + public void liveExecutionEmitsAndReplayDoesNotDuplicate() throws Exception { + TestWorkflow workflow = testWorkflowRule.newWorkflowStub(TestWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run); + + assertEquals("ok", workflow.queryAndEmit()); + workflow.proceed(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + // Replay the full workflow history and verify the run's record is suppressed on replay. + WorkflowReplayer.replayWorkflowExecution( + testWorkflowRule.getExecutionHistory(execution.getWorkflowId()), TestWorkflowImpl.class); + + List bodies = emittedBodies(); + assertEquals(bodies.toString(), 2, bodies.size()); + assertTrue(bodies.toString(), bodies.contains(RUN_BODY)); + assertTrue(bodies.toString(), bodies.contains(QUERY_BODY)); + } + + private static List emittedBodies() { + List bodies = new ArrayList<>(); + for (LogRecordData log : emittedLogs()) { + bodies.add(log.getBodyValue().asString()); + } + return bodies; + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java index 8c826ff0b6..48e10720bf 100644 --- a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java @@ -6,8 +6,13 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.data.LogRecordData; +import io.opentelemetry.sdk.logs.export.SimpleLogRecordProcessor; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryLogRecordExporter; import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; import io.opentelemetry.sdk.trace.SdkTracerProvider; @@ -26,6 +31,7 @@ public abstract class OtelTestBase { static final InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); static final InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + static final InMemoryLogRecordExporter logExporter = InMemoryLogRecordExporter.create(); private static ReplaySafeOpenTelemetry openTelemetry; @BeforeClass @@ -36,8 +42,12 @@ public static void registerGlobalOpenTelemetry() { SdkTracerProvider.builder() .addSpanProcessor(SimpleSpanProcessor.create(spanExporter))) .setMeterProviderBuilder(SdkMeterProvider.builder().registerMetricReader(metricReader)) + .setLoggerProviderBuilder( + SdkLoggerProvider.builder() + .addLogRecordProcessor(SimpleLogRecordProcessor.create(logExporter))) .build(); GlobalOpenTelemetry.set(openTelemetry); + OpenTelemetryAppender.install(openTelemetry); } @AfterClass @@ -47,8 +57,9 @@ public static void resetGlobalOpenTelemetry() { } @Before - public void clearSpans() { + public void clearSpansAndLogs() { spanExporter.reset(); + logExporter.reset(); } /** @@ -67,6 +78,10 @@ static List endedSpans() { return spanExporter.getFinishedSpanItems(); } + static List emittedLogs() { + return logExporter.getFinishedLogRecordItems(); + } + static SpanData requireSpanNamed(List spans, String name) { for (SpanData span : spans) { if (span.getName().equals(name)) { diff --git a/contrib/temporal-opentelemetry-v2/src/test/resources/logback-test.xml b/contrib/temporal-opentelemetry-v2/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..4dc91ceb52 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/resources/logback-test.xml @@ -0,0 +1,17 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + From ac5985cc357138afa0df3286bd534af304f164a7 Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Thu, 22 Oct 2026 13:19:34 -0400 Subject: [PATCH 4/6] Export read-only spans and end spans only outside replay Spans started in read-only code, such as queries, update validators, and await conditions, are no longer wrapped, so they always export. Wrapped spans skip ending while replaying, which also keeps spans open across a cache eviction. This matches the Go OpenTelemetry v2 plugin. --- .../temporal/opentelemetry/v2/internal/ReplaySafeSpan.java | 5 +++-- .../temporal/opentelemetry/v2/internal/ReplaySafeTracer.java | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java index 52a26d36c3..212c092c2c 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java @@ -6,6 +6,7 @@ import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.StatusCode; import io.temporal.workflow.Workflow; +import io.temporal.workflow.unsafe.WorkflowUnsafe; import java.util.concurrent.TimeUnit; /** Wraps a span so that replayed code does not end it, which would export a duplicate. */ @@ -23,7 +24,7 @@ public void end() { return; } ended = true; - if (OpenTelemetrySuppression.shouldSuppress()) { + if (WorkflowUnsafe.isReplaying()) { return; } delegate.end(); @@ -35,7 +36,7 @@ public void end(long timestamp, TimeUnit unit) { return; } ended = true; - if (OpenTelemetrySuppression.shouldSuppress()) { + if (WorkflowUnsafe.isReplaying()) { return; } delegate.end(timestamp, unit); diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java index 9bd2e7fe7a..7ac97a12b4 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java @@ -11,6 +11,7 @@ import io.opentelemetry.context.ContextKey; import io.opentelemetry.context.Scope; import io.temporal.workflow.Workflow; +import io.temporal.workflow.unsafe.WorkflowUnsafe; import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; @@ -52,7 +53,9 @@ public Span startSpan() { delegate.setStartTimestamp(Workflow.currentTimeMillis(), TimeUnit.MILLISECONDS); } try (Scope ignored = Context.current().with(TRACER_NAME, tracerName).makeCurrent()) { - return new ReplaySafeSpan(delegate.startSpan()); + Span span = delegate.startSpan(); + // Read-only code does not run a deterministic number of times, so its spans always export. + return WorkflowUnsafe.isReadOnly() ? span : new ReplaySafeSpan(span); } } From a9a03b35c77978f8dc91c05bd7b7023cc5eb1171 Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Thu, 22 Oct 2026 14:04:23 -0400 Subject: [PATCH 5/6] Keep isEnabled the same during replay Metric instruments and loggers now delegate isEnabled unchanged and suppress only recording and emitting during replay. Workflow code that branches on isEnabled then takes the same path live and on replay. --- .../v2/internal/ReplaySafeLogger.java | 4 ++-- .../v2/internal/ReplaySafeMeter.java | 16 ++++++++-------- .../temporal/opentelemetry/v2/MetricsTest.java | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java index 68c8297b60..864f83d5e0 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java @@ -30,12 +30,12 @@ public LogRecordBuilder logRecordBuilder() { @Override public boolean isEnabled(Severity severity, Context context) { - return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(severity, context); + return delegate.isEnabled(severity, context); } @Override public boolean isEnabled(Severity severity) { - return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(severity); + return delegate.isEnabled(severity); } private static final class ReplaySafeLogRecordBuilder implements LogRecordBuilder { diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java index 9d63c94801..921058b5ad 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java @@ -402,7 +402,7 @@ private static final class ReplaySafeLongCounter implements LongCounter { @Override public boolean isEnabled() { - return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + return delegate.isEnabled(); } @Override @@ -439,7 +439,7 @@ private static final class ReplaySafeDoubleCounter implements DoubleCounter { @Override public boolean isEnabled() { - return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + return delegate.isEnabled(); } @Override @@ -476,7 +476,7 @@ private static final class ReplaySafeLongUpDownCounter implements LongUpDownCoun @Override public boolean isEnabled() { - return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + return delegate.isEnabled(); } @Override @@ -513,7 +513,7 @@ private static final class ReplaySafeDoubleUpDownCounter implements DoubleUpDown @Override public boolean isEnabled() { - return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + return delegate.isEnabled(); } @Override @@ -550,7 +550,7 @@ private static final class ReplaySafeDoubleHistogram implements DoubleHistogram @Override public boolean isEnabled() { - return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + return delegate.isEnabled(); } @Override @@ -587,7 +587,7 @@ private static final class ReplaySafeLongHistogram implements LongHistogram { @Override public boolean isEnabled() { - return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + return delegate.isEnabled(); } @Override @@ -624,7 +624,7 @@ private static final class ReplaySafeDoubleGauge implements DoubleGauge { @Override public boolean isEnabled() { - return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + return delegate.isEnabled(); } @Override @@ -661,7 +661,7 @@ private static final class ReplaySafeLongGauge implements LongGauge { @Override public boolean isEnabled() { - return !OpenTelemetrySuppression.shouldSuppress() && delegate.isEnabled(); + return delegate.isEnabled(); } @Override diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java index 3838260248..0b5e6cdcff 100644 --- a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java @@ -111,7 +111,7 @@ public void liveExecutionRecordsAndReplayDoesNotDuplicate() throws Exception { assertEquals(1, singleLongValue(WORKFLOW_COUNTER)); assertEquals(1, singleLongValue(ACTIVITY_COUNTER)); assertEquals(ImmutableSet.of(true), liveEnabled); - assertEquals(ImmutableSet.of(false), replayEnabled); + assertEquals(ImmutableSet.of(true), replayEnabled); } private static long singleLongValue(String name) { From 05cf3dfb0e8bd5f9a58f526f24bd58615f11e39b Mon Sep 17 00:00:00 2001 From: Patrik Beqo Date: Thu, 22 Oct 2026 14:43:32 -0400 Subject: [PATCH 6/6] Remove stale trace context from schedule action headers --- .../v2/internal/InterceptorTracer.java | 4 +- ...lemetryScheduleClientCallsInterceptor.java | 5 +- .../opentelemetry/v2/internal/SpanCodec.java | 6 +++ .../v2/internal/ScheduleTraceHeaderTest.java | 53 +++++++++++++++++++ 4 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/ScheduleTraceHeaderTest.java diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java index dad1a81d95..884859de09 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java @@ -84,8 +84,8 @@ R traceOutbound( return traceOutbound(operation, name, attributes, () -> codec.write(header), call); } - void injectOutboundHeader(Header header) { - codec.write(header); + void replaceOutboundHeader(Header header) { + codec.replace(header); } R traceOutbound( diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java index 8d497b17bd..e11b13f18d 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java @@ -65,12 +65,13 @@ private ScheduleUpdate applyUpdate(UpdateScheduleInput input, ScheduleUpdateInpu /** * Writes the current trace into a copy of the action header, so the caller's header, which may be - * immutable or reused, is never modified. + * immutable or reused, is never modified. A trace already in the header is removed, because it + * can be left over from an earlier create or update. */ private Schedule withTraceHeader(Schedule schedule) { ScheduleActionStartWorkflow action = (ScheduleActionStartWorkflow) schedule.getAction(); Header header = new Header(new HashMap<>(action.getHeader().getValues())); - tracer.injectOutboundHeader(header); + tracer.replaceOutboundHeader(header); return Schedule.newBuilder(schedule) .setAction(ScheduleActionStartWorkflow.newBuilder(action).setHeader(header).build()) .build(); diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java index 750507f08a..08cb8b82ce 100644 --- a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java @@ -97,6 +97,12 @@ void write(Header header) { write(Context.current(), header.getValues()); } + /** Replaces the span and baggage in {@code header} with the current ones, if any. */ + void replace(Header header) { + header.getValues().remove(headerKey); + write(header); + } + Map serialize(Context context) { Map header = new HashMap<>(); write(context, header); diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/ScheduleTraceHeaderTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/ScheduleTraceHeaderTest.java new file mode 100644 index 0000000000..a93cd48f9a --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/ScheduleTraceHeaderTest.java @@ -0,0 +1,53 @@ +package io.temporal.opentelemetry.v2.internal; + +import static org.junit.Assert.assertFalse; + +import io.temporal.api.common.v1.Payload; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.schedules.Schedule; +import io.temporal.client.schedules.ScheduleActionStartWorkflow; +import io.temporal.client.schedules.ScheduleOptions; +import io.temporal.client.schedules.ScheduleSpec; +import io.temporal.common.interceptors.Header; +import io.temporal.common.interceptors.ScheduleClientCallsInterceptor.CreateScheduleInput; +import io.temporal.common.interceptors.ScheduleClientCallsInterceptorBase; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; + +public class ScheduleTraceHeaderTest { + private static final String HEADER_KEY = "_tracer-data"; + + @Test + public void staleTraceIsRemovedWhenNoSpanIsCurrent() { + AtomicReference sent = new AtomicReference<>(); + OpenTelemetryScheduleClientCallsInterceptor interceptor = + new OpenTelemetryScheduleClientCallsInterceptor( + new InterceptorTracer(HEADER_KEY, false), + new ScheduleClientCallsInterceptorBase(null) { + @Override + public void createSchedule(CreateScheduleInput input) { + sent.set(input.getSchedule()); + } + }); + ScheduleActionStartWorkflow action = + ScheduleActionStartWorkflow.newBuilder() + .setWorkflowType("Workflow") + .setOptions(WorkflowOptions.newBuilder().setTaskQueue("queue").build()) + .setHeader( + new Header(Collections.singletonMap(HEADER_KEY, Payload.getDefaultInstance()))) + .build(); + + interceptor.createSchedule( + new CreateScheduleInput( + "schedule", + Schedule.newBuilder() + .setAction(action) + .setSpec(ScheduleSpec.newBuilder().build()) + .build(), + ScheduleOptions.newBuilder().build())); + + ScheduleActionStartWorkflow sentAction = (ScheduleActionStartWorkflow) sent.get().getAction(); + assertFalse(sentAction.getHeader().getValues().containsKey(HEADER_KEY)); + } +}