diff --git a/CHANGELOG.md b/CHANGELOG.md index 344079ce48..25c9dc11f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,10 @@ to docs, or any other relevant information. - Added experimental `ActivityClientPlugin` and `NexusClientPlugin`, which plugins on the service stubs use to configure `ActivityClient` and `NexusClient`. `SimplePlugin` implements both, and its builder gained `addScheduleClientInterceptors`, `addActivityClientInterceptors`, and `addNexusClientInterceptors`. +- Added the experimental `temporal-opentelemetry-v2` module. `OpenTelemetryPlugin` propagates OpenTelemetry context + through Temporal headers and can optionally create spans for Temporal operations. `ReplaySafeOpenTelemetry` makes + workflow spans replay safe with deterministic span and trace IDs, and drops metric recordings and log records made + by replaying workflow code. ### Changed - Moved the Spring AI integration to [temporalio/ai-integrations](https://github.com/temporalio/ai-integrations/tree/main/java/spring-ai). diff --git a/contrib/temporal-opentelemetry-v2/README.md b/contrib/temporal-opentelemetry-v2/README.md new file mode 100644 index 0000000000..76acde1c11 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/README.md @@ -0,0 +1,164 @@ +# OpenTelemetry v2 integration for the Temporal Java SDK + +Module `io.temporal:temporal-opentelemetry-v2` provides replay-safe +OpenTelemetry tracing, metrics, and logs for Temporal. + +> [!WARNING] +> This module is experimental, and its API may change. + +## Setup + +Use the same version as the rest of your Temporal Java SDK dependencies: + +```groovy +implementation 'io.temporal:temporal-opentelemetry-v2:' +// Add the exporters you use, such as this OTLP exporter. +implementation 'io.opentelemetry:opentelemetry-exporter-otlp' +``` + +Create a replay-safe OpenTelemetry instance, register it as the global, and +attach the plugin to your service stubs: + +```java +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; +import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; +import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.temporal.client.WorkflowClient; +import io.temporal.opentelemetry.v2.OpenTelemetryPlugin; +import io.temporal.opentelemetry.v2.ReplaySafeOpenTelemetry; +import io.temporal.serviceclient.WorkflowServiceStubs; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; +import io.temporal.worker.WorkerFactory; + +ReplaySafeOpenTelemetry openTelemetry = + ReplaySafeOpenTelemetry.newBuilder() + .setTracerProviderBuilder( + SdkTracerProvider.builder() + .addSpanProcessor( + BatchSpanProcessor.builder(OtlpGrpcSpanExporter.builder().build()).build())) + .setMeterProviderBuilder( + SdkMeterProvider.builder() + .registerMetricReader( + PeriodicMetricReader.builder(OtlpGrpcMetricExporter.builder().build()) + .build())) + .setLoggerProviderBuilder( + SdkLoggerProvider.builder() + .addLogRecordProcessor( + BatchLogRecordProcessor.builder(OtlpGrpcLogRecordExporter.builder().build()) + .build())) + .build(); +GlobalOpenTelemetry.set(openTelemetry); + +WorkflowServiceStubs service = + WorkflowServiceStubs.newServiceStubs( + WorkflowServiceStubsOptions.newBuilder() + .setPlugins(OpenTelemetryPlugin.newBuilder().build()) + .build()); + +WorkflowClient client = WorkflowClient.newInstance(service); +WorkerFactory factory = WorkerFactory.newInstance(client); +``` + +Plugins configured on `WorkflowServiceStubsOptions` propagate to clients and +workers created from those stubs. It can also be configured directly on +`WorkflowClientOptions`. + +Build `ReplaySafeOpenTelemetry` before anything else uses the OpenTelemetry +`Context`, and register it as the global before building the plugin. Otherwise, +`OpenTelemetryPlugin.Builder.build()` throws `IllegalStateException`. + +Call `ReplaySafeOpenTelemetry.close()` after your workers and clients have +stopped to export buffered telemetry and shut down its providers. + +## Tracing + +The plugin propagates application trace context through Temporal headers. +Application spans remain connected across clients, workflows, activities, and +Nexus operations. + +Set `OpenTelemetryPlugin.Builder.setAddTemporalSpans(true)` to emit spans for +operations such as `StartWorkflow`, `RunWorkflow`, `RunActivity`, and +`ContinueAsNew`. + +Create spans in workflow, client, and activity code with the standard OpenTelemetry API: + +```java +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Scope; + +Span span = + GlobalOpenTelemetry.getTracer("my-workflows") + .spanBuilder("my-span") + .startSpan(); +try (Scope ignored = span.makeCurrent()) { + activity.doWork(); +} finally { + span.end(); +} +``` + +## Metrics + +Create metrics in workflow, client, and activity code with the standard +OpenTelemetry API. Workflow code supports only synchronous instruments: + +```java +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.metrics.LongCounter; + +LongCounter counter = + GlobalOpenTelemetry.getMeter("my-workflows") + .counterBuilder("workflow.items.processed") + .build(); +counter.add(1); +``` + +## Logs + +Create logs in workflow, client, and activity code with SLF4J, and send them to +OpenTelemetry by installing a logging appender, such as the +[OpenTelemetry Logback appender](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/logback/logback-appender-1.0/library), +with the replay-safe instance. The appender drops records from replaying +workflow code only when it is installed with `ReplaySafeOpenTelemetry`, not a +plain `OpenTelemetrySdk`: + +```java +import io.opentelemetry.exporter.otlp.logs.OtlpGrpcLogRecordExporter; +import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.export.BatchLogRecordProcessor; +import io.temporal.opentelemetry.v2.ReplaySafeOpenTelemetry; + +ReplaySafeOpenTelemetry openTelemetry = + ReplaySafeOpenTelemetry.newBuilder() + .setLoggerProviderBuilder( + SdkLoggerProvider.builder() + .addLogRecordProcessor( + BatchLogRecordProcessor.builder(OtlpGrpcLogRecordExporter.builder().build()) + .build())) + .build(); +OpenTelemetryAppender.install(openTelemetry); +``` + +Then log with SLF4J: + +```java +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +Logger logger = LoggerFactory.getLogger(MyWorkflowImpl.class); +logger.info("workflow step completed"); +``` + +Configure the appender as its setup guide describes. Replay is checked on the +thread that emits the record, so behind an asynchronous appender, such as +Logback's `AsyncAppender`, log from workflow code with `Workflow.getLogger()`, +which skips logging during replay. diff --git a/contrib/temporal-opentelemetry-v2/build.gradle b/contrib/temporal-opentelemetry-v2/build.gradle new file mode 100644 index 0000000000..bc4000f89c --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/build.gradle @@ -0,0 +1,29 @@ +description = '''Temporal Java SDK OpenTelemetry v2 Module''' + +ext { + otelVersion = '1.66.0' + otelInstrumentationVersion = '2.32.0-alpha' +} + +dependencies { + api platform("io.opentelemetry:opentelemetry-bom:$otelVersion") + + compileOnly project(':temporal-serviceclient') + compileOnly project(':temporal-sdk') + compileOnly "javax.annotation:javax.annotation-api:$annotationApiVersion" + + implementation "com.google.guava:guava:$guavaVersion" + + api "io.opentelemetry:opentelemetry-api" + api "io.opentelemetry:opentelemetry-sdk-trace" + api "io.opentelemetry:opentelemetry-sdk-metrics" + api "io.opentelemetry:opentelemetry-sdk-logs" + + testImplementation project(':temporal-sdk') + testImplementation project(':temporal-serviceclient') + testImplementation project(':temporal-testing') + testImplementation "io.opentelemetry:opentelemetry-sdk-testing" + testImplementation "io.opentelemetry.instrumentation:opentelemetry-logback-appender-1.0:$otelInstrumentationVersion" + testImplementation "junit:junit:${junitVersion}" + testImplementation group: 'ch.qos.logback', name: 'logback-classic', version: "${logbackVersion}" +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryActivityClientInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryActivityClientInterceptor.java new file mode 100644 index 0000000000..3a06c5bde9 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryActivityClientInterceptor.java @@ -0,0 +1,20 @@ +package io.temporal.opentelemetry.v2; + +import io.temporal.common.interceptors.ActivityClientCallsInterceptor; +import io.temporal.common.interceptors.ActivityClientInterceptorBase; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryActivityClientCallsInterceptor; + +public class OpenTelemetryActivityClientInterceptor extends ActivityClientInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryActivityClientInterceptor(InterceptorTracer tracer) { + this.tracer = tracer; + } + + @Override + public ActivityClientCallsInterceptor activityClientCallsInterceptor( + ActivityClientCallsInterceptor next) { + return new OpenTelemetryActivityClientCallsInterceptor(tracer, next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryClientInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryClientInterceptor.java new file mode 100644 index 0000000000..fb7af88eab --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryClientInterceptor.java @@ -0,0 +1,20 @@ +package io.temporal.opentelemetry.v2; + +import io.temporal.common.interceptors.WorkflowClientCallsInterceptor; +import io.temporal.common.interceptors.WorkflowClientInterceptorBase; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryWorkflowClientCallsInterceptor; + +public class OpenTelemetryClientInterceptor extends WorkflowClientInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryClientInterceptor(InterceptorTracer tracer) { + this.tracer = tracer; + } + + @Override + public WorkflowClientCallsInterceptor workflowClientCallsInterceptor( + WorkflowClientCallsInterceptor next) { + return new OpenTelemetryWorkflowClientCallsInterceptor(tracer, next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryNexusClientInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryNexusClientInterceptor.java new file mode 100644 index 0000000000..46853d0bc3 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryNexusClientInterceptor.java @@ -0,0 +1,19 @@ +package io.temporal.opentelemetry.v2; + +import io.temporal.common.interceptors.NexusClientCallsInterceptor; +import io.temporal.common.interceptors.NexusClientInterceptorBase; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryNexusClientCallsInterceptor; + +public class OpenTelemetryNexusClientInterceptor extends NexusClientInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryNexusClientInterceptor(InterceptorTracer tracer) { + this.tracer = tracer; + } + + @Override + public NexusClientCallsInterceptor nexusClientCallsInterceptor(NexusClientCallsInterceptor next) { + return new OpenTelemetryNexusClientCallsInterceptor(tracer, next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryPlugin.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryPlugin.java new file mode 100644 index 0000000000..d57ee77efe --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryPlugin.java @@ -0,0 +1,83 @@ +package io.temporal.opentelemetry.v2; + +import io.opentelemetry.context.ContextStorage; +import io.temporal.common.Experimental; +import io.temporal.common.SimplePlugin; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryContextPropagator; +import io.temporal.opentelemetry.v2.internal.TemporalContextStorage; + +/** + * OpenTelemetry v2 plugin for Temporal clients and workers. + * + *

Set it on {@code WorkflowServiceStubsOptions} or {@code WorkflowClientOptions}. The SDK + * propagates plugins down that chain. + * + *

Register a {@link ReplaySafeOpenTelemetry} with {@code GlobalOpenTelemetry.set} before calling + * {@link Builder#build()}. + */ +@Experimental +public final class OpenTelemetryPlugin extends SimplePlugin { + public static final String NAME = "io.temporal.opentelemetry.v2"; + + private OpenTelemetryPlugin(InterceptorTracer tracer, String headerKey) { + super( + SimplePlugin.newBuilder(NAME) + .addClientInterceptors(new OpenTelemetryClientInterceptor(tracer)) + .addScheduleClientInterceptors(new OpenTelemetryScheduleClientInterceptor(tracer)) + .addActivityClientInterceptors(new OpenTelemetryActivityClientInterceptor(tracer)) + .addNexusClientInterceptors(new OpenTelemetryNexusClientInterceptor(tracer)) + .addWorkerInterceptors(new OpenTelemetryWorkerInterceptor(tracer)) + .addContextPropagators(new OpenTelemetryContextPropagator(headerKey))); + } + + public static Builder newBuilder() { + return new Builder(); + } + + /** Every option is optional; an unset one keeps its default. */ + public static final class Builder { + private String headerKey = "_tracer-data"; + private boolean addTemporalSpans; + + private Builder() {} + + /** + * The Temporal header key to serialize the span to. Defaults to {@code _tracer-data}, which + * Temporal uses; overriding it breaks trace continuity with workers using the standard key. + */ + public Builder setHeaderKey(String headerKey) { + this.headerKey = headerKey; + return this; + } + + /** + * Whether to create spans for Temporal operations such as StartWorkflow, RunWorkflow, and + * RunActivity. Defaults to false: trace context still propagates through Temporal headers, so + * spans created by application code remain connected. + */ + public Builder setAddTemporalSpans(boolean addTemporalSpans) { + this.addTemporalSpans = addTemporalSpans; + return this; + } + + public OpenTelemetryPlugin build() { + if (!ReplaySafeOpenTelemetry.isRegisteredGlobally()) { + throw new IllegalStateException( + "the global OpenTelemetry must be a ReplaySafeOpenTelemetry; build one with " + + "ReplaySafeOpenTelemetry.newBuilder() and register it with " + + "GlobalOpenTelemetry.set before building this plugin"); + } + // Initializing the storage applies every registered wrapper, and a wrapper registered after + // ours ends up outermost, so check that ours was applied rather than what is outermost. + ContextStorage.get(); + if (!TemporalContextStorage.isInstalled()) { + throw new IllegalStateException( + "the OpenTelemetry ContextStorage initialized before ReplaySafeOpenTelemetry could " + + "wrap it; build ReplaySafeOpenTelemetry before anything uses the OpenTelemetry " + + "Context"); + } + return new OpenTelemetryPlugin(new InterceptorTracer(headerKey, addTemporalSpans), headerKey); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryScheduleClientInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryScheduleClientInterceptor.java new file mode 100644 index 0000000000..01ab09bad2 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryScheduleClientInterceptor.java @@ -0,0 +1,20 @@ +package io.temporal.opentelemetry.v2; + +import io.temporal.common.interceptors.ScheduleClientCallsInterceptor; +import io.temporal.common.interceptors.ScheduleClientInterceptorBase; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryScheduleClientCallsInterceptor; + +public class OpenTelemetryScheduleClientInterceptor extends ScheduleClientInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryScheduleClientInterceptor(InterceptorTracer tracer) { + this.tracer = tracer; + } + + @Override + public ScheduleClientCallsInterceptor scheduleClientCallsInterceptor( + ScheduleClientCallsInterceptor next) { + return new OpenTelemetryScheduleClientCallsInterceptor(tracer, next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryWorkerInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryWorkerInterceptor.java new file mode 100644 index 0000000000..cf7d414f4a --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/OpenTelemetryWorkerInterceptor.java @@ -0,0 +1,35 @@ +package io.temporal.opentelemetry.v2; + +import io.nexusrpc.handler.OperationContext; +import io.temporal.common.interceptors.ActivityInboundCallsInterceptor; +import io.temporal.common.interceptors.NexusOperationInboundCallsInterceptor; +import io.temporal.common.interceptors.WorkerInterceptor; +import io.temporal.common.interceptors.WorkflowInboundCallsInterceptor; +import io.temporal.opentelemetry.v2.internal.InterceptorTracer; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryActivityInboundCallsInterceptor; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryNexusOperationInboundCallsInterceptor; +import io.temporal.opentelemetry.v2.internal.OpenTelemetryWorkflowInboundCallsInterceptor; + +public class OpenTelemetryWorkerInterceptor implements WorkerInterceptor { + private final InterceptorTracer tracer; + + public OpenTelemetryWorkerInterceptor(InterceptorTracer tracer) { + this.tracer = tracer; + } + + @Override + public WorkflowInboundCallsInterceptor interceptWorkflow(WorkflowInboundCallsInterceptor next) { + return new OpenTelemetryWorkflowInboundCallsInterceptor(tracer, next); + } + + @Override + public ActivityInboundCallsInterceptor interceptActivity(ActivityInboundCallsInterceptor next) { + return new OpenTelemetryActivityInboundCallsInterceptor(tracer, next); + } + + @Override + public NexusOperationInboundCallsInterceptor interceptNexusOperation( + OperationContext context, NexusOperationInboundCallsInterceptor next) { + return new OpenTelemetryNexusOperationInboundCallsInterceptor(tracer, next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java new file mode 100644 index 0000000000..953feb9896 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetry.java @@ -0,0 +1,331 @@ +package io.temporal.opentelemetry.v2; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; +import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.api.logs.LoggerBuilder; +import io.opentelemetry.api.logs.LoggerProvider; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterBuilder; +import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerBuilder; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.SdkLoggerProviderBuilder; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.trace.IdGenerator; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.temporal.common.Experimental; +import io.temporal.opentelemetry.v2.internal.ReplaySafeIdGenerator; +import io.temporal.opentelemetry.v2.internal.ReplaySafeLogger; +import io.temporal.opentelemetry.v2.internal.ReplaySafeMeter; +import io.temporal.opentelemetry.v2.internal.ReplaySafeTracer; +import io.temporal.opentelemetry.v2.internal.TemporalContextStorage; +import java.io.Closeable; +import javax.annotation.Nonnull; + +/** + * The {@link OpenTelemetry} to use for OpenTelemetry integration with Temporal. Register it with + * {@code GlobalOpenTelemetry.set}; tracers, meters, and loggers obtained from it are replay safe + * inside workflows. + * + *

Known limitations: + * + *

+ */ +@Experimental +public final class ReplaySafeOpenTelemetry implements OpenTelemetry, Closeable { + private final ReplaySafeTracerProvider tracerProvider; + private final ReplaySafeMeterProvider meterProvider; + private final ReplaySafeLoggerProvider loggerProvider; + private final ContextPropagators propagators; + + static { + // A wrapper, so Temporal's storage works alongside any storage the application already uses. + ContextStorage.addWrapper(TemporalContextStorage::new); + } + + private ReplaySafeOpenTelemetry(Builder builder) { + this.tracerProvider = + new ReplaySafeTracerProvider( + builder + .tracerProviderBuilder + .setIdGenerator(new ReplaySafeIdGenerator(builder.idGenerator)) + .build()); + this.meterProvider = new ReplaySafeMeterProvider(builder.meterProviderBuilder.build()); + this.loggerProvider = new ReplaySafeLoggerProvider(builder.loggerProviderBuilder.build()); + this.propagators = builder.propagators; + } + + public static Builder newBuilder() { + return new Builder(); + } + + @Override + public TracerProvider getTracerProvider() { + return tracerProvider; + } + + @Override + public MeterProvider getMeterProvider() { + return meterProvider; + } + + @Override + public LoggerProvider getLogsBridge() { + return loggerProvider; + } + + @Override + public ContextPropagators getPropagators() { + return propagators; + } + + static boolean isRegisteredGlobally() { + return GlobalOpenTelemetry.getOrNoop().getTracerProvider() instanceof ReplaySafeTracerProvider; + } + + /** Shuts down every provider. Call after the clients and workers using them have stopped. */ + @Override + public void close() { + tracerProvider.close(); + meterProvider.close(); + loggerProvider.close(); + } + + /** Every provider is optional; an unset one is built from the SDK's default builder. */ + public static final class Builder { + private SdkTracerProviderBuilder tracerProviderBuilder = SdkTracerProvider.builder(); + private IdGenerator idGenerator = IdGenerator.random(); + private SdkMeterProviderBuilder meterProviderBuilder = SdkMeterProvider.builder(); + private SdkLoggerProviderBuilder loggerProviderBuilder = SdkLoggerProvider.builder(); + private ContextPropagators propagators = + ContextPropagators.create( + TextMapPropagator.composite( + W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance())); + + private Builder() {} + + /** + * Any {@link IdGenerator} set on {@code tracerProviderBuilder} is replaced; use {@link + * #setIdGenerator} instead. + */ + public Builder setTracerProviderBuilder(SdkTracerProviderBuilder tracerProviderBuilder) { + this.tracerProviderBuilder = tracerProviderBuilder; + return this; + } + + /** + * Generates span and trace IDs outside workflow code. Workflow spans always use replay safe + * IDs. Defaults to {@link IdGenerator#random()}. + */ + public Builder setIdGenerator(IdGenerator idGenerator) { + this.idGenerator = idGenerator; + return this; + } + + public Builder setMeterProviderBuilder(SdkMeterProviderBuilder meterProviderBuilder) { + this.meterProviderBuilder = meterProviderBuilder; + return this; + } + + public Builder setLoggerProviderBuilder(SdkLoggerProviderBuilder loggerProviderBuilder) { + this.loggerProviderBuilder = loggerProviderBuilder; + return this; + } + + /** + * The propagators returned by {@link ReplaySafeOpenTelemetry#getPropagators()}. Defaults to W3C + * trace context plus baggage, which is what {@link OpenTelemetryPlugin} serializes into + * Temporal headers when it is left to resolve its propagator from the global. + */ + public Builder setPropagators(ContextPropagators propagators) { + this.propagators = propagators; + return this; + } + + public ReplaySafeOpenTelemetry build() { + return new ReplaySafeOpenTelemetry(this); + } + } + + private static final class ReplaySafeTracerProvider implements TracerProvider, Closeable { + private final SdkTracerProvider delegate; + + private ReplaySafeTracerProvider(SdkTracerProvider delegate) { + this.delegate = delegate; + } + + @Override + public Tracer get(@Nonnull String instrumentationScopeName) { + return new ReplaySafeTracer(delegate.get(instrumentationScopeName), instrumentationScopeName); + } + + @Override + public Tracer get( + @Nonnull String instrumentationScopeName, @Nonnull String instrumentationScopeVersion) { + return new ReplaySafeTracer( + delegate.get(instrumentationScopeName, instrumentationScopeVersion), + instrumentationScopeName); + } + + @Override + public TracerBuilder tracerBuilder(@Nonnull String instrumentationScopeName) { + return new ReplaySafeTracerBuilder( + delegate.tracerBuilder(instrumentationScopeName), instrumentationScopeName); + } + + @Override + public void close() { + delegate.close(); + } + } + + private static final class ReplaySafeMeterProvider implements MeterProvider, Closeable { + private final SdkMeterProvider delegate; + + private ReplaySafeMeterProvider(SdkMeterProvider delegate) { + this.delegate = delegate; + } + + @Override + public Meter get(@Nonnull String instrumentationScopeName) { + return new ReplaySafeMeter(delegate.get(instrumentationScopeName)); + } + + @Override + public MeterBuilder meterBuilder(@Nonnull String instrumentationScopeName) { + return new ReplaySafeMeterBuilder(delegate.meterBuilder(instrumentationScopeName)); + } + + @Override + public void close() { + delegate.close(); + } + } + + private static final class ReplaySafeMeterBuilder implements MeterBuilder { + private final MeterBuilder delegate; + + private ReplaySafeMeterBuilder(MeterBuilder delegate) { + this.delegate = delegate; + } + + @Override + public MeterBuilder setSchemaUrl(@Nonnull String schemaUrl) { + delegate.setSchemaUrl(schemaUrl); + return this; + } + + @Override + public MeterBuilder setInstrumentationVersion(@Nonnull String instrumentationScopeVersion) { + delegate.setInstrumentationVersion(instrumentationScopeVersion); + return this; + } + + @Override + public Meter build() { + return new ReplaySafeMeter(delegate.build()); + } + } + + private static final class ReplaySafeLoggerProvider implements LoggerProvider, Closeable { + private final SdkLoggerProvider delegate; + + private ReplaySafeLoggerProvider(SdkLoggerProvider delegate) { + this.delegate = delegate; + } + + @Override + public Logger get(@Nonnull String instrumentationScopeName) { + return new ReplaySafeLogger(delegate.get(instrumentationScopeName)); + } + + @Override + public LoggerBuilder loggerBuilder(@Nonnull String instrumentationScopeName) { + return new ReplaySafeLoggerBuilder(delegate.loggerBuilder(instrumentationScopeName)); + } + + @Override + public void close() { + delegate.close(); + } + } + + private static final class ReplaySafeLoggerBuilder implements LoggerBuilder { + private final LoggerBuilder delegate; + + private ReplaySafeLoggerBuilder(LoggerBuilder delegate) { + this.delegate = delegate; + } + + @Override + public LoggerBuilder setSchemaUrl(@Nonnull String schemaUrl) { + delegate.setSchemaUrl(schemaUrl); + return this; + } + + @Override + public LoggerBuilder setInstrumentationVersion(@Nonnull String instrumentationScopeVersion) { + delegate.setInstrumentationVersion(instrumentationScopeVersion); + return this; + } + + @Override + public Logger build() { + return new ReplaySafeLogger(delegate.build()); + } + } + + private static final class ReplaySafeTracerBuilder implements TracerBuilder { + private final TracerBuilder delegate; + private final String instrumentationScopeName; + + private ReplaySafeTracerBuilder(TracerBuilder delegate, String instrumentationScopeName) { + this.delegate = delegate; + this.instrumentationScopeName = instrumentationScopeName; + } + + @Override + public TracerBuilder setSchemaUrl(@Nonnull String schemaUrl) { + delegate.setSchemaUrl(schemaUrl); + return this; + } + + @Override + public TracerBuilder setInstrumentationVersion(@Nonnull String instrumentationScopeVersion) { + delegate.setInstrumentationVersion(instrumentationScopeVersion); + return this; + } + + @Override + public Tracer build() { + return new ReplaySafeTracer(delegate.build(), instrumentationScopeName); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java new file mode 100644 index 0000000000..884859de09 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/InterceptorTracer.java @@ -0,0 +1,184 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.temporal.common.interceptors.Header; +import io.temporal.failure.ApplicationErrorCategory; +import io.temporal.failure.ApplicationFailure; +import io.temporal.internal.sync.DestroyWorkflowThreadError; +import java.util.List; +import java.util.Map; + +/** Wraps intercepted Temporal calls in a span and propagates it through headers. */ +public final class InterceptorTracer { + private static final String INSTRUMENTATION_NAME = "temporal-sdk-java"; + + private final Tracer tracer; + private final SpanCodec codec; + private final boolean addTemporalSpans; + + public InterceptorTracer(String headerKey, boolean addTemporalSpans) { + this.tracer = GlobalOpenTelemetry.getTracer(INSTRUMENTATION_NAME); + this.codec = new SpanCodec(headerKey); + this.addTemporalSpans = addTemporalSpans; + } + + /** + * An intercepted call. {@code E} is the checked exception it declares, or {@link + * RuntimeException} when it declares none. + */ + @FunctionalInterface + interface Call { + R call() throws E; + } + + /** An intercepted call with no result. */ + @FunctionalInterface + interface VoidCall { + void call() throws E; + } + + R traceInbound( + String operation, String name, Attributes attributes, Header header, Call call) + throws E { + return traceInbound(operation, name, attributes, codec.read(header), call); + } + + void traceInbound( + String operation, String name, Attributes attributes, Header header, VoidCall call) + throws E { + traceInbound(operation, name, attributes, codec.read(header), asCall(call)); + } + + R traceNexusInbound( + String operation, + String name, + Attributes attributes, + Map nexusHeaders, + Call call) + throws E { + return traceInbound(operation, name, attributes, codec.read(nexusHeaders), call); + } + + private R traceInbound( + String operation, String name, Attributes attributes, Context parent, Call call) + throws E { + try (Scope ignoredParent = parent.makeCurrent()) { + if (!addTemporalSpans) { + return call.call(); + } + + return run(startSpan(operation, name, attributes, SpanKind.SERVER), call); + } + } + + R traceOutbound( + String operation, String name, Attributes attributes, Header header, Call call) + throws E { + return traceOutbound(operation, name, attributes, () -> codec.write(header), call); + } + + void replaceOutboundHeader(Header header) { + codec.replace(header); + } + + R traceOutbound( + String operation, String name, Attributes attributes, Call call) throws E { + return traceOutbound(operation, name, attributes, () -> {}, call); + } + + void traceOutbound( + String operation, String name, Attributes attributes, Header header, VoidCall call) + throws E { + traceOutbound(operation, name, attributes, () -> codec.write(header), asCall(call)); + } + + R traceOutbound( + String operation, String name, Attributes attributes, List
headers, Call call) + throws E { + return traceOutbound(operation, name, attributes, () -> headers.forEach(codec::write), call); + } + + R traceNexusOutbound( + String operation, + String name, + Attributes attributes, + Map nexusHeaders, + Call call) + throws E { + return traceOutbound(operation, name, attributes, () -> codec.write(nexusHeaders), call); + } + + private R traceOutbound( + String operation, String name, Attributes attributes, Runnable writeHeader, Call call) + throws E { + if (!addTemporalSpans) { + writeHeader.run(); + return call.call(); + } + + return run( + startSpan(operation, name, attributes, SpanKind.CLIENT), + () -> { + writeHeader.run(); + return call.call(); + }); + } + + private static Call asCall(VoidCall call) { + return () -> { + call.call(); + return null; + }; + } + + /** Runs {@code call} in {@code span}, recording any failure, then ends the span. */ + private static R run(Span span, Call call) throws E { + try (Scope ignored = span.makeCurrent()) { + return call.call(); + } catch (DestroyWorkflowThreadError unwind) { + // Eviction and continue-as-new unwind the thread; that is not a failure of the span. + throw unwind; + } catch (Throwable failure) { + span.recordException(failure); + if (!isBenign(failure)) { + span.setStatus(StatusCode.ERROR, failure.toString()); + } + throw failure; + } finally { + span.end(); + } + } + + private Span startSpan(String operation, String name, Attributes attributes, SpanKind kind) { + try (Scope ignored = + Context.current().with(ReplaySafeIdGenerator.INTERCEPTOR_SPAN, true).makeCurrent()) { + return tracer + .spanBuilder(spanName(operation, name)) + .setSpanKind(kind) + .setAllAttributes(attributes) + .startSpan(); + } + } + + static String spanName(String operation, String name) { + if (operation.isEmpty()) { + return name; + } + if (name.isEmpty()) { + return operation; + } + return operation + ":" + name; + } + + private static boolean isBenign(Throwable failure) { + return failure instanceof ApplicationFailure + && ((ApplicationFailure) failure).getCategory() == ApplicationErrorCategory.BENIGN; + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityClientCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityClientCallsInterceptor.java new file mode 100644 index 0000000000..fb901c2f59 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityClientCallsInterceptor.java @@ -0,0 +1,28 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.ACTIVITY_ID; + +import io.opentelemetry.api.common.Attributes; +import io.temporal.common.interceptors.ActivityClientCallsInterceptor; +import io.temporal.common.interceptors.ActivityClientCallsInterceptorBase; + +public class OpenTelemetryActivityClientCallsInterceptor + extends ActivityClientCallsInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryActivityClientCallsInterceptor( + InterceptorTracer tracer, ActivityClientCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public StartActivityOutput startActivity(StartActivityInput input) { + return tracer.traceOutbound( + "StartActivity", + input.getActivityType(), + Attributes.of(ACTIVITY_ID, input.getOptions().getId()), + input.getHeader(), + () -> super.startActivity(input)); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityInboundCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityInboundCallsInterceptor.java new file mode 100644 index 0000000000..6da63b04d6 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryActivityInboundCallsInterceptor.java @@ -0,0 +1,43 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.opentelemetry.api.common.Attributes; +import io.temporal.activity.ActivityExecutionContext; +import io.temporal.activity.ActivityInfo; +import io.temporal.common.interceptors.ActivityInboundCallsInterceptor; +import io.temporal.common.interceptors.ActivityInboundCallsInterceptorBase; + +public class OpenTelemetryActivityInboundCallsInterceptor + extends ActivityInboundCallsInterceptorBase { + private final InterceptorTracer tracer; + // Activity code reaches its context through Activity.getExecutionContext(), but interceptors + // only see it in init. + private ActivityExecutionContext context; + + public OpenTelemetryActivityInboundCallsInterceptor( + InterceptorTracer tracer, ActivityInboundCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public void init(ActivityExecutionContext context) { + this.context = context; + super.init(context); + } + + @Override + public ActivityOutput execute(ActivityInput input) { + ActivityInfo info = context.getInfo(); + return tracer.traceInbound( + "RunActivity", + info.getActivityType(), + Attributes.of( + WORKFLOW_ID, info.getWorkflowId(), + RUN_ID, info.getWorkflowRunId(), + ACTIVITY_ID, info.getActivityId()), + input.getHeader(), + () -> super.execute(input)); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryContextPropagator.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryContextPropagator.java new file mode 100644 index 0000000000..b08c462420 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryContextPropagator.java @@ -0,0 +1,47 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.context.Context; +import io.temporal.api.common.v1.Payload; +import io.temporal.common.context.ContextPropagator; +import java.util.Collections; +import java.util.Map; + +/** + * Copies the current OpenTelemetry context between Temporal workflow threads. The root workflow + * thread starts from the start header, so the workflow constructor already sees the caller's span. + */ +public final class OpenTelemetryContextPropagator implements ContextPropagator { + private final SpanCodec codec; + + public OpenTelemetryContextPropagator(String headerKey) { + this.codec = new SpanCodec(headerKey); + } + + @Override + public String getName() { + return "io.temporal.opentelemetry.v2.workflow-context"; + } + + @Override + public Map serializeContext(Object context) { + if (!(context instanceof Context)) { + return Collections.emptyMap(); + } + return codec.serialize((Context) context); + } + + @Override + public Object deserializeContext(Map header) { + return codec.readFromRoot(header); + } + + @Override + public Object getCurrentContext() { + return Context.current(); + } + + @Override + public void setCurrentContext(Object context) { + TemporalContextStorage.setWorkflowContext((Context) context); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusClientCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusClientCallsInterceptor.java new file mode 100644 index 0000000000..5cd729a725 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusClientCallsInterceptor.java @@ -0,0 +1,31 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.opentelemetry.api.common.Attributes; +import io.temporal.common.interceptors.NexusClientCallsInterceptor; +import io.temporal.common.interceptors.NexusClientCallsInterceptorBase; + +public class OpenTelemetryNexusClientCallsInterceptor extends NexusClientCallsInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryNexusClientCallsInterceptor( + InterceptorTracer tracer, NexusClientCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public StartNexusOperationExecutionOutput startNexusOperationExecution( + StartNexusOperationExecutionInput input) { + return tracer.traceNexusOutbound( + "StartNexusOperation", + input.getService() + "/" + input.getOperation(), + Attributes.of( + NEXUS_ENDPOINT, input.getEndpoint(), + NEXUS_SERVICE, input.getService(), + NEXUS_OPERATION, input.getOperation()), + input.getHeaders(), + () -> super.startNexusOperationExecution(input)); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationInboundCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationInboundCallsInterceptor.java new file mode 100644 index 0000000000..c06203c3cc --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationInboundCallsInterceptor.java @@ -0,0 +1,58 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.nexusrpc.OperationException; +import io.nexusrpc.handler.OperationContext; +import io.opentelemetry.api.common.Attributes; +import io.temporal.common.interceptors.NexusOperationInboundCallsInterceptor; +import io.temporal.common.interceptors.NexusOperationInboundCallsInterceptorBase; +import io.temporal.common.interceptors.NexusOperationOutboundCallsInterceptor; + +public class OpenTelemetryNexusOperationInboundCallsInterceptor + extends NexusOperationInboundCallsInterceptorBase { + + private final InterceptorTracer tracer; + + public OpenTelemetryNexusOperationInboundCallsInterceptor( + InterceptorTracer tracer, NexusOperationInboundCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public void init(NexusOperationOutboundCallsInterceptor outboundCalls) { + super.init(new OpenTelemetryNexusOperationOutboundCallsInterceptor(outboundCalls)); + } + + @Override + public StartOperationOutput startOperation(StartOperationInput input) throws OperationException { + OperationContext context = input.getOperationContext(); + return tracer.traceNexusInbound( + "RunStartNexusOperationHandler", + spanName(context), + nexusTags(context), + context.getHeaders(), + () -> super.startOperation(input)); + } + + @Override + public CancelOperationOutput cancelOperation(CancelOperationInput input) { + OperationContext context = input.getOperationContext(); + return tracer.traceNexusInbound( + "RunCancelNexusOperationHandler", + spanName(context), + nexusTags(context), + context.getHeaders(), + () -> super.cancelOperation(input)); + } + + private static String spanName(OperationContext context) { + return context.getService() + "/" + context.getOperation(); + } + + private static Attributes nexusTags(OperationContext context) { + return Attributes.of( + NEXUS_SERVICE, context.getService(), NEXUS_OPERATION, context.getOperation()); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationOutboundCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationOutboundCallsInterceptor.java new file mode 100644 index 0000000000..c8f1a3e5c8 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryNexusOperationOutboundCallsInterceptor.java @@ -0,0 +1,12 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.temporal.common.interceptors.NexusOperationOutboundCallsInterceptor; +import io.temporal.common.interceptors.NexusOperationOutboundCallsInterceptorBase; + +public class OpenTelemetryNexusOperationOutboundCallsInterceptor + extends NexusOperationOutboundCallsInterceptorBase { + public OpenTelemetryNexusOperationOutboundCallsInterceptor( + NexusOperationOutboundCallsInterceptor next) { + super(next); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java new file mode 100644 index 0000000000..e11b13f18d --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryScheduleClientCallsInterceptor.java @@ -0,0 +1,79 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.Attributes; +import io.temporal.client.schedules.Schedule; +import io.temporal.client.schedules.ScheduleActionStartWorkflow; +import io.temporal.client.schedules.ScheduleUpdate; +import io.temporal.client.schedules.ScheduleUpdateInput; +import io.temporal.common.interceptors.Header; +import io.temporal.common.interceptors.ScheduleClientCallsInterceptor; +import io.temporal.common.interceptors.ScheduleClientCallsInterceptorBase; +import java.util.HashMap; + +public class OpenTelemetryScheduleClientCallsInterceptor + extends ScheduleClientCallsInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryScheduleClientCallsInterceptor( + InterceptorTracer tracer, ScheduleClientCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public void createSchedule(CreateScheduleInput input) { + if (!(input.getSchedule().getAction() instanceof ScheduleActionStartWorkflow)) { + super.createSchedule(input); + return; + } + + tracer.traceOutbound( + "CreateSchedule", + input.getId(), + Attributes.empty(), + () -> { + super.createSchedule( + new CreateScheduleInput( + input.getId(), withTraceHeader(input.getSchedule()), input.getOptions())); + return null; + }); + } + + @Override + public void updateSchedule(UpdateScheduleInput input) { + tracer.traceOutbound( + "UpdateSchedule", + input.getDescription().getId(), + Attributes.empty(), + () -> { + super.updateSchedule( + new UpdateScheduleInput( + input.getDescription(), updateInput -> applyUpdate(input, updateInput))); + return null; + }); + } + + private ScheduleUpdate applyUpdate(UpdateScheduleInput input, ScheduleUpdateInput updateInput) { + ScheduleUpdate update = input.getUpdater().apply(updateInput); + if (update == null + || !(update.getSchedule().getAction() instanceof ScheduleActionStartWorkflow)) { + return update; + } + return new ScheduleUpdate( + withTraceHeader(update.getSchedule()), update.getTypedSearchAttributes()); + } + + /** + * Writes the current trace into a copy of the action header, so the caller's header, which may be + * immutable or reused, is never modified. A trace already in the header is removed, because it + * can be left over from an earlier create or update. + */ + private Schedule withTraceHeader(Schedule schedule) { + ScheduleActionStartWorkflow action = (ScheduleActionStartWorkflow) schedule.getAction(); + Header header = new Header(new HashMap<>(action.getHeader().getValues())); + tracer.replaceOutboundHeader(header); + return Schedule.newBuilder(schedule) + .setAction(ScheduleActionStartWorkflow.newBuilder(action).setHeader(header).build()) + .build(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetrySuppression.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetrySuppression.java new file mode 100644 index 0000000000..881639ebc6 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetrySuppression.java @@ -0,0 +1,18 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.temporal.workflow.unsafe.WorkflowUnsafe; + +/** Where replayed workflow code must not export telemetry again. */ +public final class OpenTelemetrySuppression { + private OpenTelemetrySuppression() {} + + /** + * True on a workflow thread that is re-executing history. Read-only code, such as query handlers, + * update validators, and await conditions, is never suppressed. It does not run a deterministic + * number of times, so its telemetry is emitted every time, with IDs that do not come from a + * random stream. + */ + public static boolean shouldSuppress() { + return !WorkflowUnsafe.isReadOnly() && WorkflowUnsafe.isReplaying(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowClientCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowClientCallsInterceptor.java new file mode 100644 index 0000000000..d76609a452 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowClientCallsInterceptor.java @@ -0,0 +1,121 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.temporal.client.WorkflowUpdateHandle; +import io.temporal.common.interceptors.WorkflowClientCallsInterceptor; +import io.temporal.common.interceptors.WorkflowClientCallsInterceptorBase; +import java.util.Arrays; + +public class OpenTelemetryWorkflowClientCallsInterceptor + extends WorkflowClientCallsInterceptorBase { + private final InterceptorTracer tracer; + + public OpenTelemetryWorkflowClientCallsInterceptor( + InterceptorTracer tracer, WorkflowClientCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public WorkflowStartOutput start(WorkflowStartInput input) { + return tracer.traceOutbound( + "StartWorkflow", + input.getWorkflowType(), + Attributes.of(WORKFLOW_ID, input.getWorkflowId()), + input.getHeader(), + () -> super.start(input)); + } + + @Override + public WorkflowSignalOutput signal(WorkflowSignalInput input) { + return tracer.traceOutbound( + "SignalWorkflow", + input.getSignalName(), + workflowExecutionTags(input.getWorkflowExecution()), + input.getHeader(), + () -> super.signal(input)); + } + + @Override + public WorkflowSignalWithStartOutput signalWithStart(WorkflowSignalWithStartInput input) { + WorkflowStartInput start = input.getWorkflowStartInput(); + return tracer.traceOutbound( + "SignalWithStartWorkflow", + start.getWorkflowType(), + Attributes.of(WORKFLOW_ID, start.getWorkflowId()), + start.getHeader(), + () -> super.signalWithStart(input)); + } + + @Override + public QueryOutput query(QueryInput input) { + return tracer.traceOutbound( + "QueryWorkflow", + input.getQueryType(), + workflowExecutionTags(input.getWorkflowExecution()), + input.getHeader(), + () -> super.query(input)); + } + + @Override + public WorkflowUpdateHandle startUpdate(StartUpdateInput input) { + AttributesBuilder attributes = workflowExecutionTags(input.getWorkflowExecution()).toBuilder(); + attributes.put(UPDATE_ID, input.getUpdateId()); + return tracer.traceOutbound( + "StartWorkflowUpdate", + input.getUpdateName(), + attributes.build(), + input.getHeader(), + () -> super.startUpdate(input)); + } + + @Override + public CancelOutput cancel(CancelInput input) { + return tracer.traceOutbound( + "CancelWorkflow", + "", + workflowExecutionTags(input.getWorkflowExecution()), + () -> super.cancel(input)); + } + + @Override + public TerminateOutput terminate(TerminateInput input) { + AttributesBuilder attributes = workflowExecutionTags(input.getWorkflowExecution()).toBuilder(); + if (input.getReason() != null) { + attributes.put(TERMINATE_REASON, input.getReason()); + } + return tracer.traceOutbound( + "TerminateWorkflow", "", attributes.build(), () -> super.terminate(input)); + } + + @Override + public DescribeWorkflowOutput describe(DescribeWorkflowInput input) { + return tracer.traceOutbound( + "DescribeWorkflow", + "", + workflowExecutionTags(input.getWorkflowExecution()), + () -> super.describe(input)); + } + + @Override + public WorkflowUpdateWithStartOutput updateWithStart( + WorkflowUpdateWithStartInput input) { + WorkflowStartInput start = input.getWorkflowStartInput(); + StartUpdateInput update = input.getStartUpdateInput(); + // The start header reaches the workflow and the update header reaches the update handler. + return tracer.traceOutbound( + "UpdateWithStartWorkflow", + update.getUpdateName(), + Attributes.of(WORKFLOW_ID, start.getWorkflowId(), UPDATE_ID, update.getUpdateId()), + Arrays.asList(start.getHeader(), update.getHeader()), + () -> super.updateWithStart(input)); + } + + private static Attributes workflowExecutionTags( + io.temporal.api.common.v1.WorkflowExecution execution) { + return Attributes.of(WORKFLOW_ID, execution.getWorkflowId(), RUN_ID, execution.getRunId()); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowInboundCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowInboundCallsInterceptor.java new file mode 100644 index 0000000000..c0d033bde7 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowInboundCallsInterceptor.java @@ -0,0 +1,89 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.temporal.common.interceptors.WorkflowInboundCallsInterceptor; +import io.temporal.common.interceptors.WorkflowInboundCallsInterceptorBase; +import io.temporal.common.interceptors.WorkflowOutboundCallsInterceptor; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInfo; + +public class OpenTelemetryWorkflowInboundCallsInterceptor + extends WorkflowInboundCallsInterceptorBase { + + private final InterceptorTracer tracer; + + public OpenTelemetryWorkflowInboundCallsInterceptor( + InterceptorTracer tracer, WorkflowInboundCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + @Override + public void init(WorkflowOutboundCallsInterceptor outboundCalls) { + super.init(new OpenTelemetryWorkflowOutboundCallsInterceptor(tracer, outboundCalls)); + } + + @Override + public WorkflowOutput execute(WorkflowInput input) { + return tracer.traceInbound( + "RunWorkflow", + Workflow.getInfo().getWorkflowType(), + workflowTags(), + input.getHeader(), + () -> super.execute(input)); + } + + @Override + public void handleSignal(SignalInput input) { + tracer.traceInbound( + "HandleSignal", + input.getSignalName(), + workflowTags(), + input.getHeader(), + () -> super.handleSignal(input)); + } + + @Override + public QueryOutput handleQuery(QueryInput input) { + return tracer.traceInbound( + "HandleQuery", + input.getQueryName(), + workflowTags(), + input.getHeader(), + () -> super.handleQuery(input)); + } + + @Override + public void validateUpdate(UpdateInput input) { + tracer.traceInbound( + "ValidateUpdate", + input.getUpdateName(), + updateTags(), + input.getHeader(), + () -> super.validateUpdate(input)); + } + + @Override + public UpdateOutput executeUpdate(UpdateInput input) { + return tracer.traceInbound( + "HandleUpdate", + input.getUpdateName(), + updateTags(), + input.getHeader(), + () -> super.executeUpdate(input)); + } + + private static Attributes workflowTags() { + WorkflowInfo info = Workflow.getInfo(); + return Attributes.of(WORKFLOW_ID, info.getWorkflowId(), RUN_ID, info.getRunId()); + } + + private static Attributes updateTags() { + AttributesBuilder tags = workflowTags().toBuilder(); + Workflow.getCurrentUpdateInfo().ifPresent(update -> tags.put(UPDATE_ID, update.getUpdateId())); + return tags.build(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowOutboundCallsInterceptor.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowOutboundCallsInterceptor.java new file mode 100644 index 0000000000..9c0bad9692 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/OpenTelemetryWorkflowOutboundCallsInterceptor.java @@ -0,0 +1,236 @@ +package io.temporal.opentelemetry.v2.internal; + +import static io.temporal.opentelemetry.v2.internal.TagKeys.*; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.common.interceptors.WorkflowOutboundCallsInterceptor; +import io.temporal.common.interceptors.WorkflowOutboundCallsInterceptorBase; +import io.temporal.workflow.Functions; +import io.temporal.workflow.Promise; +import io.temporal.workflow.TimerOptions; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInfo; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.annotation.Nullable; + +public class OpenTelemetryWorkflowOutboundCallsInterceptor + extends WorkflowOutboundCallsInterceptorBase { + + private final InterceptorTracer tracer; + + public OpenTelemetryWorkflowOutboundCallsInterceptor( + InterceptorTracer tracer, WorkflowOutboundCallsInterceptor next) { + super(next); + this.tracer = tracer; + } + + /** + * Makes the context of the calling workflow code current while a callback runs. The SDK runs + * callbacks registered on an incomplete promise in a separate workflow thread, which would + * otherwise see the workflow's root context. + */ + private static class ContextPromise implements Promise { + private final Promise delegate; + + ContextPromise(Promise delegate) { + this.delegate = delegate; + } + + private static O wrap(Context context, Functions.Func fn) { + try (Scope ignored = context.makeCurrent()) { + return fn.apply(); + } + } + + @Override + public boolean isCompleted() { + return delegate.isCompleted(); + } + + @Override + public R get() { + return delegate.get(); + } + + @Override + public R cancellableGet() { + return delegate.cancellableGet(); + } + + @Override + public R get(long timeout, TimeUnit unit) throws TimeoutException { + return delegate.get(timeout, unit); + } + + @Override + public R cancellableGet(long timeout, TimeUnit unit) throws TimeoutException { + return delegate.cancellableGet(timeout, unit); + } + + @Override + public RuntimeException getFailure() { + return delegate.getFailure(); + } + + @Override + public Promise thenApply(Functions.Func1 fn) { + Context context = Context.current(); + return new ContextPromise<>(delegate.thenApply((r) -> wrap(context, () -> fn.apply(r)))); + } + + @Override + public Promise handle(Functions.Func2 fn) { + Context context = Context.current(); + return new ContextPromise<>(delegate.handle((r, e) -> wrap(context, () -> fn.apply(r, e)))); + } + + @Override + public Promise thenCompose(Functions.Func1> fn) { + Context context = Context.current(); + return new ContextPromise<>(delegate.thenCompose((r) -> wrap(context, () -> fn.apply(r)))); + } + + @Override + public Promise exceptionally(Functions.Func1 fn) { + Context context = Context.current(); + return new ContextPromise<>(delegate.exceptionally((t) -> wrap(context, () -> fn.apply(t)))); + } + } + + @Override + public ActivityOutput executeActivity(ActivityInput input) { + ActivityOutput output = + tracer.traceOutbound( + "StartActivity", + input.getActivityName(), + activityTags(input.getActivityId()), + input.getHeader(), + () -> super.executeActivity(input)); + return new ActivityOutput<>(output.getActivityId(), new ContextPromise<>(output.getResult())); + } + + @Override + public LocalActivityOutput executeLocalActivity(LocalActivityInput input) { + LocalActivityOutput output = + tracer.traceOutbound( + "StartActivity", + input.getActivityName(), + activityTags(input.getActivityId()), + input.getHeader(), + () -> super.executeLocalActivity(input)); + return new LocalActivityOutput<>(new ContextPromise<>(output.getResult())); + } + + @Override + public ChildWorkflowOutput executeChildWorkflow(ChildWorkflowInput input) { + ChildWorkflowOutput output = + tracer.traceOutbound( + "StartChildWorkflow", + input.getWorkflowType(), + childWorkflowTags(input), + input.getHeader(), + () -> super.executeChildWorkflow(input)); + return new ChildWorkflowOutput<>( + new ContextPromise<>(output.getResult()), + new ContextPromise<>(output.getWorkflowExecution())); + } + + @Override + public ExecuteNexusOperationOutput executeNexusOperation( + ExecuteNexusOperationInput input) { + ExecuteNexusOperationOutput output = + tracer.traceNexusOutbound( + "StartNexusOperation", + input.getService() + "/" + input.getOperation(), + nexusTags(input), + input.getHeaders(), + () -> super.executeNexusOperation(input)); + return new ExecuteNexusOperationOutput<>( + new ContextPromise<>(output.getResult()), + new ContextPromise<>(output.getOperationExecution())); + } + + @Override + public Promise newTimer(Duration duration) { + return new ContextPromise<>(super.newTimer(duration)); + } + + @Override + public Promise newTimer(Duration duration, TimerOptions options) { + return new ContextPromise<>(super.newTimer(duration, options)); + } + + @Override + public SignalExternalOutput signalExternalWorkflow(SignalExternalInput input) { + SignalExternalOutput output = + tracer.traceOutbound( + "SignalExternalWorkflow", + input.getSignalName(), + workflowExecutionTags(input.getExecution()), + input.getHeader(), + () -> super.signalExternalWorkflow(input)); + return new SignalExternalOutput(new ContextPromise<>(output.getResult())); + } + + @Override + public CancelWorkflowOutput cancelWorkflow(CancelWorkflowInput input) { + CancelWorkflowOutput output = + tracer.traceOutbound( + "CancelWorkflow", + "", + workflowExecutionTags(input.getExecution()), + () -> super.cancelWorkflow(input)); + return new CancelWorkflowOutput(new ContextPromise<>(output.getResult())); + } + + @Override + public void continueAsNew(ContinueAsNewInput input) { + String workflowType = input.getWorkflowType(); + if (workflowType == null) { + workflowType = Workflow.getInfo().getWorkflowType(); + } + tracer.traceOutbound( + "ContinueAsNew", + workflowType, + workflowTags(), + input.getHeader(), + () -> super.continueAsNew(input)); + } + + private static Attributes workflowTags() { + WorkflowInfo info = Workflow.getInfo(); + return Attributes.of(WORKFLOW_ID, info.getWorkflowId(), RUN_ID, info.getRunId()); + } + + private static Attributes activityTags(@Nullable String activityId) { + Attributes tags = workflowTags(); + if (activityId == null) { + return tags; + } + return tags.toBuilder().put(ACTIVITY_ID, activityId).build(); + } + + private static Attributes workflowExecutionTags(WorkflowExecution execution) { + return Attributes.of(WORKFLOW_ID, execution.getWorkflowId(), RUN_ID, execution.getRunId()); + } + + private static Attributes childWorkflowTags(ChildWorkflowInput input) { + return Attributes.of(WORKFLOW_ID, input.getWorkflowId()); + } + + private static Attributes nexusTags(ExecuteNexusOperationInput input) { + AttributesBuilder tags = + workflowTags().toBuilder() + .put(NEXUS_SERVICE, input.getService()) + .put(NEXUS_OPERATION, input.getOperation()) + .put(NEXUS_ENDPOINT, input.getEndpoint()); + + return tags.build(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeIdGenerator.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeIdGenerator.java new file mode 100644 index 0000000000..2f51a64638 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeIdGenerator.java @@ -0,0 +1,93 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.trace.SpanId; +import io.opentelemetry.api.trace.TraceId; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.sdk.trace.IdGenerator; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.unsafe.WorkflowUnsafe; +import java.util.Random; +import javax.annotation.Nullable; + +/** + * Generates span and trace IDs that are replay safe. + * + *

Mirrors OpenTelemetry's RandomIdGenerator, + * replacing its platform random source with a workflow random stream. + * + *

Note: {@link Random} has 48 bits of state, so IDs can collide across workflows once an + * installation has generated on the order of 2^24 trace IDs. The JDK has no deterministic random + * source with a wider seed that can also be reseeded. A random source with more state may be + * considered in the future. + */ +public final class ReplaySafeIdGenerator implements IdGenerator { + static final ContextKey INTERCEPTOR_SPAN = ContextKey.named("temporal-interceptor-span"); + private static final String INTERCEPTOR_STREAM = "io.temporal.opentelemetry.v2/interceptor"; + private static final String APPLICATION_STREAM = "io.temporal.opentelemetry.v2/application"; + + private static final long INVALID_ID = 0; + + private final IdGenerator delegate; + + /** + * @param delegate generates IDs wherever replay safety isn't needed + */ + public ReplaySafeIdGenerator(IdGenerator delegate) { + this.delegate = delegate; + } + + @Override + public String generateSpanId() { + Random stream = getStream(); + if (stream == null) { + return delegate.generateSpanId(); + } + + long id; + do { + id = stream.nextLong(); + } while (id == INVALID_ID); + return SpanId.fromLong(id); + } + + @Override + public String generateTraceId() { + Random stream = getStream(); + if (stream == null) { + return delegate.generateTraceId(); + } + + long idHi = stream.nextLong(); + long idLo; + do { + idLo = stream.nextLong(); + } while (idLo == INVALID_ID); + return TraceId.fromLongs(idHi, idLo); + } + + /** + * Interceptor spans and application spans draw from separate streams so their IDs never collide. + * Null means the delegate can be used. + */ + @Nullable + private static Random getStream() { + if (!WorkflowUnsafe.isWorkflowThread() || WorkflowUnsafe.isReadOnly()) { + return null; + } + + Context context = Context.current(); + if (context.get(INTERCEPTOR_SPAN) != null) { + return Workflow.getRandomStream(INTERCEPTOR_STREAM); + } + + String tracerName = context.get(ReplaySafeTracer.TRACER_NAME); + if (tracerName == null) { + throw new IllegalStateException( + "Workflow span started without a replay safe tracer. Ensure tracers used in workflows " + + "are obtained from ReplaySafeOpenTelemetry"); + } + return Workflow.getRandomStream(APPLICATION_STREAM + "/" + tracerName); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java new file mode 100644 index 0000000000..864f83d5e0 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeLogger.java @@ -0,0 +1,170 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.Value; +import io.opentelemetry.api.logs.LogRecordBuilder; +import io.opentelemetry.api.logs.Logger; +import io.opentelemetry.api.logs.Severity; +import io.opentelemetry.context.Context; +import java.time.Instant; +import java.util.concurrent.TimeUnit; + +/** + * Wraps a logger so the records it builds are not emitted by replaying workflow code, which would + * otherwise emit them again on every replay. Replay is checked on the thread that calls {@link + * LogRecordBuilder#emit()}, so records emitted on another thread, such as an asynchronous + * appender's worker, are not suppressed. + */ +public final class ReplaySafeLogger implements Logger { + private final Logger delegate; + + public ReplaySafeLogger(Logger delegate) { + this.delegate = delegate; + } + + @Override + public LogRecordBuilder logRecordBuilder() { + return new ReplaySafeLogRecordBuilder(delegate.logRecordBuilder()); + } + + @Override + public boolean isEnabled(Severity severity, Context context) { + return delegate.isEnabled(severity, context); + } + + @Override + public boolean isEnabled(Severity severity) { + return delegate.isEnabled(severity); + } + + private static final class ReplaySafeLogRecordBuilder implements LogRecordBuilder { + private final LogRecordBuilder delegate; + + ReplaySafeLogRecordBuilder(LogRecordBuilder delegate) { + this.delegate = delegate; + } + + @Override + public void emit() { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.emit(); + } + + @Override + public LogRecordBuilder setTimestamp(long timestamp, TimeUnit unit) { + delegate.setTimestamp(timestamp, unit); + return this; + } + + @Override + public LogRecordBuilder setTimestamp(Instant instant) { + delegate.setTimestamp(instant); + return this; + } + + @Override + public LogRecordBuilder setObservedTimestamp(long timestamp, TimeUnit unit) { + delegate.setObservedTimestamp(timestamp, unit); + return this; + } + + @Override + public LogRecordBuilder setObservedTimestamp(Instant instant) { + delegate.setObservedTimestamp(instant); + return this; + } + + @Override + public LogRecordBuilder setContext(Context context) { + delegate.setContext(context); + return this; + } + + @Override + public LogRecordBuilder setSeverity(Severity severity) { + delegate.setSeverity(severity); + return this; + } + + @Override + public LogRecordBuilder setSeverityText(String severityText) { + delegate.setSeverityText(severityText); + return this; + } + + @Override + public LogRecordBuilder setBody(String body) { + delegate.setBody(body); + return this; + } + + @Override + public LogRecordBuilder setBody(Value body) { + delegate.setBody(body); + return this; + } + + @Override + public LogRecordBuilder setAllAttributes(Attributes attributes) { + delegate.setAllAttributes(attributes); + return this; + } + + @Override + public LogRecordBuilder setAttribute(AttributeKey key, T value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, String value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, long value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, double value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, boolean value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, int value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setAttribute(String key, Value value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public LogRecordBuilder setEventName(String eventName) { + delegate.setEventName(eventName); + return this; + } + + @Override + public LogRecordBuilder setException(Throwable throwable) { + delegate.setException(throwable); + return this; + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java new file mode 100644 index 0000000000..921058b5ad --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeMeter.java @@ -0,0 +1,691 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.DoubleCounter; +import io.opentelemetry.api.metrics.DoubleCounterBuilder; +import io.opentelemetry.api.metrics.DoubleGauge; +import io.opentelemetry.api.metrics.DoubleGaugeBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.DoubleHistogramBuilder; +import io.opentelemetry.api.metrics.DoubleUpDownCounter; +import io.opentelemetry.api.metrics.DoubleUpDownCounterBuilder; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongCounterBuilder; +import io.opentelemetry.api.metrics.LongGauge; +import io.opentelemetry.api.metrics.LongGaugeBuilder; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.api.metrics.LongHistogramBuilder; +import io.opentelemetry.api.metrics.LongUpDownCounter; +import io.opentelemetry.api.metrics.LongUpDownCounterBuilder; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.ObservableDoubleCounter; +import io.opentelemetry.api.metrics.ObservableDoubleGauge; +import io.opentelemetry.api.metrics.ObservableDoubleMeasurement; +import io.opentelemetry.api.metrics.ObservableDoubleUpDownCounter; +import io.opentelemetry.api.metrics.ObservableLongCounter; +import io.opentelemetry.api.metrics.ObservableLongGauge; +import io.opentelemetry.api.metrics.ObservableLongMeasurement; +import io.opentelemetry.api.metrics.ObservableLongUpDownCounter; +import io.opentelemetry.api.metrics.ObservableMeasurement; +import io.opentelemetry.context.Context; +import io.temporal.workflow.unsafe.WorkflowUnsafe; +import java.util.List; +import java.util.function.Consumer; + +/** + * Wraps a meter so the synchronous instruments it builds drop recordings made by replaying workflow + * code, which would otherwise be recorded again on every replay. + * + *

Observable instruments cannot be created from workflow code. Their callbacks stay registered + * until the meter provider closes, so every replay would register another callback, and each + * callback would read workflow state from the reader thread without workflow synchronization. + */ +public final class ReplaySafeMeter implements Meter { + private final Meter delegate; + + public ReplaySafeMeter(Meter delegate) { + this.delegate = delegate; + } + + @Override + public LongCounterBuilder counterBuilder(String name) { + return new ReplaySafeLongCounterBuilder(delegate.counterBuilder(name)); + } + + @Override + public LongUpDownCounterBuilder upDownCounterBuilder(String name) { + return new ReplaySafeLongUpDownCounterBuilder(delegate.upDownCounterBuilder(name)); + } + + @Override + public DoubleHistogramBuilder histogramBuilder(String name) { + return new ReplaySafeDoubleHistogramBuilder(delegate.histogramBuilder(name)); + } + + @Override + public DoubleGaugeBuilder gaugeBuilder(String name) { + return new ReplaySafeDoubleGaugeBuilder(delegate.gaugeBuilder(name)); + } + + @Override + public BatchCallback batchCallback( + Runnable callback, + ObservableMeasurement observableMeasurement, + ObservableMeasurement... additionalMeasurements) { + rejectWorkflowThread(); + return delegate.batchCallback(callback, observableMeasurement, additionalMeasurements); + } + + private static void rejectWorkflowThread() { + if (WorkflowUnsafe.isWorkflowThread()) { + throw new IllegalStateException( + "Observable instruments cannot be created from workflow code."); + } + } + + private static final class ReplaySafeLongCounterBuilder implements LongCounterBuilder { + private final LongCounterBuilder delegate; + + ReplaySafeLongCounterBuilder(LongCounterBuilder delegate) { + this.delegate = delegate; + } + + @Override + public LongCounterBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public LongCounterBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public DoubleCounterBuilder ofDoubles() { + return new ReplaySafeDoubleCounterBuilder(delegate.ofDoubles()); + } + + @Override + public LongCounter build() { + return new ReplaySafeLongCounter(delegate.build()); + } + + @Override + public ObservableLongCounter buildWithCallback(Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableLongMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + } + + private static final class ReplaySafeDoubleCounterBuilder implements DoubleCounterBuilder { + private final DoubleCounterBuilder delegate; + + ReplaySafeDoubleCounterBuilder(DoubleCounterBuilder delegate) { + this.delegate = delegate; + } + + @Override + public DoubleCounterBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public DoubleCounterBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public DoubleCounter build() { + return new ReplaySafeDoubleCounter(delegate.build()); + } + + @Override + public ObservableDoubleCounter buildWithCallback( + Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableDoubleMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + } + + private static final class ReplaySafeLongUpDownCounterBuilder + implements LongUpDownCounterBuilder { + private final LongUpDownCounterBuilder delegate; + + ReplaySafeLongUpDownCounterBuilder(LongUpDownCounterBuilder delegate) { + this.delegate = delegate; + } + + @Override + public LongUpDownCounterBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public LongUpDownCounterBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public DoubleUpDownCounterBuilder ofDoubles() { + return new ReplaySafeDoubleUpDownCounterBuilder(delegate.ofDoubles()); + } + + @Override + public LongUpDownCounter build() { + return new ReplaySafeLongUpDownCounter(delegate.build()); + } + + @Override + public ObservableLongUpDownCounter buildWithCallback( + Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableLongMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + } + + private static final class ReplaySafeDoubleUpDownCounterBuilder + implements DoubleUpDownCounterBuilder { + private final DoubleUpDownCounterBuilder delegate; + + ReplaySafeDoubleUpDownCounterBuilder(DoubleUpDownCounterBuilder delegate) { + this.delegate = delegate; + } + + @Override + public DoubleUpDownCounterBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public DoubleUpDownCounterBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public DoubleUpDownCounter build() { + return new ReplaySafeDoubleUpDownCounter(delegate.build()); + } + + @Override + public ObservableDoubleUpDownCounter buildWithCallback( + Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableDoubleMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + } + + private static final class ReplaySafeDoubleHistogramBuilder implements DoubleHistogramBuilder { + private final DoubleHistogramBuilder delegate; + + ReplaySafeDoubleHistogramBuilder(DoubleHistogramBuilder delegate) { + this.delegate = delegate; + } + + @Override + public DoubleHistogramBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public DoubleHistogramBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public DoubleHistogramBuilder setExplicitBucketBoundariesAdvice(List bucketBoundaries) { + delegate.setExplicitBucketBoundariesAdvice(bucketBoundaries); + return this; + } + + @Override + public LongHistogramBuilder ofLongs() { + return new ReplaySafeLongHistogramBuilder(delegate.ofLongs()); + } + + @Override + public DoubleHistogram build() { + return new ReplaySafeDoubleHistogram(delegate.build()); + } + } + + private static final class ReplaySafeLongHistogramBuilder implements LongHistogramBuilder { + private final LongHistogramBuilder delegate; + + ReplaySafeLongHistogramBuilder(LongHistogramBuilder delegate) { + this.delegate = delegate; + } + + @Override + public LongHistogramBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public LongHistogramBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public LongHistogramBuilder setExplicitBucketBoundariesAdvice(List bucketBoundaries) { + delegate.setExplicitBucketBoundariesAdvice(bucketBoundaries); + return this; + } + + @Override + public LongHistogram build() { + return new ReplaySafeLongHistogram(delegate.build()); + } + } + + private static final class ReplaySafeDoubleGaugeBuilder implements DoubleGaugeBuilder { + private final DoubleGaugeBuilder delegate; + + ReplaySafeDoubleGaugeBuilder(DoubleGaugeBuilder delegate) { + this.delegate = delegate; + } + + @Override + public DoubleGaugeBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public DoubleGaugeBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public LongGaugeBuilder ofLongs() { + return new ReplaySafeLongGaugeBuilder(delegate.ofLongs()); + } + + @Override + public ObservableDoubleGauge buildWithCallback(Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableDoubleMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + + @Override + public DoubleGauge build() { + return new ReplaySafeDoubleGauge(delegate.build()); + } + } + + private static final class ReplaySafeLongGaugeBuilder implements LongGaugeBuilder { + private final LongGaugeBuilder delegate; + + ReplaySafeLongGaugeBuilder(LongGaugeBuilder delegate) { + this.delegate = delegate; + } + + @Override + public LongGaugeBuilder setDescription(String description) { + delegate.setDescription(description); + return this; + } + + @Override + public LongGaugeBuilder setUnit(String unit) { + delegate.setUnit(unit); + return this; + } + + @Override + public ObservableLongGauge buildWithCallback(Consumer callback) { + rejectWorkflowThread(); + return delegate.buildWithCallback(callback); + } + + @Override + public ObservableLongMeasurement buildObserver() { + rejectWorkflowThread(); + return delegate.buildObserver(); + } + + @Override + public LongGauge build() { + return new ReplaySafeLongGauge(delegate.build()); + } + } + + private static final class ReplaySafeLongCounter implements LongCounter { + private final LongCounter delegate; + + ReplaySafeLongCounter(LongCounter delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return delegate.isEnabled(); + } + + @Override + public void add(long value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value); + } + + @Override + public void add(long value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes); + } + + @Override + public void add(long value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes, context); + } + } + + private static final class ReplaySafeDoubleCounter implements DoubleCounter { + private final DoubleCounter delegate; + + ReplaySafeDoubleCounter(DoubleCounter delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return delegate.isEnabled(); + } + + @Override + public void add(double value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value); + } + + @Override + public void add(double value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes); + } + + @Override + public void add(double value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes, context); + } + } + + private static final class ReplaySafeLongUpDownCounter implements LongUpDownCounter { + private final LongUpDownCounter delegate; + + ReplaySafeLongUpDownCounter(LongUpDownCounter delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return delegate.isEnabled(); + } + + @Override + public void add(long value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value); + } + + @Override + public void add(long value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes); + } + + @Override + public void add(long value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes, context); + } + } + + private static final class ReplaySafeDoubleUpDownCounter implements DoubleUpDownCounter { + private final DoubleUpDownCounter delegate; + + ReplaySafeDoubleUpDownCounter(DoubleUpDownCounter delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return delegate.isEnabled(); + } + + @Override + public void add(double value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value); + } + + @Override + public void add(double value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes); + } + + @Override + public void add(double value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.add(value, attributes, context); + } + } + + private static final class ReplaySafeDoubleHistogram implements DoubleHistogram { + private final DoubleHistogram delegate; + + ReplaySafeDoubleHistogram(DoubleHistogram delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return delegate.isEnabled(); + } + + @Override + public void record(double value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value); + } + + @Override + public void record(double value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value, attributes); + } + + @Override + public void record(double value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value, attributes, context); + } + } + + private static final class ReplaySafeLongHistogram implements LongHistogram { + private final LongHistogram delegate; + + ReplaySafeLongHistogram(LongHistogram delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return delegate.isEnabled(); + } + + @Override + public void record(long value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value); + } + + @Override + public void record(long value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value, attributes); + } + + @Override + public void record(long value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.record(value, attributes, context); + } + } + + private static final class ReplaySafeDoubleGauge implements DoubleGauge { + private final DoubleGauge delegate; + + ReplaySafeDoubleGauge(DoubleGauge delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return delegate.isEnabled(); + } + + @Override + public void set(double value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value); + } + + @Override + public void set(double value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value, attributes); + } + + @Override + public void set(double value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value, attributes, context); + } + } + + private static final class ReplaySafeLongGauge implements LongGauge { + private final LongGauge delegate; + + ReplaySafeLongGauge(LongGauge delegate) { + this.delegate = delegate; + } + + @Override + public boolean isEnabled() { + return delegate.isEnabled(); + } + + @Override + public void set(long value) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value); + } + + @Override + public void set(long value, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value, attributes); + } + + @Override + public void set(long value, Attributes attributes, Context context) { + if (OpenTelemetrySuppression.shouldSuppress()) { + return; + } + delegate.set(value, attributes, context); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java new file mode 100644 index 0000000000..212c092c2c --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeSpan.java @@ -0,0 +1,94 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.StatusCode; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.unsafe.WorkflowUnsafe; +import java.util.concurrent.TimeUnit; + +/** Wraps a span so that replayed code does not end it, which would export a duplicate. */ +public final class ReplaySafeSpan implements Span { + private final Span delegate; + private boolean ended; + + public ReplaySafeSpan(Span delegate) { + this.delegate = delegate; + } + + @Override + public void end() { + if (ended) { + return; + } + ended = true; + if (WorkflowUnsafe.isReplaying()) { + return; + } + delegate.end(); + } + + @Override + public void end(long timestamp, TimeUnit unit) { + if (ended) { + return; + } + ended = true; + if (WorkflowUnsafe.isReplaying()) { + return; + } + delegate.end(timestamp, unit); + } + + @Override + public Span setAttribute(AttributeKey key, T value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public Span addEvent(String name, Attributes attributes) { + if (OpenTelemetrySuppression.shouldSuppress()) { + delegate.addEvent(name, attributes, Workflow.currentTimeMillis(), TimeUnit.MILLISECONDS); + } else { + delegate.addEvent(name, attributes); + } + return this; + } + + @Override + public Span addEvent(String name, Attributes attributes, long timestamp, TimeUnit unit) { + delegate.addEvent(name, attributes, timestamp, unit); + return this; + } + + @Override + public Span setStatus(StatusCode statusCode, String description) { + delegate.setStatus(statusCode, description); + return this; + } + + @Override + public Span recordException(Throwable exception, Attributes additionalAttributes) { + delegate.recordException(exception, additionalAttributes); + return this; + } + + @Override + public Span updateName(String name) { + delegate.updateName(name); + return this; + } + + @Override + public SpanContext getSpanContext() { + return delegate.getSpanContext(); + } + + @Override + public boolean isRecording() { + return !ended && delegate.isRecording(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java new file mode 100644 index 0000000000..7ac97a12b4 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/ReplaySafeTracer.java @@ -0,0 +1,129 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.context.Scope; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.unsafe.WorkflowUnsafe; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nonnull; + +/** + * Wraps a tracer so the spans it starts inside a workflow are replay safe. The tracer name is + * published on the current {@link Context} while the span starts, which is where {@link + * ReplaySafeIdGenerator} reads it. + */ +public final class ReplaySafeTracer implements Tracer { + /** The instrumentation name of the tracer starting the current span. */ + @Nonnull static final ContextKey TRACER_NAME = ContextKey.named("temporal-tracer-name"); + + private final Tracer delegate; + private final String name; + + public ReplaySafeTracer(Tracer delegate, String name) { + this.delegate = delegate; + this.name = name; + } + + @Override + public SpanBuilder spanBuilder(String spanName) { + return new NamedStreamSpanBuilder(delegate.spanBuilder(spanName), name); + } + + private static final class NamedStreamSpanBuilder implements SpanBuilder { + private final SpanBuilder delegate; + private final String tracerName; + private boolean startTimestampSet; + + NamedStreamSpanBuilder(SpanBuilder delegate, String tracerName) { + this.delegate = delegate; + this.tracerName = tracerName; + } + + @Override + public Span startSpan() { + if (OpenTelemetrySuppression.shouldSuppress() && !startTimestampSet) { + delegate.setStartTimestamp(Workflow.currentTimeMillis(), TimeUnit.MILLISECONDS); + } + try (Scope ignored = Context.current().with(TRACER_NAME, tracerName).makeCurrent()) { + Span span = delegate.startSpan(); + // Read-only code does not run a deterministic number of times, so its spans always export. + return WorkflowUnsafe.isReadOnly() ? span : new ReplaySafeSpan(span); + } + } + + @Override + public SpanBuilder setParent(Context context) { + delegate.setParent(context); + return this; + } + + @Override + public SpanBuilder setNoParent() { + delegate.setNoParent(); + return this; + } + + @Override + public SpanBuilder addLink(SpanContext spanContext) { + delegate.addLink(spanContext); + return this; + } + + @Override + public SpanBuilder addLink(SpanContext spanContext, Attributes attributes) { + delegate.addLink(spanContext, attributes); + return this; + } + + @Override + public SpanBuilder setAttribute(String key, String value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public SpanBuilder setAttribute(String key, long value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public SpanBuilder setAttribute(String key, double value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public SpanBuilder setAttribute(String key, boolean value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public SpanBuilder setAttribute(AttributeKey key, T value) { + delegate.setAttribute(key, value); + return this; + } + + @Override + public SpanBuilder setSpanKind(SpanKind spanKind) { + delegate.setSpanKind(spanKind); + return this; + } + + @Override + public SpanBuilder setStartTimestamp(long startTimestamp, TimeUnit unit) { + startTimestampSet = true; + delegate.setStartTimestamp(startTimestamp, unit); + return this; + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java new file mode 100644 index 0000000000..08cb8b82ce --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/SpanCodec.java @@ -0,0 +1,142 @@ +package io.temporal.opentelemetry.v2.internal; + +import com.google.common.reflect.TypeToken; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.propagation.TextMapGetter; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.opentelemetry.context.propagation.TextMapSetter; +import io.temporal.api.common.v1.Payload; +import io.temporal.common.converter.DefaultDataConverter; +import io.temporal.common.converter.StdConverterBackwardsCompatAdapter; +import io.temporal.common.interceptors.Header; +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.TreeMap; +import javax.annotation.Nullable; + +/** + * Serializes the current span and baggage into Temporal and Nexus headers and reads them back. + * + *

Temporal headers carry one {@link Properties} payload under the configured key. Nexus headers + * are flat and use HTTP header semantics, so they are read case-insensitively. + */ +final class SpanCodec { + private static final TextMapSetter> MAP_SETTER = Map::put; + + private static final TextMapGetter> MAP_GETTER = + new TextMapGetter>() { + @Override + public Iterable keys(Map carrier) { + return carrier.keySet(); + } + + @Override + @Nullable + public String get(Map carrier, String key) { + return carrier.get(key); + } + }; + private static final TextMapSetter PROPERTIES_SETTER = Properties::setProperty; + private static final TextMapGetter PROPERTIES_GETTER = + new TextMapGetter() { + @Override + public Iterable keys(Properties carrier) { + return carrier.stringPropertyNames(); + } + + @Override + @Nullable + public String get(Properties carrier, String key) { + return carrier.getProperty(key); + } + }; + private static final Type HASH_MAP_STRING_STRING_TYPE = + new TypeToken>() {}.getType(); + + private final TextMapPropagator propagator; + private final String headerKey; + + SpanCodec(String headerKey) { + this.propagator = GlobalOpenTelemetry.getPropagators().getTextMapPropagator(); + this.headerKey = headerKey; + } + + /** The current context extended with the span and baggage carried by {@code header}. */ + Context read(Header header) { + Payload payload = header.getValues().get(headerKey); + if (payload == null) { + return Context.current(); + } + return extract(decode(payload), PROPERTIES_GETTER); + } + + /** The root context extended with the span and baggage carried by {@code header}. */ + Context readFromRoot(Map header) { + Payload payload = header.get(headerKey); + if (payload == null) { + return Context.root(); + } + return propagator.extract(Context.root(), decode(payload), PROPERTIES_GETTER); + } + + /** The current context extended with the span and baggage carried by {@code nexusHeaders}. */ + Context read(Map nexusHeaders) { + // Nexus headers use HTTP header semantics, so the propagator must see them case-insensitively. + // See https://opentelemetry.io/docs/specs/otel/context/api-propagators/#get. + Map carrier = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + carrier.putAll(nexusHeaders); + return extract(carrier, MAP_GETTER); + } + + /** Writes the current span and baggage into {@code header}; leaves it untouched if empty. */ + void write(Header header) { + write(Context.current(), header.getValues()); + } + + /** Replaces the span and baggage in {@code header} with the current ones, if any. */ + void replace(Header header) { + header.getValues().remove(headerKey); + write(header); + } + + Map serialize(Context context) { + Map header = new HashMap<>(); + write(context, header); + return header.isEmpty() ? Collections.emptyMap() : header; + } + + private void write(Context context, Map header) { + Properties carrier = new Properties(); + propagator.inject(context, carrier, PROPERTIES_SETTER); + if (!carrier.isEmpty()) { + header.put(headerKey, encode(carrier)); + } + } + + /** Writes the current span and baggage into {@code nexusHeaders}. */ + void write(Map nexusHeaders) { + propagator.inject(Context.current(), nexusHeaders, MAP_SETTER); + } + + private Context extract(C carrier, TextMapGetter getter) { + Context current = Context.current(); + return propagator.extract(current, carrier, getter); + } + + private static Payload encode(Properties carrier) { + return DefaultDataConverter.STANDARD_INSTANCE.toPayload(carrier).get(); + } + + static Properties decode(Payload payload) { + Map decoded = + StdConverterBackwardsCompatAdapter.fromPayload( + payload, HashMap.class, HASH_MAP_STRING_STRING_TYPE); + Properties properties = new Properties(); + properties.putAll(decoded); + return properties; + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TagKeys.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TagKeys.java new file mode 100644 index 0000000000..a1ce777011 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TagKeys.java @@ -0,0 +1,19 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.api.common.AttributeKey; + +final class TagKeys { + static final AttributeKey WORKFLOW_ID = AttributeKey.stringKey("temporalWorkflowID"); + static final AttributeKey RUN_ID = AttributeKey.stringKey("temporalRunID"); + static final AttributeKey ACTIVITY_ID = AttributeKey.stringKey("temporalActivityID"); + static final AttributeKey UPDATE_ID = AttributeKey.stringKey("temporalUpdateID"); + static final AttributeKey TERMINATE_REASON = + AttributeKey.stringKey("temporalTerminateReason"); + static final AttributeKey NEXUS_SERVICE = AttributeKey.stringKey("temporalNexusService"); + static final AttributeKey NEXUS_OPERATION = + AttributeKey.stringKey("temporalNexusOperation"); + static final AttributeKey NEXUS_ENDPOINT = + AttributeKey.stringKey("temporalNexusEndpoint"); + + private TagKeys() {} +} diff --git a/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TemporalContextStorage.java b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TemporalContextStorage.java new file mode 100644 index 0000000000..c9ea1c0598 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/main/java/io/temporal/opentelemetry/v2/internal/TemporalContextStorage.java @@ -0,0 +1,59 @@ +package io.temporal.opentelemetry.v2.internal; + +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; +import io.temporal.workflow.WorkflowThreadLocal; +import io.temporal.workflow.unsafe.WorkflowUnsafe; + +/** Stores the current OpenTelemetry context in Temporal workflow threads. */ +public final class TemporalContextStorage implements ContextStorage { + private static final WorkflowThreadLocal WORKFLOW_CONTEXT = new WorkflowThreadLocal<>(); + private static volatile boolean installed; + + private final ContextStorage delegate; + + public TemporalContextStorage(ContextStorage delegate) { + this.delegate = delegate; + installed = true; + } + + /** True once OpenTelemetry has wrapped its context storage with this class. */ + public static boolean isInstalled() { + return installed; + } + + static void setWorkflowContext(Context context) { + if (WorkflowUnsafe.isWorkflowThread()) { + WORKFLOW_CONTEXT.set(context); + } + } + + @Override + public Scope attach(Context context) { + if (!WorkflowUnsafe.isWorkflowThread()) { + return delegate.attach(context); + } + + Context previous = WORKFLOW_CONTEXT.get(); + if (context == previous) { + return () -> {}; + } + WORKFLOW_CONTEXT.set(context); + return () -> { + if (WORKFLOW_CONTEXT.get() == context) { + WORKFLOW_CONTEXT.set(previous); + } + }; + } + + @Override + public Context current() { + return WorkflowUnsafe.isWorkflowThread() ? WORKFLOW_CONTEXT.get() : delegate.current(); + } + + @Override + public Context root() { + return delegate.root(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ContextStorageWrapperTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ContextStorageWrapperTest.java new file mode 100644 index 0000000000..5e461b8edd --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ContextStorageWrapperTest.java @@ -0,0 +1,46 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertTrue; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; +import org.junit.Test; + +public class ContextStorageWrapperTest { + @Test + public void pluginAcceptsAWrapperRegisteredAfterReplaySafeOpenTelemetry() { + try (ReplaySafeOpenTelemetry openTelemetry = ReplaySafeOpenTelemetry.newBuilder().build()) { + ContextStorage.addWrapper(OuterStorage::new); + GlobalOpenTelemetry.set(openTelemetry); + OpenTelemetryPlugin.newBuilder().build(); + } finally { + GlobalOpenTelemetry.resetForTest(); + } + assertTrue(ContextStorage.get() instanceof OuterStorage); + } + + private static final class OuterStorage implements ContextStorage { + private final ContextStorage delegate; + + OuterStorage(ContextStorage delegate) { + this.delegate = delegate; + } + + @Override + public Scope attach(Context context) { + return delegate.attach(context); + } + + @Override + public Context current() { + return delegate.current(); + } + + @Override + public Context root() { + return delegate.root(); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/InterceptorTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/InterceptorTest.java new file mode 100644 index 0000000000..ab09fb59a4 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/InterceptorTest.java @@ -0,0 +1,393 @@ +package io.temporal.opentelemetry.v2; + +import static io.temporal.opentelemetry.v2.TestWorkflows.OUTBOUND_ACTIVITY_ID; +import static io.temporal.opentelemetry.v2.TestWorkflows.OUTBOUND_LOCAL_ACTIVITY_ID; +import static io.temporal.opentelemetry.v2.TestWorkflows.TASK_TOKENS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.api.enums.v1.WorkflowIdConflictPolicy; +import io.temporal.client.UpdateOptions; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowFailedException; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.client.WorkflowTargetOptions; +import io.temporal.client.WorkflowUpdateStage; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncCompletionWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncCompletionWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.BenignErrorWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.BenignErrorWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ChildWorkflowWithSignal; +import io.temporal.opentelemetry.v2.TestWorkflows.ChildWorkflowWithSignalImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ErrorWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.ErrorWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.SpanKindWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.SpanKindWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TestActivitiesImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.UnservedWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.UnservedWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.UpdateTargetWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.WorkflowOutboundCall; +import io.temporal.opentelemetry.v2.TestWorkflows.WorkflowOutboundTagsWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.WorkflowOutboundTagsWorkflowImpl; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.junit.Rule; +import org.junit.Test; + +/** Tests interceptor behavior. */ +public class InterceptorTest extends OtelTestBase { + private static final String UNSERVED_TASK_QUEUE = "opentelemetry-v2-unserved"; + private static final String UPDATE_WITH_START_ID = "interceptor-update-with-start"; + private static final String TARGET_UPDATE_ID = "interceptor-update"; + private static final AttributeKey WORKFLOW_ID = + AttributeKey.stringKey("temporalWorkflowID"); + private static final AttributeKey RUN_ID = AttributeKey.stringKey("temporalRunID"); + private static final AttributeKey UPDATE_ID = AttributeKey.stringKey("temporalUpdateID"); + private static final AttributeKey ACTIVITY_ID = + AttributeKey.stringKey("temporalActivityID"); + + @Rule + public SDKTestWorkflowRule testWorkflowRule = + newRuleBuilder(true) + .setWorkflowTypes( + SpanKindWorkflowImpl.class, + BenignErrorWorkflowImpl.class, + ErrorWorkflowImpl.class, + AsyncCompletionWorkflowImpl.class, + TracerWorkflowImpl.class, + ChildWorkflowWithSignalImpl.class, + UnservedWorkflowImpl.class, + UpdateTargetWorkflowImpl.class, + WorkflowOutboundTagsWorkflowImpl.class) + .setActivityImplementations(new TestActivitiesImpl()) + .build(); + + @Test + public void spanKind() { + testWorkflowRule.newWorkflowStub(SpanKindWorkflow.class).run(); + + Map kinds = new HashMap<>(); + for (SpanData span : endedSpans()) { + kinds.put(span.getName(), span.getKind()); + } + assertEquals(SpanKind.SERVER, kinds.get("RunWorkflow:SpanKindWorkflow")); + assertEquals(SpanKind.CLIENT, kinds.get("StartActivity:NopActivity")); + assertEquals(SpanKind.SERVER, kinds.get("RunActivity:NopActivity")); + } + + @Test + public void historyReplayEmitsNoSpans() throws Exception { + SpanKindWorkflow workflow = testWorkflowRule.newWorkflowStub(SpanKindWorkflow.class); + workflow.run(); + String workflowId = WorkflowStub.fromTyped(workflow).getExecution().getWorkflowId(); + spanExporter.reset(); + + testWorkflowRule + .getWorker() + .replayWorkflowExecution(testWorkflowRule.getExecutionHistory(workflowId)); + + assertSpanTree(Collections.emptyList(), endedSpans()); + } + + @Test + public void workflowClientSignalIncludesTargetRunId() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run, false); + TracerWorkflow target = targetWorkflow(execution); + target.gate(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + assertWorkflowExecutionTags(requireSpanNamed(endedSpans(), "SignalWorkflow:gate"), execution); + } + + @Test + public void workflowClientQueryIncludesTargetRunId() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run, false); + TracerWorkflow target = targetWorkflow(execution); + + assertEquals("ok", target.query()); + target.gate(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + assertWorkflowExecutionTags(requireSpanNamed(endedSpans(), "QueryWorkflow:query"), execution); + } + + @Test + public void workflowClientUpdateIncludesTargetRunIdAndUpdateId() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run, false); + TracerWorkflow target = targetWorkflow(execution); + + WorkflowStub.fromTyped(target) + .startUpdate( + UpdateOptions.newBuilder(Void.class) + .setUpdateName("update") + .setUpdateId(TARGET_UPDATE_ID) + .setWaitForStage(WorkflowUpdateStage.COMPLETED) + .build()) + .getResult(); + target.gate(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + SpanData update = requireSpanNamed(endedSpans(), "StartWorkflowUpdate:update"); + assertWorkflowExecutionTags(update, execution); + assertEquals(TARGET_UPDATE_ID, update.getAttributes().get(UPDATE_ID)); + } + + @Test + public void workflowClientStartIncludesWorkflowId() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run, true); + + SpanData start = requireSpanNamed(endedSpans(), "StartWorkflow:TracerWorkflow"); + assertEquals(execution.getWorkflowId(), start.getAttributes().get(WORKFLOW_ID)); + assertNull(start.getAttributes().get(RUN_ID)); + } + + @Test + public void workflowClientUpdateWithStartIncludesWorkflowIdAndUpdateId() { + String workflowId = "interceptor-update-with-start-target"; + WorkflowStub target = + testWorkflowRule + .getWorkflowClient() + .newUntypedWorkflowStub( + "UpdateTargetWorkflow", + WorkflowOptions.newBuilder() + .setTaskQueue(testWorkflowRule.getTaskQueue()) + .setWorkflowId(workflowId) + .setWorkflowIdConflictPolicy( + WorkflowIdConflictPolicy.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING) + .build()); + + target + .startUpdateWithStart( + UpdateOptions.newBuilder(Void.class) + .setUpdateName("doUpdate") + .setUpdateId(UPDATE_WITH_START_ID) + .setWaitForStage(WorkflowUpdateStage.COMPLETED) + .build(), + new Object[0], + new Object[0]) + .getResult(); + target.signal("updateSignal"); + target.getResult(Void.class); + + SpanData updateWithStart = requireSpanNamed(endedSpans(), "UpdateWithStartWorkflow:doUpdate"); + assertEquals(workflowId, updateWithStart.getAttributes().get(WORKFLOW_ID)); + assertEquals(UPDATE_WITH_START_ID, updateWithStart.getAttributes().get(UPDATE_ID)); + } + + @Test + public void workflowOutboundActivityStartIncludesSuppliedActivityId() { + runWorkflowOutboundCall(WorkflowOutboundCall.ACTIVITY); + + SpanData start = requireSpanNamed(endedSpans(), "StartActivity:Activity"); + SpanData run = requireSpanNamed(endedSpans(), "RunActivity:Activity"); + assertEquals(OUTBOUND_ACTIVITY_ID, start.getAttributes().get(ACTIVITY_ID)); + assertEquals(OUTBOUND_ACTIVITY_ID, run.getAttributes().get(ACTIVITY_ID)); + } + + @Test + public void workflowOutboundLocalActivityStartIncludesSuppliedActivityId() { + runWorkflowOutboundCall(WorkflowOutboundCall.LOCAL_ACTIVITY); + + SpanData start = requireSpanNamed(endedSpans(), "StartActivity:LocalActivity"); + SpanData run = requireSpanNamed(endedSpans(), "RunActivity:LocalActivity"); + assertEquals(OUTBOUND_LOCAL_ACTIVITY_ID, start.getAttributes().get(ACTIVITY_ID)); + assertEquals(OUTBOUND_LOCAL_ACTIVITY_ID, run.getAttributes().get(ACTIVITY_ID)); + } + + @Test + public void workflowOutboundActivityStartOmitsGeneratedActivityId() { + testWorkflowRule.newWorkflowStub(SpanKindWorkflow.class).run(); + + SpanData start = requireSpanNamed(endedSpans(), "StartActivity:NopActivity"); + assertNull(start.getAttributes().get(ACTIVITY_ID)); + } + + @Test + public void workflowOutboundChildStartIncludesWorkflowId() { + runWorkflowOutboundCall(WorkflowOutboundCall.CHILD_WORKFLOW); + + SpanData start = requireSpanNamed(endedSpans(), "StartChildWorkflow:ChildWorkflowWithSignal"); + SpanData run = requireSpanNamed(endedSpans(), "RunWorkflow:ChildWorkflowWithSignal"); + assertEquals(run.getAttributes().get(WORKFLOW_ID), start.getAttributes().get(WORKFLOW_ID)); + assertNull(start.getAttributes().get(RUN_ID)); + } + + @Test + public void workflowOutboundChildSignalIncludesTargetExecution() { + runWorkflowOutboundCall(WorkflowOutboundCall.CHILD_WORKFLOW); + + SpanData run = requireSpanNamed(endedSpans(), "RunWorkflow:ChildWorkflowWithSignal"); + SpanData signal = requireSpanNamed(endedSpans(), "SignalExternalWorkflow:childSignal"); + assertWorkflowExecutionTags(signal, workflowExecution(run)); + } + + @Test + public void workflowOutboundExternalSignalIncludesTargetExecution() { + ChildWorkflowWithSignal target = + testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + ChildWorkflowWithSignal.class, + WorkflowOptions.newBuilder() + .setTaskQueue(testWorkflowRule.getTaskQueue()) + .setWorkflowId("external-signal-target-" + UUID.randomUUID()) + .build()); + WorkflowExecution execution = WorkflowClient.start(target::run); + + WorkflowOutboundTagsWorkflow workflow = + testWorkflowRule.newWorkflowStub(WorkflowOutboundTagsWorkflow.class); + workflow.run(WorkflowOutboundCall.EXTERNAL_SIGNAL, execution); + + assertWorkflowExecutionTags( + requireSpanNamed(endedSpans(), "SignalExternalWorkflow:childSignal"), execution); + } + + @Test + public void workflowOutboundCancelIncludesTargetExecution() { + UnservedWorkflow target = + testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + UnservedWorkflow.class, + WorkflowOptions.newBuilder() + .setTaskQueue(UNSERVED_TASK_QUEUE) + .setWorkflowId("external-cancel-target-" + UUID.randomUUID()) + .build()); + WorkflowExecution execution = WorkflowClient.start(target::run); + + WorkflowOutboundTagsWorkflow workflow = + testWorkflowRule.newWorkflowStub(WorkflowOutboundTagsWorkflow.class); + workflow.run(WorkflowOutboundCall.EXTERNAL_CANCEL, execution); + + assertWorkflowExecutionTags(requireSpanNamed(endedSpans(), "CancelWorkflow"), execution); + } + + private TracerWorkflow targetWorkflow(WorkflowExecution execution) { + return testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + TracerWorkflow.class, + WorkflowTargetOptions.newBuilder().setWorkflowExecution(execution).build()); + } + + private void runWorkflowOutboundCall(WorkflowOutboundCall call) { + WorkflowOutboundTagsWorkflow workflow = + testWorkflowRule.newWorkflowStub(WorkflowOutboundTagsWorkflow.class); + workflow.run(call, WorkflowExecution.getDefaultInstance()); + } + + private static WorkflowExecution workflowExecution(SpanData span) { + return WorkflowExecution.newBuilder() + .setWorkflowId(span.getAttributes().get(WORKFLOW_ID)) + .setRunId(span.getAttributes().get(RUN_ID)) + .build(); + } + + private static void assertWorkflowExecutionTags(SpanData span, WorkflowExecution execution) { + assertEquals(execution.getWorkflowId(), span.getAttributes().get(WORKFLOW_ID)); + assertEquals(execution.getRunId(), span.getAttributes().get(RUN_ID)); + } + + @Test + public void benignErrorLeavesSpanStatusUnset() { + assertThrows( + WorkflowFailedException.class, + () -> testWorkflowRule.newWorkflowStub(BenignErrorWorkflow.class).run()); + + assertSpanTree( + Arrays.asList("StartWorkflow:BenignErrorWorkflow", " RunWorkflow:BenignErrorWorkflow"), + endedSpans()); + SpanData run = requireSpanNamed(endedSpans(), "RunWorkflow:BenignErrorWorkflow"); + assertEquals(StatusCode.UNSET, run.getStatus().getStatusCode()); + } + + @Test + public void errorSetsSpanStatusError() { + assertThrows( + WorkflowFailedException.class, + () -> testWorkflowRule.newWorkflowStub(ErrorWorkflow.class).run()); + + assertSpanTree( + Arrays.asList("StartWorkflow:ErrorWorkflow", " RunWorkflow:ErrorWorkflow"), endedSpans()); + SpanData run = requireSpanNamed(endedSpans(), "RunWorkflow:ErrorWorkflow"); + assertEquals(StatusCode.ERROR, run.getStatus().getStatusCode()); + } + + @Test + public void continueAsNewLeavesWorkflowSpanUnset() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowClient.start(workflow::run, false); + workflow.gate(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + SpanData run = requireSpanNamed(endedSpans(), "RunWorkflow:TracerWorkflow"); + assertEquals(StatusCode.UNSET, run.getStatus().getStatusCode()); + assertTrue(run.getEvents().toString(), run.getEvents().isEmpty()); + } + + @Test + public void pendingActivityLeavesActivitySpanUnset() { + AsyncCompletionWorkflow workflow = + testWorkflowRule.newWorkflowStub(AsyncCompletionWorkflow.class); + WorkflowClient.start(workflow::run); + byte[] taskToken = takeTaskToken(); + testWorkflowRule.getWorkflowClient().newActivityCompletionClient().complete(taskToken, null); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + SpanData activity = requireSpanNamed(endedSpans(), "RunActivity:AsyncCompletionActivity"); + assertEquals(StatusCode.UNSET, activity.getStatus().getStatusCode()); + assertTrue(activity.getEvents().toString(), activity.getEvents().isEmpty()); + } + + @Test + public void evictedWorkflowLeavesRunWorkflowSpanOpen() { + ChildWorkflowWithSignal workflow = + testWorkflowRule.newWorkflowStub(ChildWorkflowWithSignal.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run); + testWorkflowRule.waitForTheEndOfWFT(execution.getWorkflowId()); + testWorkflowRule.invalidateWorkflowCache(); + + assertEquals(0, countSpansNamed("RunWorkflow:ChildWorkflowWithSignal")); + + workflow.childSignal(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + assertEquals(1, countSpansNamed("RunWorkflow:ChildWorkflowWithSignal")); + } + + private static long countSpansNamed(String name) { + return endedSpans().stream().filter(span -> span.getName().equals(name)).count(); + } + + private static byte[] takeTaskToken() { + try { + byte[] taskToken = TASK_TOKENS.poll(30, TimeUnit.SECONDS); + assertNotNull("timed out waiting for activity task token", taskToken); + return taskToken; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/LoggerTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/LoggerTest.java new file mode 100644 index 0000000000..24e6992613 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/LoggerTest.java @@ -0,0 +1,116 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import io.opentelemetry.api.logs.Severity; +import io.opentelemetry.sdk.logs.data.LogRecordData; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowStub; +import io.temporal.testing.WorkflowReplayer; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import io.temporal.workflow.QueryMethod; +import io.temporal.workflow.SignalMethod; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInterface; +import io.temporal.workflow.WorkflowMethod; +import java.util.ArrayList; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Verifies that log records emit during live execution and that replaying a workflow run does not + * emit duplicate records. + */ +public class LoggerTest extends OtelTestBase { + private static final Logger logger = LoggerFactory.getLogger(LoggerTest.class); + private static final String RUN_BODY = "custom workflow run"; + private static final String QUERY_BODY = "custom workflow query"; + + @WorkflowInterface + public interface TestWorkflow { + @WorkflowMethod + void run(); + + @QueryMethod + String queryAndEmit(); + + @SignalMethod + void proceed(); + } + + public static class TestWorkflowImpl implements TestWorkflow { + private boolean proceed; + + @Override + public void run() { + emit(RUN_BODY); + Workflow.await(() -> proceed); + } + + @Override + public String queryAndEmit() { + emit(QUERY_BODY); + return "ok"; + } + + @Override + public void proceed() { + proceed = true; + } + + private static void emit(String body) { + logger.info(body); + } + } + + @Rule + public SDKTestWorkflowRule testWorkflowRule = + newRuleBuilder(false).setWorkflowTypes(TestWorkflowImpl.class).build(); + + @Test + public void recordsEmitOutsideWorkflows() { + logger.warn("outside"); + + List logs = emittedLogs(); + assertEquals(1, logs.size()); + LogRecordData log = logs.get(0); + assertEquals(Severity.WARN, log.getSeverity()); + assertEquals("outside", log.getBodyValue().asString()); + } + + /** + * The live run and the live query handler each emit once; replaying the finished run drops its + * record. + */ + @Test + public void liveExecutionEmitsAndReplayDoesNotDuplicate() throws Exception { + TestWorkflow workflow = testWorkflowRule.newWorkflowStub(TestWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run); + + assertEquals("ok", workflow.queryAndEmit()); + workflow.proceed(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + // Replay the full workflow history and verify the run's record is suppressed on replay. + WorkflowReplayer.replayWorkflowExecution( + testWorkflowRule.getExecutionHistory(execution.getWorkflowId()), TestWorkflowImpl.class); + + List bodies = emittedBodies(); + assertEquals(bodies.toString(), 2, bodies.size()); + assertTrue(bodies.toString(), bodies.contains(RUN_BODY)); + assertTrue(bodies.toString(), bodies.contains(QUERY_BODY)); + } + + private static List emittedBodies() { + List bodies = new ArrayList<>(); + for (LogRecordData log : emittedLogs()) { + bodies.add(log.getBodyValue().asString()); + } + return bodies; + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java new file mode 100644 index 0000000000..0b5e6cdcff --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/MetricsTest.java @@ -0,0 +1,122 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableSet; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.sdk.metrics.data.LongPointData; +import io.temporal.activity.ActivityInterface; +import io.temporal.activity.ActivityMethod; +import io.temporal.activity.ActivityOptions; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowStub; +import io.temporal.testing.WorkflowReplayer; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import io.temporal.workflow.SignalMethod; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInterface; +import io.temporal.workflow.WorkflowMethod; +import io.temporal.workflow.unsafe.WorkflowUnsafe; +import java.time.Duration; +import java.util.Collection; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.junit.Rule; +import org.junit.Test; + +/** + * Verifies that workflow and activity metrics record during live execution and that replaying a + * workflow run does not emit duplicate metrics. + */ +public class MetricsTest extends OtelTestBase { + private static final String METER_NAME = "custom-metrics"; + private static final String WORKFLOW_COUNTER = "custom_workflow_counter"; + private static final String ACTIVITY_COUNTER = "custom_activity_counter"; + private static final Set liveEnabled = ConcurrentHashMap.newKeySet(); + private static final Set replayEnabled = ConcurrentHashMap.newKeySet(); + + @ActivityInterface + public interface TestActivity { + @ActivityMethod + void doActivity(); + } + + public static class TestActivityImpl implements TestActivity { + @Override + public void doActivity() { + GlobalOpenTelemetry.getMeter(METER_NAME).counterBuilder(ACTIVITY_COUNTER).build().add(1); + } + } + + @WorkflowInterface + public interface TestWorkflow { + @WorkflowMethod + void run(); + + @SignalMethod + void proceed(); + } + + public static class TestWorkflowImpl implements TestWorkflow { + private boolean proceed; + + @Override + public void run() { + LongCounter counter = + GlobalOpenTelemetry.getMeter(METER_NAME).counterBuilder(WORKFLOW_COUNTER).build(); + (WorkflowUnsafe.isReplaying() ? replayEnabled : liveEnabled).add(counter.isEnabled()); + counter.add(1); + + TestActivity activity = + Workflow.newActivityStub( + TestActivity.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(5)).build()); + activity.doActivity(); + + Workflow.await(() -> proceed); + } + + @Override + public void proceed() { + proceed = true; + } + } + + @Rule + public SDKTestWorkflowRule testWorkflowRule = + newRuleBuilder(false) + .setWorkflowTypes(TestWorkflowImpl.class) + .setActivityImplementations(new TestActivityImpl()) + .build(); + + @Test + public void liveExecutionRecordsAndReplayDoesNotDuplicate() throws Exception { + liveEnabled.clear(); + replayEnabled.clear(); + TestWorkflow workflow = testWorkflowRule.newWorkflowStub(TestWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run); + + workflow.proceed(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + assertEquals(1, singleLongValue(WORKFLOW_COUNTER)); + assertEquals(1, singleLongValue(ACTIVITY_COUNTER)); + + // Replay the full workflow history and verify workflow metrics are suppressed on replay + WorkflowReplayer.replayWorkflowExecution( + testWorkflowRule.getExecutionHistory(execution.getWorkflowId()), TestWorkflowImpl.class); + + assertEquals(1, singleLongValue(WORKFLOW_COUNTER)); + assertEquals(1, singleLongValue(ACTIVITY_COUNTER)); + assertEquals(ImmutableSet.of(true), liveEnabled); + assertEquals(ImmutableSet.of(true), replayEnabled); + } + + private static long singleLongValue(String name) { + Collection points = requireMetricNamed(name).getLongSumData().getPoints(); + assertEquals(name + " points: " + points, 1, points.size()); + return points.iterator().next().getValue(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OpenTelemetryPluginGlobalTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OpenTelemetryPluginGlobalTest.java new file mode 100644 index 0000000000..384f4c9c35 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OpenTelemetryPluginGlobalTest.java @@ -0,0 +1,44 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.ContextStorage; +import io.temporal.opentelemetry.v2.internal.TemporalContextStorage; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class OpenTelemetryPluginGlobalTest { + @Before + @After + public void resetGlobalOpenTelemetry() { + GlobalOpenTelemetry.resetForTest(); + } + + @Test + public void buildRejectsAnUnregisteredGlobal() { + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> OpenTelemetryPlugin.newBuilder().build()); + assertTrue(e.getMessage(), e.getMessage().contains("ReplaySafeOpenTelemetry")); + } + + @Test + public void buildWithAnUnregisteredGlobalStillAllowsRegistration() { + assertThrows(IllegalStateException.class, () -> OpenTelemetryPlugin.newBuilder().build()); + try (ReplaySafeOpenTelemetry openTelemetry = ReplaySafeOpenTelemetry.newBuilder().build()) { + GlobalOpenTelemetry.set(openTelemetry); + OpenTelemetryPlugin.newBuilder().build(); + } + } + + @Test + public void buildAcceptsAReplaySafeGlobal() { + try (ReplaySafeOpenTelemetry openTelemetry = ReplaySafeOpenTelemetry.newBuilder().build()) { + GlobalOpenTelemetry.set(openTelemetry); + OpenTelemetryPlugin.newBuilder().build(); + } + assertTrue(ContextStorage.get() instanceof TemporalContextStorage); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java new file mode 100644 index 0000000000..48e10720bf --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/OtelTestBase.java @@ -0,0 +1,178 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.instrumentation.logback.appender.v1_0.OpenTelemetryAppender; +import io.opentelemetry.sdk.logs.SdkLoggerProvider; +import io.opentelemetry.sdk.logs.data.LogRecordData; +import io.opentelemetry.sdk.logs.export.SimpleLogRecordProcessor; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryLogRecordExporter; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import io.temporal.serviceclient.WorkflowServiceStubsOptions; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +public abstract class OtelTestBase { + static final InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + static final InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + static final InMemoryLogRecordExporter logExporter = InMemoryLogRecordExporter.create(); + private static ReplaySafeOpenTelemetry openTelemetry; + + @BeforeClass + public static void registerGlobalOpenTelemetry() { + openTelemetry = + ReplaySafeOpenTelemetry.newBuilder() + .setTracerProviderBuilder( + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter))) + .setMeterProviderBuilder(SdkMeterProvider.builder().registerMetricReader(metricReader)) + .setLoggerProviderBuilder( + SdkLoggerProvider.builder() + .addLogRecordProcessor(SimpleLogRecordProcessor.create(logExporter))) + .build(); + GlobalOpenTelemetry.set(openTelemetry); + OpenTelemetryAppender.install(openTelemetry); + } + + @AfterClass + public static void resetGlobalOpenTelemetry() { + GlobalOpenTelemetry.resetForTest(); + openTelemetry.close(); + } + + @Before + public void clearSpansAndLogs() { + spanExporter.reset(); + logExporter.reset(); + } + + /** + * A rule whose service stubs carry the plugin, so it propagates to every client and the worker. + */ + static SDKTestWorkflowRule.Builder newRuleBuilder(boolean addTemporalSpans) { + return SDKTestWorkflowRule.newBuilder() + .setWorkflowServiceStubsOptions( + WorkflowServiceStubsOptions.newBuilder() + .setPlugins( + OpenTelemetryPlugin.newBuilder().setAddTemporalSpans(addTemporalSpans).build()) + .build()); + } + + static List endedSpans() { + return spanExporter.getFinishedSpanItems(); + } + + static List emittedLogs() { + return logExporter.getFinishedLogRecordItems(); + } + + static SpanData requireSpanNamed(List spans, String name) { + for (SpanData span : spans) { + if (span.getName().equals(name)) { + return span; + } + } + fail(name + " span not found in " + spanTree(spans)); + return null; + } + + static MetricData requireMetricNamed(String name) { + for (MetricData metric : metricReader.collectAllMetrics()) { + if (metric.getName().equals(name)) { + return metric; + } + } + fail(name + " metric not found"); + return null; + } + + static String requireSpanAttribute(SpanData span, AttributeKey key) { + String value = span.getAttributes().get(key); + assertNotNull(key.getKey() + " attribute not found on " + span.getName(), value); + return value; + } + + static void requireUniqueSpanIds(List spans) { + Map namesById = new HashMap<>(); + for (SpanData span : spans) { + String previous = namesById.put(span.getSpanId(), span.getName()); + if (previous != null) { + fail("span " + span.getName() + " shares an ID with span " + previous); + } + } + } + + static void assertSpanTree(List expected, List spans) { + assertEquals(String.join("\n", expected), String.join("\n", spanTree(spans))); + } + + /** Returns the spans as an indented tree in end order. */ + static List spanTree(List spans) { + Map> childrenByParent = new HashMap<>(); + for (int child = 0; child < spans.size(); child++) { + childrenByParent + .computeIfAbsent(closestParentIndex(spans, spans.get(child)), k -> new ArrayList<>()) + .add(child); + } + List tree = new ArrayList<>(); + appendChildren(spans, childrenByParent, -1, 0, tree); + return tree; + } + + /** + * Resets can emit the same span ID and start time more than once; the nearest end time identifies + * the matching parent. + */ + private static int closestParentIndex(List spans, SpanData child) { + String parentId = child.getParentSpanContext().getSpanId(); + int closest = -1; + long closestDistance = Long.MAX_VALUE; + for (int i = 0; i < spans.size(); i++) { + if (!spans.get(i).getSpanId().equals(parentId)) { + continue; + } + long distance = Math.abs(child.getEndEpochNanos() - spans.get(i).getEndEpochNanos()); + if (distance < closestDistance) { + closest = i; + closestDistance = distance; + } + } + return closest; + } + + private static void appendChildren( + List spans, + Map> childrenByParent, + int parent, + int depth, + List tree) { + for (int child : childrenByParent.getOrDefault(parent, new ArrayList<>())) { + tree.add(repeat(" ", depth) + spans.get(child).getName()); + appendChildren(spans, childrenByParent, child, depth + 1, tree); + } + } + + private static String repeat(String s, int times) { + StringBuilder out = new StringBuilder(); + for (int i = 0; i < times; i++) { + out.append(s); + } + return out.toString(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/PropagationTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/PropagationTest.java new file mode 100644 index 0000000000..9502e51e40 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/PropagationTest.java @@ -0,0 +1,569 @@ +package io.temporal.opentelemetry.v2; + +import static io.temporal.opentelemetry.v2.TestWorkflows.NEXUS_CANCEL_OPERATION_NAME; +import static io.temporal.opentelemetry.v2.TestWorkflows.NEXUS_OPERATION_NAME; +import static io.temporal.opentelemetry.v2.TestWorkflows.NEXUS_SERVICE_NAME; +import static io.temporal.opentelemetry.v2.TestWorkflows.TASK_TOKENS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assume.assumeTrue; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.temporal.api.enums.v1.WorkflowIdConflictPolicy; +import io.temporal.client.StartActivityOptions; +import io.temporal.client.StartNexusOperationOptions; +import io.temporal.client.UpdateOptions; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.client.WorkflowUpdateStage; +import io.temporal.client.schedules.Schedule; +import io.temporal.client.schedules.ScheduleActionStartWorkflow; +import io.temporal.client.schedules.ScheduleClient; +import io.temporal.client.schedules.ScheduleClientOptions; +import io.temporal.client.schedules.ScheduleHandle; +import io.temporal.client.schedules.ScheduleOptions; +import io.temporal.client.schedules.ScheduleSpec; +import io.temporal.client.schedules.ScheduleUpdate; +import io.temporal.common.interceptors.Header; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncCompletionWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncCompletionWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncLambdaWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.AsyncLambdaWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ChildWorkflowWithSignalImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ComprehensiveNexusServiceImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ComprehensiveWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.ComprehensiveWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ConstructorSpanWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.ConstructorSpanWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ContinueAsNewToDifferentWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.ContinueAsNewToDifferentWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.DifferentWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.ExternalWorkflowWithSignal; +import io.temporal.opentelemetry.v2.TestWorkflows.ExternalWorkflowWithSignalImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.NexusCancelHandlerWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.NexusHandlerWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.SchedulePropagationReceiver; +import io.temporal.opentelemetry.v2.TestWorkflows.SchedulePropagationReceiverImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.SchedulePropagationWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.SignalWithStartTargetImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.StandaloneWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.StandaloneWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TestActivities; +import io.temporal.opentelemetry.v2.TestWorkflows.TestActivitiesImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.UnservedWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.UpdateTargetWorkflowImpl; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Covers one scenario touching every traced operation, asserted as a span tree with Temporal spans + * on and off. + * + *

Needs a real server: client-started Nexus operations, standalone activities, and schedules are + * not implemented by the in-memory test server. CI runs it in the dev-server job. + */ +@RunWith(Parameterized.class) +public class PropagationTest extends OtelTestBase { + private static final String COMPREHENSIVE_UPDATE_ID = "comprehensive-update"; + private static final String UPDATE_WITH_START_UPDATE_ID = "comprehensive-update-with-start"; + private static final String UNSERVED_TASK_QUEUE = "opentelemetry-v2-unserved"; + private static final String TERMINATE_REASON = "otel-terminate-reason"; + + @Parameters(name = "addTemporalSpans={0}") + public static List addTemporalSpans() { + return Arrays.asList(true, false); + } + + private final boolean addTemporalSpans; + + private final String runId = UUID.randomUUID().toString(); + private final String scheduleId = "otel-schedule-" + runId; + private final String reusedScheduleSeedId = "otel-reused-schedule-seed-" + runId; + private final String reusedScheduleId = "otel-reused-schedule-" + runId; + private final String reusedScheduleWorkflowId = "otel-reused-schedule-workflow-" + runId; + private final String reusedScheduleReceiverId = "otel-reused-schedule-receiver-" + runId; + private final String externalWorkflowId = "externalWorkflowWithSignal-" + runId; + private final String cancelTargetId = "externalCancelTarget-" + runId; + private final String comprehensiveWorkflowId = "comprehensive-outbound-" + runId; + private final String updateWithStartWorkflowId = "otel-update-with-start-" + runId; + + @Rule public SDKTestWorkflowRule testWorkflowRule; + + public PropagationTest(boolean addTemporalSpans) { + this.addTemporalSpans = addTemporalSpans; + this.testWorkflowRule = + newRuleBuilder(addTemporalSpans) + .setWorkflowTypes( + ComprehensiveWorkflowImpl.class, + ContinueAsNewToDifferentWorkflowImpl.class, + DifferentWorkflowImpl.class, + ChildWorkflowWithSignalImpl.class, + ExternalWorkflowWithSignalImpl.class, + NexusHandlerWorkflowImpl.class, + NexusCancelHandlerWorkflowImpl.class, + StandaloneWorkflowImpl.class, + SchedulePropagationWorkflowImpl.class, + SchedulePropagationReceiverImpl.class, + SignalWithStartTargetImpl.class, + UpdateTargetWorkflowImpl.class, + AsyncCompletionWorkflowImpl.class, + UnservedWorkflowImpl.class, + AsyncLambdaWorkflowImpl.class, + ConstructorSpanWorkflowImpl.class) + .setActivityImplementations(new TestActivitiesImpl()) + .setNexusServiceImplementation(new ComprehensiveNexusServiceImpl()) + .build(); + } + + @Before + public void requireRealServer() { + assumeTrue( + "Test Server doesn't support client Nexus operations, standalone activities, or schedules", + SDKTestWorkflowRule.useExternalService); + } + + @Test + public void comprehensive() { + List spans = runScenario(); + assertSpanTree(addTemporalSpans ? fullTree() : noTemporalSpansTree(), spans); + requireUniqueSpanIds(spans); + } + + /** Drives every traced operation under one client span and returns the ended spans. */ + private List runScenario() { + WorkflowClient client = testWorkflowRule.getWorkflowClient(); + String taskQueue = testWorkflowRule.getTaskQueue(); + String nexusEndpoint = testWorkflowRule.getNexusEndpoint().getSpec().getName(); + + // All client calls share this parent span. + Span clientSpan = + GlobalOpenTelemetry.getTracer("client").spanBuilder("client-span").startSpan(); + try (Scope ignored = clientSpan.makeCurrent()) { + // Start the external signal target first. + ExternalWorkflowWithSignal external = + client.newWorkflowStub( + ExternalWorkflowWithSignal.class, options(taskQueue, externalWorkflowId)); + WorkflowClient.start(external::run); + + WorkflowStub cancelCallbackTarget = + client.newUntypedWorkflowStub( + "UnservedWorkflow", options(UNSERVED_TASK_QUEUE, cancelTargetId)); + cancelCallbackTarget.start(); + + client + .newWorkflowStub( + AsyncLambdaWorkflow.class, + options(taskQueue, "otel-async-lambda-" + UUID.randomUUID())) + .run(); + client + .newWorkflowStub( + ConstructorSpanWorkflow.class, + options(taskQueue, "otel-constructor-span-" + UUID.randomUUID())) + .run(); + + client + .newWorkflowStub( + ContinueAsNewToDifferentWorkflow.class, + options(taskQueue, "otel-continue-as-new-" + UUID.randomUUID())) + .run(); + + ComprehensiveWorkflow comprehensive = + client.newWorkflowStub( + ComprehensiveWorkflow.class, options(taskQueue, comprehensiveWorkflowId)); + WorkflowClient.start( + comprehensive::run, false, nexusEndpoint, externalWorkflowId, cancelTargetId); + WorkflowStub comprehensiveStub = WorkflowStub.fromTyped(comprehensive); + + comprehensiveStub + .startUpdate( + UpdateOptions.newBuilder(Void.class) + .setUpdateName("testUpdate") + .setUpdateId(COMPREHENSIVE_UPDATE_ID) + .setWaitForStage(WorkflowUpdateStage.COMPLETED) + .build()) + .getResult(); + assertEquals("ok", comprehensive.getStatus()); + comprehensive.proceed(); + comprehensiveStub.getResult(Void.class); + assertEquals("ok", comprehensive.getStatus()); + + testWorkflowRule + .getActivityClient() + .execute( + TestActivities.class, + TestActivities::standaloneActivity, + StartActivityOptions.newBuilder() + .setId("otel-standalone-activity-" + UUID.randomUUID()) + .setTaskQueue(taskQueue) + .setStartToCloseTimeout(Duration.ofSeconds(10)) + .build()); + + client + .newWorkflowStub( + StandaloneWorkflow.class, + options(taskQueue, "otel-standalone-workflow-" + UUID.randomUUID())) + .run(null); + + AsyncCompletionWorkflow asyncCompletion = + client.newWorkflowStub( + AsyncCompletionWorkflow.class, + options(taskQueue, "otel-async-completion-" + UUID.randomUUID())); + WorkflowClient.start(asyncCompletion::run); + byte[] taskToken = takeTaskToken(); + client.newActivityCompletionClient().complete(taskToken, null); + WorkflowStub.fromTyped(asyncCompletion).getResult(Void.class); + + WorkflowStub cancelTarget = + client.newUntypedWorkflowStub( + "UnservedWorkflow", + options(UNSERVED_TASK_QUEUE, "otel-cancel-target-" + UUID.randomUUID())); + cancelTarget.start(); + cancelTarget.cancel(); + + WorkflowStub terminateTarget = + client.newUntypedWorkflowStub( + "UnservedWorkflow", + options(UNSERVED_TASK_QUEUE, "otel-terminate-target-" + UUID.randomUUID())); + terminateTarget.start(); + terminateTarget.terminate(TERMINATE_REASON); + + comprehensiveStub.describe(); + + // A protobuf-backed header is immutable, so create and update must not write into it. + ScheduleActionStartWorkflow scheduleAction = + ScheduleActionStartWorkflow.newBuilder() + .setWorkflowType("StandaloneWorkflow") + .setHeader(new Header(io.temporal.api.common.v1.Header.getDefaultInstance())) + .setOptions(options(taskQueue, "otel-schedule-workflow-" + UUID.randomUUID())) + .build(); + ScheduleClient scheduleClient = + ScheduleClient.newInstance( + testWorkflowRule.getWorkflowServiceStubs(), + ScheduleClientOptions.newBuilder() + .setNamespace(client.getOptions().getNamespace()) + .build()); + ScheduleHandle schedule = + scheduleClient.createSchedule( + scheduleId, + Schedule.newBuilder() + .setAction(scheduleAction) + .setSpec(ScheduleSpec.newBuilder().build()) + .build(), + ScheduleOptions.newBuilder().build()); + try { + String updatedScheduleWorkflowId = "otel-schedule-updated-workflow-" + UUID.randomUUID(); + try (Scope rootScope = Context.root().makeCurrent()) { + schedule.update( + input -> + new ScheduleUpdate( + Schedule.newBuilder(input.getDescription().getSchedule()) + .setAction( + ScheduleActionStartWorkflow.newBuilder(scheduleAction) + .setOptions(options(taskQueue, updatedScheduleWorkflowId)) + .setArguments(externalWorkflowId) + .build()) + .build())); + } + schedule.trigger(); + String scheduledWorkflowId = WorkflowStub.fromTyped(external).getResult(String.class); + client.newUntypedWorkflowStub(scheduledWorkflowId).getResult(Void.class); + + // Reusing this action verifies a root-context create cannot retain the first create's + // trace. + SchedulePropagationReceiver reusedScheduleReceiver = + client.newWorkflowStub( + SchedulePropagationReceiver.class, options(taskQueue, reusedScheduleReceiverId)); + WorkflowClient.start(reusedScheduleReceiver::run); + ScheduleActionStartWorkflow reusedScheduleAction = + ScheduleActionStartWorkflow.newBuilder() + .setWorkflowType("SchedulePropagationWorkflow") + .setOptions(options(taskQueue, reusedScheduleWorkflowId)) + .setArguments(reusedScheduleReceiverId) + .build(); + ScheduleHandle reusedScheduleSeed = + scheduleClient.createSchedule( + reusedScheduleSeedId, + Schedule.newBuilder() + .setAction(reusedScheduleAction) + .setSpec(ScheduleSpec.newBuilder().build()) + .build(), + ScheduleOptions.newBuilder().build()); + ScheduleHandle reusedSchedule = null; + try { + try (Scope rootScope = Context.root().makeCurrent()) { + reusedSchedule = + scheduleClient.createSchedule( + reusedScheduleId, + Schedule.newBuilder() + .setAction(reusedScheduleAction) + .setSpec(ScheduleSpec.newBuilder().build()) + .build(), + ScheduleOptions.newBuilder().build()); + } + reusedSchedule.trigger(); + String reusedScheduledWorkflowId = + WorkflowStub.fromTyped(reusedScheduleReceiver).getResult(String.class); + client.newUntypedWorkflowStub(reusedScheduledWorkflowId).getResult(Void.class); + } finally { + reusedScheduleSeed.delete(); + if (reusedSchedule != null) { + reusedSchedule.delete(); + } + } + + WorkflowStub signalWithStart = + client.newUntypedWorkflowStub( + "SignalWithStartTarget", + options(taskQueue, "otel-signal-with-start-" + UUID.randomUUID())); + signalWithStart.signalWithStart("startSignal", new Object[0], new Object[0]); + signalWithStart.getResult(Void.class); + + WorkflowStub updateWithStart = + client.newUntypedWorkflowStub( + "UpdateTargetWorkflow", + options(taskQueue, updateWithStartWorkflowId).toBuilder() + .setWorkflowIdConflictPolicy( + WorkflowIdConflictPolicy.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING) + .build()); + updateWithStart + .startUpdateWithStart( + UpdateOptions.newBuilder(Void.class) + .setUpdateName("doUpdate") + .setUpdateId(UPDATE_WITH_START_UPDATE_ID) + .setWaitForStage(WorkflowUpdateStage.COMPLETED) + .build(), + new Object[0], + new Object[0]) + .getResult(); + updateWithStart.signal("updateSignal"); + updateWithStart.getResult(Void.class); + + testWorkflowRule + .getNexusClient() + .newUntypedNexusServiceClient(nexusEndpoint, NEXUS_SERVICE_NAME) + .execute( + NEXUS_OPERATION_NAME, + Void.class, + StartNexusOperationOptions.newBuilder() + .setId("otel-nexus-operation-" + UUID.randomUUID()) + .setScheduleToCloseTimeout(Duration.ofSeconds(10)) + .build(), + ""); + } finally { + schedule.delete(); + } + } finally { + clientSpan.end(); + } + return endedSpans(); + } + + private static WorkflowOptions options(String taskQueue, String workflowId) { + return WorkflowOptions.newBuilder().setTaskQueue(taskQueue).setWorkflowId(workflowId).build(); + } + + private static byte[] takeTaskToken() { + try { + byte[] taskToken = TASK_TOKENS.poll(30, TimeUnit.SECONDS); + assertNotNull("timed out waiting for activity task token", taskToken); + return taskToken; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + + private static String nexusOp(String operation) { + return NEXUS_SERVICE_NAME + "/" + operation; + } + + /** The span tree with Temporal spans enabled. */ + private List fullTree() { + return Arrays.asList( + // The scheduled workflow finishes before the client span, so its root appears first. + "UpdateSchedule:" + scheduleId, + " RunWorkflow:StandaloneWorkflow", + " standalone-workflow-span", + " SignalExternalWorkflow:scheduleStarted", + " HandleSignal:scheduleStarted", + "CreateSchedule:" + reusedScheduleId, + " RunWorkflow:SchedulePropagationWorkflow", + " schedule-propagation-workflow-span", + " SignalExternalWorkflow:scheduleStarted", + " HandleSignal:scheduleStarted", + "client-span", + " StartWorkflow:ExternalWorkflowWithSignal", + " RunWorkflow:ExternalWorkflowWithSignal", + " external-workflow-with-signal-span", + " StartWorkflow:UnservedWorkflow", + " StartWorkflow:AsyncLambdaWorkflow", + " RunWorkflow:AsyncLambdaWorkflow", + " parent", + " child", + " StartActivity:NopActivity", + " RunActivity:NopActivity", + " StartWorkflow:ConstructorSpanWorkflow", + " constructor", + " RunWorkflow:ConstructorSpanWorkflow", + " StartWorkflow:ContinueAsNewToDifferentWorkflow", + " RunWorkflow:ContinueAsNewToDifferentWorkflow", + " ContinueAsNew:DifferentWorkflow", + " RunWorkflow:DifferentWorkflow", + " StartWorkflow:ComprehensiveWorkflow", + " RunWorkflow:ComprehensiveWorkflow", + " StartActivity:Activity", + " RunActivity:Activity", + " activity-span", + " StartActivity:LocalActivity", + " RunActivity:LocalActivity", + " local-activity-span", + " timer-registration-span", + " timer-callback-span", + " StartChildWorkflow:ChildWorkflowWithSignal", + " RunWorkflow:ChildWorkflowWithSignal", + " child-workflow-with-signal-span", + " promise-registration-span", + " promise-callback-span", + // There is no SignalChildWorkflow outbound method; child signals go through + // signalExternalWorkflow, so this span is named for that method. + " SignalExternalWorkflow:childSignal", + " HandleSignal:childSignal", + " SignalExternalWorkflow:externalSignal", + " HandleSignal:externalSignal", + " cancel-registration-span", + " CancelWorkflow", + " cancel-callback-span", + " StartNexusOperation:" + nexusOp(NEXUS_OPERATION_NAME), + " RunStartNexusOperationHandler:" + nexusOp(NEXUS_OPERATION_NAME), + " StartWorkflow:NexusHandlerWorkflow", + " RunWorkflow:NexusHandlerWorkflow", + " workflow-with-nexus-handler-span", + " StartNexusOperation:" + nexusOp(NEXUS_CANCEL_OPERATION_NAME), + " RunStartNexusOperationHandler:" + nexusOp(NEXUS_CANCEL_OPERATION_NAME), + " StartWorkflow:NexusCancelHandlerWorkflow", + " RunWorkflow:NexusCancelHandlerWorkflow", + " nexus-cancel-handler-span", + " RunCancelNexusOperationHandler:" + nexusOp(NEXUS_CANCEL_OPERATION_NAME), + " CancelWorkflow", + // Continue-as-new links the outbound, continued-run, and user spans. + " ContinueAsNew:ComprehensiveWorkflow", + " RunWorkflow:ComprehensiveWorkflow", + " comprehensive-outbound-workflow-span", + " comprehensive-outbound-workflow-span", + // Update user spans follow their current inbound operation. + " StartWorkflowUpdate:testUpdate", + " ValidateUpdate:testUpdate", + " validate-update-span", + " validate-update-span-child", + " HandleUpdate:testUpdate", + " update-handler-span", + " update-handler-child-span", + // Query handler spans parent under the query that ran them. + " QueryWorkflow:getStatus", + " HandleQuery:getStatus", + " query-handler-span", + " query-handler-child-span", + " SignalWorkflow:proceed", + " HandleSignal:proceed", + // The query after completion replays the run, but its handler spans are still exported. + " QueryWorkflow:getStatus", + " HandleQuery:getStatus", + " query-handler-span", + " query-handler-child-span", + // Headers link standalone StartActivity and RunActivity spans. + " StartActivity:StandaloneActivity", + " RunActivity:StandaloneActivity", + " StartWorkflow:StandaloneWorkflow", + " RunWorkflow:StandaloneWorkflow", + " StartWorkflow:AsyncCompletionWorkflow", + " RunWorkflow:AsyncCompletionWorkflow", + " StartActivity:AsyncCompletionActivity", + " RunActivity:AsyncCompletionActivity", + // Cancel, terminate, and describe have no propagation carrier but are still traced. + " StartWorkflow:UnservedWorkflow", + " CancelWorkflow", + " StartWorkflow:UnservedWorkflow", + " TerminateWorkflow", + " DescribeWorkflow", + " CreateSchedule:" + scheduleId, + " StartWorkflow:SchedulePropagationReceiver", + " RunWorkflow:SchedulePropagationReceiver", + " CreateSchedule:" + reusedScheduleSeedId, + // Signal-with-start links client, worker, signal, and user spans. + " SignalWithStartWorkflow:SignalWithStartTarget", + " HandleSignal:startSignal", + " RunWorkflow:SignalWithStartTarget", + " signal-with-start-target-span", + // Update-with-start links validation, execution, worker, and user spans. + " UpdateWithStartWorkflow:doUpdate", + " ValidateUpdate:doUpdate", + " HandleUpdate:doUpdate", + " update start", + " RunWorkflow:UpdateTargetWorkflow", + " update-target-workflow-span", + " SignalWorkflow:updateSignal", + " HandleSignal:updateSignal", + " StartNexusOperation:" + nexusOp(NEXUS_OPERATION_NAME), + " RunStartNexusOperationHandler:" + nexusOp(NEXUS_OPERATION_NAME), + " StartWorkflow:NexusHandlerWorkflow", + " RunWorkflow:NexusHandlerWorkflow", + " workflow-with-nexus-handler-span"); + } + + /** + * {@link #fullTree()} without any Temporal spans. Only user spans remain, and each reattaches to + * the nearest surviving ancestor. + */ + private static List noTemporalSpansTree() { + return Arrays.asList( + "standalone-workflow-span", + "schedule-propagation-workflow-span", + "client-span", + " parent", + " child", + " constructor", + " validate-update-span", + " validate-update-span-child", + " update-handler-span", + " update-handler-child-span", + " query-handler-span", + " query-handler-child-span", + " activity-span", + " local-activity-span", + " timer-registration-span", + " timer-callback-span", + // The child span ends after its signal, which follows the promise spans. + " promise-registration-span", + " promise-callback-span", + " child-workflow-with-signal-span", + " cancel-registration-span", + " cancel-callback-span", + " workflow-with-nexus-handler-span", + " nexus-cancel-handler-span", + // Continue-as-new emits the user span once per run. + " comprehensive-outbound-workflow-span", + " comprehensive-outbound-workflow-span", + " query-handler-span", + " query-handler-child-span", + " external-workflow-with-signal-span", + " signal-with-start-target-span", + " update start", + " update-target-workflow-span", + " workflow-with-nexus-handler-span"); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetryTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetryTest.java new file mode 100644 index 0000000000..722af201bc --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/ReplaySafeOpenTelemetryTest.java @@ -0,0 +1,19 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertThrows; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.context.Context; +import org.junit.Test; + +public class ReplaySafeOpenTelemetryTest { + @Test + public void pluginRejectsContextUsedBeforeReplaySafeOpenTelemetry() { + Context.current(); + + try (ReplaySafeOpenTelemetry openTelemetry = ReplaySafeOpenTelemetry.newBuilder().build()) { + GlobalOpenTelemetry.set(openTelemetry); + assertThrows(IllegalStateException.class, () -> OpenTelemetryPlugin.newBuilder().build()); + } + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TestWorkflows.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TestWorkflows.java new file mode 100644 index 0000000000..c052ea36ca --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TestWorkflows.java @@ -0,0 +1,1045 @@ +package io.temporal.opentelemetry.v2; + +import io.nexusrpc.Operation; +import io.nexusrpc.Service; +import io.nexusrpc.handler.OperationHandler; +import io.nexusrpc.handler.OperationImpl; +import io.nexusrpc.handler.ServiceImpl; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import io.temporal.activity.Activity; +import io.temporal.activity.ActivityInterface; +import io.temporal.activity.ActivityOptions; +import io.temporal.activity.LocalActivityOptions; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.client.WorkflowOptions; +import io.temporal.failure.ApplicationErrorCategory; +import io.temporal.failure.ApplicationFailure; +import io.temporal.failure.CanceledFailure; +import io.temporal.failure.NexusOperationFailure; +import io.temporal.nexus.Nexus; +import io.temporal.nexus.WorkflowRunOperation; +import io.temporal.workflow.ActivityInvocationOptions; +import io.temporal.workflow.Async; +import io.temporal.workflow.CancellationScope; +import io.temporal.workflow.NexusOperationHandle; +import io.temporal.workflow.NexusOperationOptions; +import io.temporal.workflow.NexusServiceOptions; +import io.temporal.workflow.Promise; +import io.temporal.workflow.QueryMethod; +import io.temporal.workflow.SignalMethod; +import io.temporal.workflow.UpdateMethod; +import io.temporal.workflow.UpdateValidatorMethod; +import io.temporal.workflow.Workflow; +import io.temporal.workflow.WorkflowInterface; +import io.temporal.workflow.WorkflowMethod; +import java.time.Duration; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** Workflow and activity fixtures used by the OpenTelemetry v2 tests. */ +public final class TestWorkflows { + private TestWorkflows() {} + + static final String TRACER_TEST_QUERY_NAME = "query"; + static final String TRACER_TEST_UPDATE_NAME = "update"; + static final String TRACER_TEST_SIGNAL_NAME = "gate"; + static final String NEXUS_SERVICE_NAME = "ComprehensiveNexusService"; + static final String NEXUS_OPERATION_NAME = "nexusHandlerWorkflow"; + static final String NEXUS_CANCEL_OPERATION_NAME = "nexusCancelHandlerWorkflow"; + + /** Stands in for the external system an async activity hands its task token to. */ + static final BlockingQueue TASK_TOKENS = new LinkedBlockingQueue<>(); + + static Span startSpan(String tracerName, String spanName) { + return GlobalOpenTelemetry.getTracer(tracerName).spanBuilder(spanName).startSpan(); + } + + static void spanAround(String tracerName, String spanName, Runnable body) { + Span span = startSpan(tracerName, spanName); + try (Scope ignored = span.makeCurrent()) { + body.run(); + } finally { + span.end(); + } + } + + // --------------------------------------------------------------------------------------------- + // Activities + // --------------------------------------------------------------------------------------------- + + @ActivityInterface + public interface TestActivities { + void activity(); + + void localActivity(); + + void asyncCompletionActivity(); + + void standaloneActivity(); + + void nopActivity(); + } + + public static class TestActivitiesImpl implements TestActivities { + @Override + public void activity() { + spanAround("activity", "activity-span", () -> {}); + } + + @Override + public void localActivity() { + spanAround("localActivity", "local-activity-span", () -> {}); + } + + @Override + public void asyncCompletionActivity() { + TASK_TOKENS.add(Activity.getExecutionContext().getTaskToken()); + Activity.getExecutionContext().doNotCompleteOnReturn(); + } + + @Override + public void standaloneActivity() {} + + @Override + public void nopActivity() {} + } + + // --------------------------------------------------------------------------------------------- + // Interceptor tests + // --------------------------------------------------------------------------------------------- + + @WorkflowInterface + public interface SpanKindWorkflow { + @WorkflowMethod + void run(); + } + + public static class SpanKindWorkflowImpl implements SpanKindWorkflow { + @Override + public void run() { + activities().nopActivity(); + } + } + + @WorkflowInterface + public interface AsyncLambdaWorkflow { + @WorkflowMethod + void run(); + } + + public static class AsyncLambdaWorkflowImpl implements AsyncLambdaWorkflow { + @Override + public void run() { + Span parent = startSpan("asyncLambda", "parent"); + try (Scope ignored = parent.makeCurrent()) { + Async.function( + () -> { + Span child = startSpan("asyncLambda", "child"); + try (Scope ignoredChild = child.makeCurrent()) { + activities().nopActivity(); + } finally { + child.end(); + } + return null; + }) + .get(); + } finally { + parent.end(); + } + } + } + + @WorkflowInterface + public interface ConstructorSpanWorkflow { + @WorkflowMethod + void run(); + } + + public static class ConstructorSpanWorkflowImpl implements ConstructorSpanWorkflow { + public ConstructorSpanWorkflowImpl() { + startSpan("constructorSpan", "constructor").end(); + } + + @Override + public void run() {} + } + + @WorkflowInterface + public interface PromiseCallbackWorkflow { + @WorkflowMethod + void run(); + } + + public static class PromiseCallbackWorkflowImpl implements PromiseCallbackWorkflow { + @Override + public void run() { + Span application = startSpan("promiseCallback", "application"); + try (Scope ignored = application.makeCurrent()) { + Promise continuation = + Async.procedure(activities()::asyncCompletionActivity) + .thenApply(result -> result) + .thenApply( + ignoredResult -> { + Span callback = startSpan("promiseCallback", "callback"); + try (Scope ignoredCallback = callback.makeCurrent()) { + activities().nopActivity(); + } finally { + callback.end(); + } + return null; + }); + continuation.get(); + } finally { + application.end(); + } + } + } + + @WorkflowInterface + public interface BenignErrorWorkflow { + @WorkflowMethod + void run(); + } + + public static class BenignErrorWorkflowImpl implements BenignErrorWorkflow { + @Override + public void run() { + throw ApplicationFailure.newBuilder() + .setMessage("expected error") + .setType("BenignError") + .setCategory(ApplicationErrorCategory.BENIGN) + .build(); + } + } + + @WorkflowInterface + public interface ErrorWorkflow { + @WorkflowMethod + void run(); + } + + public static class ErrorWorkflowImpl implements ErrorWorkflow { + @Override + public void run() { + throw ApplicationFailure.newFailure("unexpected error", "UnexpectedError"); + } + } + + // --------------------------------------------------------------------------------------------- + // Tracer tests + // --------------------------------------------------------------------------------------------- + + @WorkflowInterface + public interface TracerWorkflow { + @WorkflowMethod + void run(boolean end); + + @QueryMethod(name = TRACER_TEST_QUERY_NAME) + String query(); + + @UpdateValidatorMethod(updateName = TRACER_TEST_UPDATE_NAME) + void validateUpdate(); + + @UpdateMethod(name = TRACER_TEST_UPDATE_NAME) + void update(); + + @SignalMethod(name = TRACER_TEST_SIGNAL_NAME) + void gate(); + } + + public static class TracerWorkflowImpl implements TracerWorkflow { + private boolean gated; + + @Override + public void run(boolean end) { + if (!end) { + Workflow.await(() -> gated); + } + + // Both spans end before continue-as-new, so the next run's context starts from the root. + Span beginProcessing = startSpan("processorTracer", "process start"); + try (Scope ignored = beginProcessing.makeCurrent()) { + startSpan("recorderTracer", "record results").end(); + } finally { + beginProcessing.end(); + } + + if (!end) { + Workflow.continueAsNew(true); + } + } + + @Override + public String query() { + startSpan("queryTracer", "query start").end(); + return "ok"; + } + + @Override + public void update() { + startSpan("updateTracer", "update start").end(); + } + + @Override + public void validateUpdate() { + startSpan("validatorTracer", "validate start").end(); + } + + @Override + public void gate() { + gated = true; + } + } + + public static final String OUTBOUND_ACTIVITY_ID = "outbound-activity"; + public static final String OUTBOUND_LOCAL_ACTIVITY_ID = "outbound-local-activity"; + + public enum WorkflowOutboundCall { + ACTIVITY, + LOCAL_ACTIVITY, + CHILD_WORKFLOW, + EXTERNAL_SIGNAL, + EXTERNAL_CANCEL + } + + @WorkflowInterface + public interface WorkflowOutboundTagsWorkflow { + @WorkflowMethod + void run(WorkflowOutboundCall call, WorkflowExecution targetExecution); + } + + public static class WorkflowOutboundTagsWorkflowImpl implements WorkflowOutboundTagsWorkflow { + @Override + public void run(WorkflowOutboundCall call, WorkflowExecution targetExecution) { + switch (call) { + case ACTIVITY: + Workflow.executeActivity( + activities()::activity, + ActivityInvocationOptions.newBuilder().setActivityId(OUTBOUND_ACTIVITY_ID).build()); + break; + case LOCAL_ACTIVITY: + TestActivities localActivities = + Workflow.newLocalActivityStub( + TestActivities.class, + LocalActivityOptions.newBuilder() + .setStartToCloseTimeout(Duration.ofSeconds(10)) + .build()); + Workflow.executeActivity( + localActivities::localActivity, + ActivityInvocationOptions.newBuilder() + .setActivityId(OUTBOUND_LOCAL_ACTIVITY_ID) + .build()); + break; + case CHILD_WORKFLOW: + ChildWorkflowWithSignal child = + Workflow.newChildWorkflowStub(ChildWorkflowWithSignal.class); + Promise childResult = Async.procedure(child::run); + Workflow.getWorkflowExecution(child).get(); + child.childSignal(); + childResult.get(); + break; + case EXTERNAL_SIGNAL: + Workflow.newUntypedExternalWorkflowStub(targetExecution).signal("childSignal"); + break; + case EXTERNAL_CANCEL: + Workflow.newUntypedExternalWorkflowStub(targetExecution).cancel(); + break; + } + } + } + + @WorkflowInterface + public interface ChainedContinueAsNewWorkflow { + @WorkflowMethod + void run(boolean finalRun); + } + + /** Continues as new inside its user span, so the next run parents under that span. */ + public static class ChainedContinueAsNewWorkflowImpl implements ChainedContinueAsNewWorkflow { + @Override + public void run(boolean finalRun) { + spanAround( + "chainedContinueAsNewWorkflow", + "chained-span", + () -> { + if (!finalRun) { + Workflow.continueAsNew(true); + } + }); + } + } + + @WorkflowInterface + public interface ContinueAsNewToDifferentWorkflow { + @WorkflowMethod + void run(); + } + + public static class ContinueAsNewToDifferentWorkflowImpl + implements ContinueAsNewToDifferentWorkflow { + @Override + public void run() { + Workflow.continueAsNew(DifferentWorkflow.class.getSimpleName(), null); + } + } + + @WorkflowInterface + public interface DifferentWorkflow { + @WorkflowMethod + void run(); + } + + public static class DifferentWorkflowImpl implements DifferentWorkflow { + @Override + public void run() {} + } + + @WorkflowInterface + public interface TracerResetWorkflow { + @WorkflowMethod + void run(); + } + + public static class TracerResetWorkflowImpl implements TracerResetWorkflow { + @Override + public void run() { + // Ended but kept current, so later spans parent to it. + Span beginProcessing = startSpan("processorTracer", "process start"); + try (Scope ignored = beginProcessing.makeCurrent()) { + beginProcessing.end(); + Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()) + .nopActivity(); + startSpan("recorderTracer", "record results").end(); + } + } + } + + @WorkflowInterface + public interface TracerResetLateSourceWorkflow { + @WorkflowMethod + void run(); + } + + /** Obtains the second tracer only after the reset point. */ + public static class TracerResetLateSourceWorkflowImpl implements TracerResetLateSourceWorkflow { + @Override + public void run() { + Span beginProcessing = startSpan("processorTracer", "process start"); + try (Scope ignored = beginProcessing.makeCurrent()) { + beginProcessing.end(); + Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()) + .nopActivity(); + Tracer recorder = GlobalOpenTelemetry.getTracer("recorderTracer"); + recorder.spanBuilder("record results").startSpan().end(); + } + } + } + + @WorkflowInterface + public interface TracerResetDuringSpan { + @WorkflowMethod + void run(); + } + + /** Both spans stay open across the reset point. */ + public static class TracerResetDuringSpanImpl implements TracerResetDuringSpan { + @Override + public void run() { + Span beginProcessing = startSpan("processorTracer", "process start"); + try (Scope ignored = beginProcessing.makeCurrent()) { + Span recordingResults = startSpan("recorderTracer", "record results"); + Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()) + .nopActivity(); + beginProcessing.end(); + recordingResults.end(); + } + } + } + + @WorkflowInterface + public interface TracerWorkflowTaskRetry { + @WorkflowMethod + void run(); + } + + public static class TracerWorkflowTaskRetryImpl implements TracerWorkflowTaskRetry { + @Override + public void run() { + startSpan("test", "workflow-task-retry-span").end(); + throw new RuntimeException("intentional workflow task failure"); + } + } + + @WorkflowInterface + public interface TracerSpanTimestampWorkflow { + @WorkflowMethod + void run(); + + @QueryMethod + String query(); + } + + public static class TracerSpanTimestampWorkflowImpl implements TracerSpanTimestampWorkflow { + @Override + public void run() { + Span span = + GlobalOpenTelemetry.getTracer("timestampTracer") + .spanBuilder("explicit timestamp") + .setStartTimestamp(123456789L, TimeUnit.MILLISECONDS) + .startSpan(); + try { + Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()) + .nopActivity(); + } finally { + span.end(); + } + } + + @Override + public String query() { + return "ok"; + } + } + + @WorkflowInterface + public interface TracerSpanEventWorkflow { + @WorkflowMethod + void run(); + + @QueryMethod + String query(); + } + + public static class TracerSpanEventWorkflowImpl implements TracerSpanEventWorkflow { + @Override + public void run() { + Span span = startSpan("eventTracer", "span with event"); + span.addEvent("event"); + span.end(); + } + + @Override + public String query() { + return "ok"; + } + } + + @WorkflowInterface + public interface TracerSpanAcrossEvictionWorkflow { + @WorkflowMethod + void run(); + + @SignalMethod + void finish(); + } + + public static class TracerSpanAcrossEvictionWorkflowImpl + implements TracerSpanAcrossEvictionWorkflow { + private boolean finished; + + @Override + public void run() { + spanAround("evictionTracer", "span across eviction", () -> Workflow.await(() -> finished)); + } + + @Override + public void finish() { + finished = true; + } + } + + @WorkflowInterface + public interface TracerSpanEndedTwiceWorkflow { + @WorkflowMethod + void run(); + + @SignalMethod + void finish(); + } + + public static class TracerSpanEndedTwiceWorkflowImpl implements TracerSpanEndedTwiceWorkflow { + private boolean finished; + + @Override + public void run() { + Span span = startSpan("endedTwiceTracer", "span ended twice"); + span.end(); + Workflow.await(() -> finished); + span.end(); + } + + @Override + public void finish() { + finished = true; + } + } + + // --------------------------------------------------------------------------------------------- + // Comprehensive scenario + // --------------------------------------------------------------------------------------------- + + @WorkflowInterface + public interface ExternalWorkflowWithSignal { + @WorkflowMethod + String run(); + + @SignalMethod + void externalSignal(); + + @SignalMethod + void scheduleStarted(String workflowId); + } + + public static class ExternalWorkflowWithSignalImpl implements ExternalWorkflowWithSignal { + private boolean externalSignaled; + private String scheduledWorkflowId; + + @Override + public String run() { + spanAround( + "externalWorkflowWithSignal", + "external-workflow-with-signal-span", + () -> Workflow.await(() -> externalSignaled && scheduledWorkflowId != null)); + return scheduledWorkflowId; + } + + @Override + public void externalSignal() { + externalSignaled = true; + } + + @Override + public void scheduleStarted(String workflowId) { + scheduledWorkflowId = workflowId; + } + } + + @WorkflowInterface + public interface ChildWorkflowWithSignal { + @WorkflowMethod + void run(); + + @SignalMethod + void childSignal(); + } + + public static class ChildWorkflowWithSignalImpl implements ChildWorkflowWithSignal { + private boolean signaled; + + @Override + public void run() { + spanAround( + "childWorkflowWithSignal", + "child-workflow-with-signal-span", + () -> Workflow.await(() -> signaled)); + } + + @Override + public void childSignal() { + signaled = true; + } + } + + @WorkflowInterface + public interface NexusHandlerWorkflow { + @WorkflowMethod + Void run(String input); + } + + public static class NexusHandlerWorkflowImpl implements NexusHandlerWorkflow { + @Override + public Void run(String input) { + spanAround("workflowWithNexusHandler", "workflow-with-nexus-handler-span", () -> {}); + return null; + } + } + + @WorkflowInterface + public interface NexusCancelHandlerWorkflow { + @WorkflowMethod + Void run(String input); + } + + /** Waits until the Nexus caller cancels it. */ + public static class NexusCancelHandlerWorkflowImpl implements NexusCancelHandlerWorkflow { + @Override + public Void run(String input) { + spanAround( + "nexusCancelHandlerWorkflow", + "nexus-cancel-handler-span", + () -> Workflow.await(() -> false)); + return null; + } + } + + @Service(name = NEXUS_SERVICE_NAME) + public interface ComprehensiveNexusService { + @Operation(name = NEXUS_OPERATION_NAME) + Void nexusHandlerWorkflow(String input); + + @Operation(name = NEXUS_CANCEL_OPERATION_NAME) + Void nexusCancelHandlerWorkflow(String input); + } + + @ServiceImpl(service = ComprehensiveNexusService.class) + public static class ComprehensiveNexusServiceImpl { + @OperationImpl + public OperationHandler nexusHandlerWorkflow() { + return WorkflowRunOperation.fromWorkflowMethod( + (context, details, input) -> + Nexus.getOperationContext() + .getWorkflowClient() + .newWorkflowStub( + NexusHandlerWorkflow.class, + WorkflowOptions.newBuilder() + .setWorkflowId("nexus-handler-" + details.getRequestId()) + .build()) + ::run); + } + + @OperationImpl + public OperationHandler nexusCancelHandlerWorkflow() { + return WorkflowRunOperation.fromWorkflowMethod( + (context, details, input) -> + Nexus.getOperationContext() + .getWorkflowClient() + .newWorkflowStub( + NexusCancelHandlerWorkflow.class, + WorkflowOptions.newBuilder() + .setWorkflowId("nexus-cancel-handler-" + details.getRequestId()) + .build()) + ::run); + } + } + + @WorkflowInterface + public interface ComprehensiveWorkflow { + @WorkflowMethod + void run( + boolean finalRun, String nexusEndpoint, String externalWorkflowId, String cancelTargetId); + + @QueryMethod(name = "getStatus") + String getStatus(); + + @UpdateMethod(name = "testUpdate") + void testUpdate(); + + @UpdateValidatorMethod(updateName = "testUpdate") + void validateTestUpdate(); + + @SignalMethod + void proceed(); + } + + public static class ComprehensiveWorkflowImpl implements ComprehensiveWorkflow { + private boolean proceed; + + @Override + public void run( + boolean finalRun, String nexusEndpoint, String externalWorkflowId, String cancelTargetId) { + // The returned context is discarded, so the span is not made current. The outbound calls + // below parent to the RunWorkflow span and this span is their sibling. + Span span = startSpan("comprehensiveWorkflow", "comprehensive-outbound-workflow-span"); + try { + if (finalRun) { + return; + } + + Workflow.await(() -> proceed); + + activities().activity(); + Workflow.newLocalActivityStub( + TestActivities.class, + LocalActivityOptions.newBuilder() + .setStartToCloseTimeout(Duration.ofSeconds(10)) + .build()) + .localActivity(); + + Span timerRegistration = startSpan("comprehensiveWorkflow", "timer-registration-span"); + Promise timerContinuation; + try (Scope ignoredRegistration = timerRegistration.makeCurrent()) { + timerContinuation = + Workflow.newTimer(Duration.ofMillis(1)) + .thenApply( + ignoredResult -> { + startSpan("comprehensiveWorkflow", "timer-callback-span").end(); + return null; + }); + } finally { + timerRegistration.end(); + } + timerContinuation.get(); + + ChildWorkflowWithSignal child = + Workflow.newChildWorkflowStub(ChildWorkflowWithSignal.class); + Promise childResult = Async.procedure(child::run); + Span registration = startSpan("comprehensiveWorkflow", "promise-registration-span"); + Promise childContinuation; + try (Scope ignoredRegistration = registration.makeCurrent()) { + childContinuation = + childResult.thenApply( + ignoredResult -> { + startSpan("comprehensiveWorkflow", "promise-callback-span").end(); + return null; + }); + } finally { + registration.end(); + } + Workflow.getWorkflowExecution(child).get(); + child.childSignal(); + childContinuation.get(); + + Workflow.newExternalWorkflowStub(ExternalWorkflowWithSignal.class, externalWorkflowId) + .externalSignal(); + + Span cancelRegistration = startSpan("comprehensiveWorkflow", "cancel-registration-span"); + Promise cancelCallback; + try (Scope ignoredRegistration = cancelRegistration.makeCurrent()) { + cancelCallback = + Async.procedure( + () -> Workflow.newUntypedExternalWorkflowStub(cancelTargetId).cancel()) + .thenApply( + ignoredResult -> { + startSpan("comprehensiveWorkflow", "cancel-callback-span").end(); + return null; + }); + } finally { + cancelRegistration.end(); + } + cancelCallback.get(); + + ComprehensiveNexusService nexus = + Workflow.newNexusServiceStub( + ComprehensiveNexusService.class, + NexusServiceOptions.newBuilder() + .setEndpoint(nexusEndpoint) + .setOperationOptions( + NexusOperationOptions.newBuilder() + .setScheduleToCloseTimeout(Duration.ofSeconds(10)) + .build()) + .build()); + nexus.nexusHandlerWorkflow(""); + + try { + Workflow.newCancellationScope( + () -> { + NexusOperationHandle handle = + Workflow.startNexusOperation(nexus::nexusCancelHandlerWorkflow, ""); + handle.getExecution().get(); + CancellationScope.current().cancel(); + handle.getResult().get(); + }) + .run(); + } catch (NexusOperationFailure failure) { + // Cancellation is expected. + if (!(failure.getCause() instanceof CanceledFailure)) { + throw failure; + } + } + + Workflow.continueAsNew(true, nexusEndpoint, externalWorkflowId, cancelTargetId); + } finally { + span.end(); + } + } + + @Override + public String getStatus() { + spanAround( + "comprehensiveWorkflow", + "query-handler-span", + () -> startSpan("comprehensiveWorkflow", "query-handler-child-span").end()); + return "ok"; + } + + @Override + public void testUpdate() { + spanAround( + "comprehensiveWorkflow", + "update-handler-span", + () -> startSpan("comprehensiveWorkflow", "update-handler-child-span").end()); + } + + @Override + public void validateTestUpdate() { + spanAround( + "comprehensiveWorkflow", + "validate-update-span", + () -> startSpan("comprehensiveWorkflow", "validate-update-span-child").end()); + } + + @Override + public void proceed() { + proceed = true; + } + } + + @WorkflowInterface + public interface StandaloneWorkflow { + @WorkflowMethod + void run(String signalReceiverWorkflowId); + } + + public static class StandaloneWorkflowImpl implements StandaloneWorkflow { + @Override + public void run(String signalReceiverWorkflowId) { + if (signalReceiverWorkflowId != null) { + spanAround( + "standaloneWorkflow", + "standalone-workflow-span", + () -> + Workflow.newExternalWorkflowStub( + ExternalWorkflowWithSignal.class, signalReceiverWorkflowId) + .scheduleStarted(Workflow.getInfo().getWorkflowId())); + } + } + } + + @WorkflowInterface + public interface SchedulePropagationWorkflow { + @WorkflowMethod + void run(String signalReceiverWorkflowId); + } + + public static class SchedulePropagationWorkflowImpl implements SchedulePropagationWorkflow { + @Override + public void run(String signalReceiverWorkflowId) { + spanAround( + "schedulePropagationWorkflow", + "schedule-propagation-workflow-span", + () -> + Workflow.newExternalWorkflowStub( + SchedulePropagationReceiver.class, signalReceiverWorkflowId) + .scheduleStarted(Workflow.getInfo().getWorkflowId())); + } + } + + @WorkflowInterface + public interface SchedulePropagationReceiver { + @WorkflowMethod + String run(); + + @SignalMethod + void scheduleStarted(String workflowId); + } + + public static class SchedulePropagationReceiverImpl implements SchedulePropagationReceiver { + private String scheduledWorkflowId; + + @Override + public String run() { + Workflow.await(() -> scheduledWorkflowId != null); + return scheduledWorkflowId; + } + + @Override + public void scheduleStarted(String workflowId) { + scheduledWorkflowId = workflowId; + } + } + + @WorkflowInterface + public interface UnservedWorkflow { + @WorkflowMethod + void run(); + } + + public static class UnservedWorkflowImpl implements UnservedWorkflow { + @Override + public void run() {} + } + + @WorkflowInterface + public interface SignalWithStartTarget { + @WorkflowMethod + void run(); + + @SignalMethod + void startSignal(); + } + + public static class SignalWithStartTargetImpl implements SignalWithStartTarget { + private boolean signaled; + + @Override + public void run() { + spanAround( + "signalWithStartTarget", + "signal-with-start-target-span", + () -> Workflow.await(() -> signaled)); + } + + @Override + public void startSignal() { + signaled = true; + } + } + + @WorkflowInterface + public interface UpdateTargetWorkflow { + @WorkflowMethod + void run(); + + @UpdateMethod(name = "doUpdate") + void doUpdate(); + + @SignalMethod + void updateSignal(); + } + + public static class UpdateTargetWorkflowImpl implements UpdateTargetWorkflow { + private boolean signaled; + + @Override + public void run() { + spanAround( + "updateTargetWorkflow", + "update-target-workflow-span", + () -> Workflow.await(() -> signaled)); + } + + @Override + public void doUpdate() { + startSpan("updateTracer", "update start").end(); + } + + @Override + public void updateSignal() { + signaled = true; + } + } + + @WorkflowInterface + public interface AsyncCompletionWorkflow { + @WorkflowMethod + void run(); + } + + public static class AsyncCompletionWorkflowImpl implements AsyncCompletionWorkflow { + @Override + public void run() { + Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(15)).build()) + .asyncCompletionActivity(); + } + } + + private static TestActivities activities() { + return Workflow.newActivityStub( + TestActivities.class, + ActivityOptions.newBuilder().setStartToCloseTimeout(Duration.ofSeconds(10)).build()); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TracerTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TracerTest.java new file mode 100644 index 0000000000..5c151be4e0 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/TracerTest.java @@ -0,0 +1,281 @@ +package io.temporal.opentelemetry.v2; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +import io.opentelemetry.sdk.trace.data.SpanData; +import io.temporal.api.common.v1.WorkflowExecution; +import io.temporal.api.enums.v1.EventType; +import io.temporal.api.history.v1.HistoryEvent; +import io.temporal.api.workflowservice.v1.ResetWorkflowExecutionRequest; +import io.temporal.api.workflowservice.v1.ResetWorkflowExecutionResponse; +import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowFailedException; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; +import io.temporal.client.WorkflowTargetOptions; +import io.temporal.common.RetryOptions; +import io.temporal.failure.TimeoutFailure; +import io.temporal.opentelemetry.v2.TestWorkflows.ChainedContinueAsNewWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.ChainedContinueAsNewWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TestActivitiesImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerResetDuringSpanImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerResetLateSourceWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerResetWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanAcrossEvictionWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanAcrossEvictionWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanEndedTwiceWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanEndedTwiceWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanEventWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanEventWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanTimestampWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerSpanTimestampWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflow; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflowImpl; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflowTaskRetry; +import io.temporal.opentelemetry.v2.TestWorkflows.TracerWorkflowTaskRetryImpl; +import io.temporal.testing.internal.SDKTestWorkflowRule; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.junit.Rule; +import org.junit.Test; + +/** + * Verifies that spans started by workflow code through the replay-safe global preserve parentage, + * with Temporal spans disabled. + */ +public class TracerTest extends OtelTestBase { + @Rule + public SDKTestWorkflowRule testWorkflowRule = + newRuleBuilder(false) + .setWorkflowTypes( + TracerWorkflowImpl.class, + ChainedContinueAsNewWorkflowImpl.class, + TracerResetWorkflowImpl.class, + TracerResetLateSourceWorkflowImpl.class, + TracerResetDuringSpanImpl.class, + TracerWorkflowTaskRetryImpl.class, + TracerSpanTimestampWorkflowImpl.class, + TracerSpanEventWorkflowImpl.class, + TracerSpanAcrossEvictionWorkflowImpl.class, + TracerSpanEndedTwiceWorkflowImpl.class) + .setActivityImplementations(new TestActivitiesImpl()) + .build(); + + @Test + public void tracerWorkflow() { + TracerWorkflow workflow = testWorkflowRule.newWorkflowStub(TracerWorkflow.class); + WorkflowClient.start(workflow::run, false); + + workflow.update(); + workflow.query(); + workflow.gate(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + workflow.query(); + + List spans = endedSpans(); + assertSpanTree( + Arrays.asList( + "validate start", + "update start", + "query start", + "process start", + " record results", + "process start", // ContinueAsNew + " record results", + // The query after completion replays the run with no span current, so its span is a + // root like the first query's. + "query start"), + spans); + requireUniqueSpanIds(spans); + } + + @Test + public void continueAsNewUnderUserSpan() { + testWorkflowRule.newWorkflowStub(ChainedContinueAsNewWorkflow.class).run(false); + + List spans = endedSpans(); + assertSpanTree(Arrays.asList("chained-span", " chained-span"), spans); + requireUniqueSpanIds(spans); + } + + @Test + public void workflowTaskRetryReusesSpanId() { + TracerWorkflowTaskRetry workflow = + testWorkflowRule + .getWorkflowClient() + .newWorkflowStub( + TracerWorkflowTaskRetry.class, + WorkflowOptions.newBuilder() + .setTaskQueue(testWorkflowRule.getTaskQueue()) + .setWorkflowRunTimeout(Duration.ofSeconds(1)) + .setRetryOptions(RetryOptions.newBuilder().setMaximumAttempts(1).build()) + .build()); + try { + workflow.run(); + throw new AssertionError("expected the workflow run to time out"); + } catch (WorkflowFailedException e) { + assertTrue(e.getCause().toString(), e.getCause() instanceof TimeoutFailure); + } + + List spans = endedSpans(); + assertTrue("expected the span from more than one task attempt", spans.size() > 1); + Set spanIds = new HashSet<>(); + for (SpanData span : spans) { + spanIds.add(span.getSpanId()); + } + assertEquals(spanTree(spans).toString(), 1, spanIds.size()); + } + + @Test + public void explicitSpanStartTimestampSurvivesReplay() { + TracerSpanTimestampWorkflow workflow = + testWorkflowRule.newWorkflowStub(TracerSpanTimestampWorkflow.class); + workflow.run(); + + assertEquals(1, endedSpans().size()); + SpanData span = requireSpanNamed(endedSpans(), "explicit timestamp"); + assertEquals(TimeUnit.MILLISECONDS.toNanos(123456789L), span.getStartEpochNanos()); + + // Querying the completed workflow replays its history. + workflow.query(); + + assertEquals(1, endedSpans().size()); + span = requireSpanNamed(endedSpans(), "explicit timestamp"); + assertEquals(TimeUnit.MILLISECONDS.toNanos(123456789L), span.getStartEpochNanos()); + } + + @Test + public void spanEventTimestampSurvivesReplay() { + TracerSpanEventWorkflow workflow = + testWorkflowRule.newWorkflowStub(TracerSpanEventWorkflow.class); + workflow.run(); + + List spans = endedSpans(); + assertEquals(1, spans.size()); + assertEquals(1, spans.get(0).getEvents().size()); + long eventNanos = spans.get(0).getEvents().get(0).getEpochNanos(); + + // Querying the completed workflow replays its history. + workflow.query(); + + spans = endedSpans(); + assertEquals(1, spans.size()); + assertEquals(1, spans.get(0).getEvents().size()); + assertEquals(eventNanos, spans.get(0).getEvents().get(0).getEpochNanos()); + } + + @Test + public void evictionDoesNotEndOpenSpan() { + TracerSpanAcrossEvictionWorkflow workflow = + testWorkflowRule.newWorkflowStub(TracerSpanAcrossEvictionWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run); + testWorkflowRule.waitForTheEndOfWFT(execution.getWorkflowId()); + testWorkflowRule.invalidateWorkflowCache(); + + assertEquals(0, endedSpans().size()); + + workflow.finish(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + List spans = endedSpans(); + assertEquals(1, spans.size()); + assertEquals("span across eviction", spans.get(0).getName()); + } + + @Test + public void secondEndAfterEvictionDoesNotExportAgain() { + TracerSpanEndedTwiceWorkflow workflow = + testWorkflowRule.newWorkflowStub(TracerSpanEndedTwiceWorkflow.class); + WorkflowExecution execution = WorkflowClient.start(workflow::run); + testWorkflowRule.waitForTheEndOfWFT(execution.getWorkflowId()); + testWorkflowRule.invalidateWorkflowCache(); + + workflow.finish(); + WorkflowStub.fromTyped(workflow).getResult(Void.class); + + List spans = endedSpans(); + assertEquals(1, spans.size()); + assertEquals("span ended twice", spans.get(0).getName()); + } + + @Test + public void resetWithTracerCreatedBeforeResetPoint() { + List spans = runAndReset(TracerResetWorkflowImpl.class); + assertSpanTree(Arrays.asList("process start", " record results", " record results"), spans); + requireUniqueSpanIds(spans); + } + + @Test + public void resetWithTracerCreatedAfterResetPoint() { + List spans = runAndReset(TracerResetLateSourceWorkflowImpl.class); + assertSpanTree(Arrays.asList("process start", " record results", " record results"), spans); + requireUniqueSpanIds(spans); + } + + @Test + public void resetWithSpanCrossingResetPoint() { + List spans = runAndReset(TracerResetDuringSpanImpl.class); + assertSpanTree( + Arrays.asList("process start", " record results", "process start", " record results"), + spans); + // These spans reuse their IDs because they were created before the reset point. + assertEquals(spans.get(0).getSpanContext(), spans.get(2).getSpanContext()); + assertEquals(spans.get(1).getSpanContext(), spans.get(3).getSpanContext()); + // The new spans end after the old ones. + assertTrue(spans.get(2).getEndEpochNanos() > spans.get(0).getEndEpochNanos()); + assertTrue(spans.get(3).getEndEpochNanos() > spans.get(1).getEndEpochNanos()); + } + + /** + * Runs a gated workflow to completion, resets it to the task that handled the gate signal so the + * work after the signal is redone, and returns every span from both runs. + */ + private List runAndReset(Class workflowImpl) { + assumeTrue( + "Test Server doesn't support reset workflow", SDKTestWorkflowRule.useExternalService); + WorkflowClient client = testWorkflowRule.getWorkflowClient(); + WorkflowStub stub = + client.newUntypedWorkflowStub( + workflowImpl.getInterfaces()[0].getSimpleName(), + WorkflowOptions.newBuilder().setTaskQueue(testWorkflowRule.getTaskQueue()).build()); + stub.start(); + WorkflowExecution execution = stub.getExecution(); + stub.getResult(Void.class); + + ResetWorkflowExecutionResponse response = + client + .getWorkflowServiceStubs() + .blockingStub() + .resetWorkflowExecution( + ResetWorkflowExecutionRequest.newBuilder() + .setNamespace(client.getOptions().getNamespace()) + .setWorkflowExecution(execution) + .setWorkflowTaskFinishEventId(secondWorkflowTaskCompletedEventId(execution)) + .setReason("Integration test") + .setRequestId(UUID.randomUUID().toString()) + .build()); + client + .newUntypedWorkflowStub( + WorkflowTargetOptions.newBuilder() + .setWorkflowId(execution.getWorkflowId()) + .setRunId(response.getRunId()) + .build()) + .getResult(Void.class); + + return endedSpans(); + } + + private long secondWorkflowTaskCompletedEventId(WorkflowExecution execution) { + List completed = + testWorkflowRule.getHistoryEvents( + execution.getWorkflowId(), EventType.EVENT_TYPE_WORKFLOW_TASK_COMPLETED); + return completed.get(1).getEventId(); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/CodecTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/CodecTest.java new file mode 100644 index 0000000000..45462f7e23 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/CodecTest.java @@ -0,0 +1,172 @@ +package io.temporal.opentelemetry.v2.internal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import com.google.protobuf.ByteString; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.baggage.Baggage; +import io.opentelemetry.api.baggage.propagation.W3CBaggagePropagator; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanId; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceId; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.context.propagation.TextMapPropagator; +import io.temporal.api.common.v1.Payload; +import io.temporal.common.converter.DataConverter; +import io.temporal.common.converter.DefaultDataConverter; +import io.temporal.common.converter.GlobalDataConverter; +import io.temporal.common.converter.JacksonJsonPayloadConverter; +import io.temporal.common.interceptors.Header; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** Unit tests for {@link SpanCodec} write/read on Temporal headers and Nexus maps. */ +public class CodecTest { + private static final String HEADER_KEY = "_tracer-data"; + private static final SpanContext SPAN = + SpanContext.create( + TraceId.fromLongs(0, 1), + SpanId.fromLong(2), + TraceFlags.getSampled(), + TraceState.getDefault()); + private static final Baggage BAGGAGE = Baggage.builder().put("key", "value").build(); + + private SpanCodec codec; + + @Before + public void installPropagators() { + GlobalOpenTelemetry.resetForTest(); + GlobalOpenTelemetry.set( + OpenTelemetry.propagating( + ContextPropagators.create( + TextMapPropagator.composite( + W3CTraceContextPropagator.getInstance(), W3CBaggagePropagator.getInstance())))); + codec = new SpanCodec(HEADER_KEY); + } + + @After + public void resetGlobalOpenTelemetry() { + GlobalOpenTelemetry.resetForTest(); + } + + @Test + public void writesAndReadsTemporalHeaders() { + Header header = new Header(new HashMap<>()); + try (Scope ignored = current().makeCurrent()) { + codec.write(header); + } + assertPropagated(codec.read(header)); + } + + @Test + public void serializesSuppliedContext() { + Map header; + try (Scope ignored = Context.root().makeCurrent()) { + header = codec.serialize(current()); + } + assertPropagated(codec.readFromRoot(header)); + } + + @Test + public void writesAndReadsNexusHeaders() { + Map headers = new HashMap<>(); + try (Scope ignored = current().makeCurrent()) { + codec.write(headers); + } + assertPropagated(codec.read(headers)); + } + + @Test + public void leavesTemporalHeaderAloneWhenNothingToInject() { + Header header = new Header(new HashMap<>()); + try (Scope ignored = Context.root().makeCurrent()) { + codec.write(header); + } + assertNull(header.getValues().get(HEADER_KEY)); + } + + @Test + public void missingTemporalHeaderKeepsCurrentContext() { + try (Scope ignored = current().makeCurrent()) { + assertPropagated(codec.read(new Header(new HashMap<>()))); + } + } + + @Test + public void readsNexusHeadersCaseInsensitively() { + Map written = new HashMap<>(); + try (Scope ignored = current().makeCurrent()) { + codec.write(written); + } + Map upper = new HashMap<>(); + written.forEach((key, value) -> upper.put(key.toUpperCase(), value)); + assertPropagated(codec.read(upper)); + } + + @Test + public void preservesNexusHeaderKeyCase() { + Map headers = new HashMap<>(); + try (Scope ignored = current().makeCurrent()) { + codec.write(headers); + } + assertTrue(headers.containsKey("traceparent")); + assertTrue(headers.containsKey("baggage")); + } + + @Test + public void decodesPropertiesPayload() { + Properties carrier = new Properties(); + carrier.setProperty("traceparent", "00-trace-id-span-id-01"); + Payload payload = DefaultDataConverter.STANDARD_INSTANCE.toPayload(carrier).get(); + assertEquals(carrier, SpanCodec.decode(payload)); + } + + @Test + public void decodesLegacyMapPayloadViaGlobalConverter() { + Map carrier = new HashMap<>(); + carrier.put("traceparent", "00-trace-id-span-id-01"); + Payload payload = + Payload.newBuilder() + .putMetadata("encoding", ByteString.copyFromUtf8("legacy/json")) + .build(); + + DataConverter original = GlobalDataConverter.get(); + GlobalDataConverter.register( + new DefaultDataConverter(new JacksonJsonPayloadConverter()) { + @Override + public T fromPayload(Payload ignored, Class valueClass, Type type) { + return valueClass.cast(carrier); + } + }); + try { + assertEquals(carrier, SpanCodec.decode(payload)); + } finally { + GlobalDataConverter.register(original); + } + } + + private static Context current() { + return Context.root().with(Span.wrap(SPAN)).with(BAGGAGE); + } + + private static void assertPropagated(Context context) { + SpanContext span = Span.fromContext(context).getSpanContext(); + assertEquals(SPAN.getTraceId(), span.getTraceId()); + assertEquals(SPAN.getSpanId(), span.getSpanId()); + assertEquals("value", Baggage.fromContext(context).getEntryValue("key")); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/ScheduleTraceHeaderTest.java b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/ScheduleTraceHeaderTest.java new file mode 100644 index 0000000000..a93cd48f9a --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/java/io/temporal/opentelemetry/v2/internal/ScheduleTraceHeaderTest.java @@ -0,0 +1,53 @@ +package io.temporal.opentelemetry.v2.internal; + +import static org.junit.Assert.assertFalse; + +import io.temporal.api.common.v1.Payload; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.schedules.Schedule; +import io.temporal.client.schedules.ScheduleActionStartWorkflow; +import io.temporal.client.schedules.ScheduleOptions; +import io.temporal.client.schedules.ScheduleSpec; +import io.temporal.common.interceptors.Header; +import io.temporal.common.interceptors.ScheduleClientCallsInterceptor.CreateScheduleInput; +import io.temporal.common.interceptors.ScheduleClientCallsInterceptorBase; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; + +public class ScheduleTraceHeaderTest { + private static final String HEADER_KEY = "_tracer-data"; + + @Test + public void staleTraceIsRemovedWhenNoSpanIsCurrent() { + AtomicReference sent = new AtomicReference<>(); + OpenTelemetryScheduleClientCallsInterceptor interceptor = + new OpenTelemetryScheduleClientCallsInterceptor( + new InterceptorTracer(HEADER_KEY, false), + new ScheduleClientCallsInterceptorBase(null) { + @Override + public void createSchedule(CreateScheduleInput input) { + sent.set(input.getSchedule()); + } + }); + ScheduleActionStartWorkflow action = + ScheduleActionStartWorkflow.newBuilder() + .setWorkflowType("Workflow") + .setOptions(WorkflowOptions.newBuilder().setTaskQueue("queue").build()) + .setHeader( + new Header(Collections.singletonMap(HEADER_KEY, Payload.getDefaultInstance()))) + .build(); + + interceptor.createSchedule( + new CreateScheduleInput( + "schedule", + Schedule.newBuilder() + .setAction(action) + .setSpec(ScheduleSpec.newBuilder().build()) + .build(), + ScheduleOptions.newBuilder().build())); + + ScheduleActionStartWorkflow sentAction = (ScheduleActionStartWorkflow) sent.get().getAction(); + assertFalse(sentAction.getHeader().getValues().containsKey(HEADER_KEY)); + } +} diff --git a/contrib/temporal-opentelemetry-v2/src/test/resources/logback-test.xml b/contrib/temporal-opentelemetry-v2/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..4dc91ceb52 --- /dev/null +++ b/contrib/temporal-opentelemetry-v2/src/test/resources/logback-test.xml @@ -0,0 +1,17 @@ + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + diff --git a/settings.gradle b/settings.gradle index a2ee73e91a..808f83481d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -20,6 +20,8 @@ include 'temporal-opentracing' project(':temporal-opentracing').projectDir = file('contrib/temporal-opentracing') include 'temporal-opentelemetry' project(':temporal-opentelemetry').projectDir = file('contrib/temporal-opentelemetry') +include 'temporal-opentelemetry-v2' +project(':temporal-opentelemetry-v2').projectDir = file('contrib/temporal-opentelemetry-v2') include 'temporal-kotlin' // Retained for the one-time relocation POM; implementation lives in ai-integrations. include 'temporal-spring-ai' diff --git a/temporal-bom/build.gradle b/temporal-bom/build.gradle index 0bbae3e4bd..7c7847fa2a 100644 --- a/temporal-bom/build.gradle +++ b/temporal-bom/build.gradle @@ -8,6 +8,7 @@ dependencies { constraints { api project(':temporal-kotlin') api project(':temporal-opentelemetry') + api project(':temporal-opentelemetry-v2') api project(':temporal-opentracing') api project(':temporal-aws-lambda') api project(':temporal-gcp-cloud-run-opentelemetry')