[OPIK-7496] [BE] fix: strip otel attribute rule prefix from saved key (#7499)

* [OPIK-7496] [BE] fix: strip otel attribute rule prefix from saved key

* fix: require boundary before stripping prefix in storageKey

* refactor: remove storageKeyExactRuleNeverStrips test

uu

* fix: parse JSON before branching on isObject

u

* fix: filter reserved metadata keys from metadata

* fix: narrow exception catch and log parse failures

* fix: remove PII from debug log and parameterize reserved key test

* fix: guard storageKey substring and add @nonnull validation

* test: update testRuleMapping for prefix-stripped storage keys

* fix: log as warn and eextract block

* fix: only parse json looking strings

---------

Co-authored-by: Andres Cruz <andresc@comet.com>

Author: albe2669

apps/opik-backend/src/main/java/com/comet/opik/domain/OpenTelemetryMapper.java

@@ -10,6 +10,7 @@
 import com.comet.opik.domain.retention.RetentionUtils;
 import com.comet.opik.utils.JsonUtils;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.opentelemetry.proto.common.v1.AnyValue;
 import io.opentelemetry.proto.common.v1.KeyValue;
 import io.opentelemetry.proto.trace.v1.Span;
 import io.opentelemetry.proto.trace.v1.Status;
@@ -19,6 +20,7 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 
+import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
 import java.time.Duration;
 import java.time.Instant;
@@ -35,6 +37,7 @@
 import static com.comet.opik.domain.mapping.OpenTelemetryMappingUtils.extractTags;
 import static com.comet.opik.domain.mapping.OpenTelemetryMappingUtils.extractToJsonColumn;
 import static com.comet.opik.domain.mapping.OpenTelemetryMappingUtils.extractUsageField;
+import static com.comet.opik.domain.mapping.OpenTelemetryMappingUtils.storageKey;
 
 @UtilityClass
 @Slf4j
@@ -133,6 +136,9 @@ public static void enrichSpanWithAttributes(SpanBuilder spanBuilder, List<KeyVal
     private static final String CLAUDE_CODE_LLM_SPAN = "claude_code.llm_request";
     private static final String NEW_CONTEXT_ATTR = "new_context";
 
+    // Reserved metadata keys that must not be overwritten by user-supplied JSON merged from opik.metadata.
+    private static final Set<String> RESERVED_METADATA_KEYS = Set.of("thread_id", "integration", "server.address");
+
     /**
      * Same as {@link #enrichSpanWithAttributes(SpanBuilder, List, String, List)} but with the OTEL
      * span name, used for span-name-aware routing (e.g. Claude Code's {@code new_context} maps to
@@ -216,7 +222,16 @@ public static void enrichSpanWithAttributes(SpanBuilder spanBuilder, List<KeyVal
                         default -> metadata;
                     };
 
-                    extractToJsonColumn(node, key, value);
+                    String jsonKey = storageKey(rule, key);
+                    // If the suffix is empty then try merging as a JSON object,
+                    // otherwise nest under the stripped key or the rule key.
+                    if (jsonKey.isEmpty() && value.getValueCase() == AnyValue.ValueCase.STRING_VALUE) {
+                        mergeJsonObjectOrFallback(node, rule.getRule(), key, value);
+                    } else if (jsonKey.isEmpty()) {
+                        extractToJsonColumn(node, rule.getRule(), value);
+                    } else {
+                        extractToJsonColumn(node, jsonKey, value);
+                    }
                     break;
 
                 case TAGS :
@@ -298,6 +313,39 @@ public static void enrichSpanWithAttributes(SpanBuilder spanBuilder, List<KeyVal
         }
     }
 
+    /**
+     * When the storage key is empty and the value is a string, try parsing it as a JSON object
+     * and merge its fields into {@code node}, skipping {@link #RESERVED_METADATA_KEYS}. On a
+     * non-object JSON value or a parse failure, fall back to storing the raw value under the
+     * rule's key via {@link #extractToJsonColumn}.
+     */
+    private static void mergeJsonObjectOrFallback(ObjectNode node, String ruleKey, String key, AnyValue value) {
+        String stringValue = value.getStringValue();
+
+        // Only try to parse JSON-looking strings
+        String trimmed = StringUtils.trimToEmpty(stringValue);
+        if (!trimmed.startsWith("{") || !trimmed.endsWith("}")) {
+            extractToJsonColumn(node, ruleKey, value);
+            return;
+        }
+        try {
+            var jsonNode = JsonUtils.getJsonNodeFromString(stringValue);
+            if (jsonNode.isObject()) {
+                jsonNode.fields()
+                        .forEachRemaining(entry -> {
+                            if (!RESERVED_METADATA_KEYS.contains(entry.getKey())) {
+                                node.set(entry.getKey(), entry.getValue());
+                            }
+                        });
+            } else {
+                extractToJsonColumn(node, ruleKey, value);
+            }
+        } catch (UncheckedIOException e) {
+            log.warn("Failed to parse JSON, falling back to text for key '{}'", key, e);
+            extractToJsonColumn(node, ruleKey, value);
+        }
+    }
+
     private static final String TOOL_OUTPUT_EVENT_NAME = "tool.output";
     private static final Set<String> TOOL_OUTPUT_CONTENT_KEYS = Set.of("output", "content");
 

apps/opik-backend/src/main/java/com/comet/opik/domain/mapping/OpenTelemetryMappingUtils.java

@@ -82,6 +82,43 @@ public static void extractToJsonColumn(ObjectNode node, String key, @NonNull Any
         }
     }
 
+    /**
+     * Computes the storage key for an attribute under a mapping rule.
+     * <p>
+     * For exact-match rules the original key is returned unchanged. For prefix rules the
+     * prefix is stripped and any leading dot on the suffix is removed. The suffix may be
+     * empty, in which case the caller should typically merge the value into the enclosing object.
+     * If a prefix rule does not actually match the key (key shorter than the prefix, or the
+     * stripped suffix does not start with a dot when the prefix lacks one), the original
+     * key is returned unchanged.
+     *
+     * @param rule the mapping rule to apply; must not be {@code null}
+     * @param key the attribute key to transform; must not be {@code null}
+     * @return the storage key to use, never {@code null}
+     */
+    public static String storageKey(@NonNull OpenTelemetryMappingRule rule, @NonNull String key) {
+        if (!rule.isPrefix()) {
+            return key;
+        }
+
+        String prefix = rule.getRule();
+        if (key.length() < prefix.length()) {
+            return key;
+        }
+        String suffix = key.substring(prefix.length());
+
+        // Strip when suffix is empty, suffix starts with '.', or the rule itself ends with '.' (e.g. "gen_ai.input.").
+        // A prefix match like "input" on "input_tokens" must not strip.
+        if (suffix.isEmpty() || suffix.startsWith(".") || prefix.endsWith(".")) {
+            if (suffix.startsWith(".")) {
+                suffix = suffix.substring(1);
+            }
+            return suffix;
+        }
+
+        return key;
+    }
+
     // Prefix rules (e.g. `gen_ai.usage.`) carry the usage name in the suffix; exact-match rules
     // (e.g. Claude Code's `input_tokens`) use the full key.
     private static String usageKey(OpenTelemetryMappingRule rule, String key) {

apps/opik-backend/src/test/java/com/comet/opik/api/resources/v1/priv/OpenTelemetryResourceTest.java

@@ -406,12 +406,12 @@ void testRuleMapping(String modelKey) {
             assertThat(span.provider()).isEqualTo("openai");
             assertThat(span.type()).isEqualTo(SpanType.llm);
 
-            assertThat(span.metadata().get("code.line").asInt()).isEqualTo(11);
-            assertThat(span.metadata().get("smolagents.single").asText()).isEqualTo("value");
-            assertThat(span.metadata().get("smolagents.node").get("key").asText()).isEqualTo("value");
-            assertThat(span.metadata().get("smolagents.array").isArray()).isTrue();
+            assertThat(span.metadata().get("line").asInt()).isEqualTo(11);
+            assertThat(span.metadata().get("single").asText()).isEqualTo("value");
+            assertThat(span.metadata().get("node").get("key").asText()).isEqualTo("value");
+            assertThat(span.metadata().get("array").isArray()).isTrue();
 
-            assertThat(span.input().get("input").get("key").asText()).isEqualTo("value");
+            assertThat(span.input().get("key").asText()).isEqualTo("value");
             assertThat(span.input().get("tools").isArray()).isEqualTo(Boolean.TRUE);
             assertThat(span.input().get("all_messages").isArray()).isEqualTo(Boolean.TRUE);
 
@@ -423,8 +423,8 @@ void testRuleMapping(String modelKey) {
 
             // check metadata
             assertThat(span.metadata()).isNotEmpty();
-            assertThat(span.metadata().get("opik.metadata").get("foo").asText()).isEqualTo("bar");
-            assertThat(span.metadata().get("opik.metadata.inline").asText()).isEqualTo("inline_value");
+            assertThat(span.metadata().get("foo").asText()).isEqualTo("bar");
+            assertThat(span.metadata().get("inline").asText()).isEqualTo("inline_value");
         }
 
         @Test