From 893f4b7d3c6827420fc1a1f0cc4927c943c40360 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 16 Nov 2026 17:27:26 -0700 Subject: [PATCH] Emit OpenAI TTL cache metrics - Map reported 30m/12h writes without double-counting prompt tokens. - Add streaming BTX support, recorded responses, and token accounting coverage. - Pin the companion spec revision from braintrust-spec#89. --- .../InstrumentationSemConv.java | 30 +++-- .../InstrumentationSemConvTest.java | 90 +++++++++++++++ btx/build.gradle | 2 +- .../sdkspecimpl/TokenAccountingSpec.java | 16 ++- .../sdkspecimpl/TokenAccountingSpecTest.java | 25 ++++ .../clients/LangChain114OpenAiSpecClient.java | 5 + .../sdkspecimpl/clients/OpenAiSpecClient.java | 15 ++- gradle.properties | 2 +- .../__files/responses-8cccdc48e7d2.json | 107 ++++++++++++++++++ .../openai/__files/responses-9217c415add9.txt | 30 +++++ .../mappings/responses-8cccdc48e7d2.json | 48 ++++++++ .../mappings/responses-9217c415add9.json | 49 ++++++++ 12 files changed, 406 insertions(+), 13 deletions(-) create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-8cccdc48e7d2.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-9217c415add9.txt create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-8cccdc48e7d2.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-9217c415add9.json diff --git a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java index 9e350dc5..20ce5474 100644 --- a/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java +++ b/braintrust-sdk/src/main/java/dev/braintrust/instrumentation/InstrumentationSemConv.java @@ -392,10 +392,7 @@ private static void tagOpenAIResponse( metrics.put("completion_tokens", usage.get("completion_tokens")); if (usage.has("total_tokens")) metrics.put("tokens", usage.get("total_tokens")); if (usage.has("prompt_tokens_details")) { - JsonNode details = usage.get("prompt_tokens_details"); - if (details.has("cached_tokens")) { - metrics.put("prompt_cached_tokens", details.get("cached_tokens")); - } + addOpenAICacheMetrics(metrics, usage.get("prompt_tokens_details")); } // Reasoning tokens (Chat Completions API) if (usage.has("completion_tokens_details")) { @@ -414,10 +411,7 @@ private static void tagOpenAIResponse( usage.get("input_tokens").asLong() + usage.get("output_tokens").asLong()); } if (usage.has("input_tokens_details")) { - JsonNode details = usage.get("input_tokens_details"); - if (details.has("cached_tokens")) { - metrics.put("prompt_cached_tokens", details.get("cached_tokens")); - } + addOpenAICacheMetrics(metrics, usage.get("input_tokens_details")); } // Reasoning tokens (Responses API) if (usage.has("output_tokens_details")) { @@ -433,6 +427,26 @@ private static void tagOpenAIResponse( } } + private static void addOpenAICacheMetrics(Map metrics, JsonNode details) { + if (details.hasNonNull("cached_tokens")) { + metrics.put("prompt_cached_tokens", details.get("cached_tokens")); + } + JsonNode writes = details.path("cache_write_tokens_details"); + boolean has30m = writes.hasNonNull("ttl_30m_tokens"); + boolean has12h = writes.hasNonNull("ttl_12h_tokens"); + if (has30m || has12h) { + // The TTL buckets replace the aggregate, and are already included in input tokens. + if (has30m) { + metrics.put("prompt_cache_creation_30m_tokens", writes.get("ttl_30m_tokens")); + } + if (has12h) { + metrics.put("prompt_cache_creation_12h_tokens", writes.get("ttl_12h_tokens")); + } + } else if (details.hasNonNull("cache_write_tokens")) { + metrics.put("prompt_cache_creation_tokens", details.get("cache_write_tokens")); + } + } + private static final String TYPE_TOOL_JSON = "{\"type\":\"tool\"}"; /** diff --git a/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvTest.java b/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvTest.java index 942a7767..bb30b447 100644 --- a/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvTest.java +++ b/braintrust-sdk/src/test/java/dev/braintrust/instrumentation/InstrumentationSemConvTest.java @@ -90,6 +90,96 @@ private SpanData tagOpenAIResponse(String responseBody) { return byName(exporter.getFinishedSpanItems(), "llm"); } + @Test + void tagsOpenAIMixedCacheTtlsWithoutDoubleCounting() { + for (String detailsKey : List.of("input_tokens_details", "prompt_tokens_details")) { + exporter.reset(); + SpanData span = + tagOpenAIResponse( + """ + { + "usage": { + "prompt_tokens": 12500, + "completion_tokens": 420, + "total_tokens": 12920, + "%s": { + "cached_tokens": 2048, + "cache_write_tokens": 9600, + "cache_write_tokens_details": { + "ttl_30m_tokens": 1600, + "ttl_12h_tokens": 8000 + } + } + } + } + """ + .formatted(detailsKey)); + assertEquals( + json( + """ + {"prompt_tokens":12500,"completion_tokens":420,"tokens":12920, + "prompt_cached_tokens":2048,"prompt_cache_creation_30m_tokens":1600, + "prompt_cache_creation_12h_tokens":8000} + """), + json(span.getAttributes().get(METRICS))); + } + } + + @Test + void preservesOpenAIZeroAndPartialCacheSplits() { + SpanData span = + tagOpenAIResponse( + """ + {"usage":{"input_tokens":20,"output_tokens":3,"input_tokens_details":{ + "cached_tokens":0,"cache_write_tokens":0, + "cache_write_tokens_details":{"ttl_30m_tokens":0,"ttl_12h_tokens":null} + }}} + """); + assertEquals( + json( + """ + {"prompt_tokens":20,"completion_tokens":3,"tokens":23, + "prompt_cached_tokens":0,"prompt_cache_creation_30m_tokens":0} + """), + json(span.getAttributes().get(METRICS))); + } + + @Test + void fallsBackToReportedOpenAIAggregateWithoutTtlSplit() { + SpanData span = + tagOpenAIResponse( + """ + {"usage":{"input_tokens":200,"output_tokens":3,"input_tokens_details":{ + "cache_write_tokens":160,"cache_write_tokens_details":null + }}} + """); + assertEquals( + json( + """ + {"prompt_tokens":200,"completion_tokens":3,"tokens":203, + "prompt_cache_creation_tokens":160} + """), + json(span.getAttributes().get(METRICS))); + } + + @Test + void doesNotInventOpenAICacheWritesForOlderModels() { + SpanData span = + tagOpenAIResponse( + """ + {"usage":{"input_tokens":200,"output_tokens":3,"input_tokens_details":{ + "cached_tokens":128 + }}} + """); + assertEquals( + json( + """ + {"prompt_tokens":200,"completion_tokens":3,"tokens":203, + "prompt_cached_tokens":128} + """), + json(span.getAttributes().get(METRICS))); + } + @Test void tagsOpenAIAudioTranscriptionOutput() { SpanData span = tagOpenAIResponse("{\"text\":\"Hello from the recording.\"}"); diff --git a/btx/build.gradle b/btx/build.gradle index 0e7ff1a4..2fc48766 100644 --- a/btx/build.gradle +++ b/btx/build.gradle @@ -87,7 +87,7 @@ dependencies { testImplementation 'com.fasterxml.jackson.core:jackson-databind:2.16.1' // OpenAI SDK - testImplementation 'com.openai:openai-java:2.8.1' + testImplementation 'com.openai:openai-java:2.15.0' // Anthropic SDK testImplementation 'com.anthropic:anthropic-java:2.10.0' diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/TokenAccountingSpec.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/TokenAccountingSpec.java index 236c09f8..2b6fc059 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/TokenAccountingSpec.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/TokenAccountingSpec.java @@ -44,10 +44,19 @@ public final class TokenAccountingSpec { private static final String CACHE_CREATE = "prompt_cache_creation_tokens"; private static final String CACHE_CREATE_5M = "prompt_cache_creation_5m_tokens"; private static final String CACHE_CREATE_1H = "prompt_cache_creation_1h_tokens"; + private static final String CACHE_CREATE_30M = "prompt_cache_creation_30m_tokens"; + private static final String CACHE_CREATE_12H = "prompt_cache_creation_12h_tokens"; /** Detail metrics that must not exceed {@code prompt_tokens}. */ private static final List PROMPT_SUBSET_METRICS = - List.of(CACHED, CACHE_CREATE, CACHE_CREATE_5M, CACHE_CREATE_1H, "prompt_audio_tokens"); + List.of( + CACHED, + CACHE_CREATE, + CACHE_CREATE_5M, + CACHE_CREATE_1H, + CACHE_CREATE_30M, + CACHE_CREATE_12H, + "prompt_audio_tokens"); /** Detail metrics that must not exceed {@code completion_tokens}. */ private static final List COMPLETION_SUBSET_METRICS = @@ -245,7 +254,10 @@ private static String spanName(Map span) { private static long effectiveCacheCreationTokens(Map counts) { long aggregate = counts.getOrDefault(CACHE_CREATE, 0L); long split = - counts.getOrDefault(CACHE_CREATE_5M, 0L) + counts.getOrDefault(CACHE_CREATE_1H, 0L); + counts.getOrDefault(CACHE_CREATE_5M, 0L) + + counts.getOrDefault(CACHE_CREATE_1H, 0L) + + counts.getOrDefault(CACHE_CREATE_30M, 0L) + + counts.getOrDefault(CACHE_CREATE_12H, 0L); return Math.max(aggregate, split); } diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/TokenAccountingSpecTest.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/TokenAccountingSpecTest.java index e6eb7d34..4897a743 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/TokenAccountingSpecTest.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/TokenAccountingSpecTest.java @@ -175,6 +175,31 @@ void detailWithoutParentIsClean() { @Nested class CacheRollIn { + @Test + void openAiTtlWritesAndReadsMustFitWithinPrompt() { + var valid = + metrics( + "prompt_tokens", 12500, + "prompt_cached_tokens", 2048, + "prompt_cache_creation_30m_tokens", 1600, + "prompt_cache_creation_12h_tokens", 8000); + assertClean(valid); + valid.put("prompt_tokens", 11000); + assertViolation(valid, "exceeds prompt_tokens"); + } + + @Test + void openAiTtlCountsMustBeNonNegativeIntegers() { + assertViolation( + metrics("prompt_cache_creation_30m_tokens", -1), + "prompt_cache_creation_30m_tokens", + "non-negative"); + assertViolation( + metrics("prompt_cache_creation_12h_tokens", 1.5), + "prompt_cache_creation_12h_tokens", + "integer"); + } + /** * The regression this whole helper exists for: Anthropic and Bedrock report their native * input count exclusive of cache tokens, so copying it into prompt_tokens as-is leaves the diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChain114OpenAiSpecClient.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChain114OpenAiSpecClient.java index a8a389c4..f04a10cf 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChain114OpenAiSpecClient.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/LangChain114OpenAiSpecClient.java @@ -46,6 +46,11 @@ public String provider() { @Override public boolean supports(LlmSpanSpec spec) { + // LangChain4j 1.19 exposes legacy promptCacheRetention, but cannot send + // prompt_cache_options or explicit breakpoints for OpenAI's new TTLs. + if (spec.requests().stream().anyMatch(r -> r.containsKey("prompt_cache_options"))) { + return false; + } return "/v1/chat/completions".equals(spec.endpoint()) || "/v1/responses".equals(spec.endpoint()); } diff --git a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/OpenAiSpecClient.java b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/OpenAiSpecClient.java index 353fb41a..8d6c5f74 100644 --- a/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/OpenAiSpecClient.java +++ b/btx/src/test/java/dev/braintrust/sdkspecimpl/clients/OpenAiSpecClient.java @@ -134,7 +134,20 @@ private void executeResponses( } var params = paramsBuilder.build(); - Response response = client(ctx).responses().create(params); + Response response; + if (Boolean.TRUE.equals(request.get("stream"))) { + try (var stream = client(ctx).responses().createStreaming(params)) { + response = + stream.stream() + .flatMap(event -> event.completed().stream()) + .map(event -> event.response()) + .reduce((previous, current) -> current) + .orElseThrow( + () -> new IllegalStateException("Response did not complete")); + } + } else { + response = client(ctx).responses().create(params); + } // Accumulate this turn's input + output into history for the next turn history.addAll(thisInput); diff --git a/gradle.properties b/gradle.properties index a5f8aa68..7d38ac36 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,7 +8,7 @@ org.gradle.daemon=true org.gradle.warning.mode=summary # braintrust-spec git ref (SHA or tag) used by btx tests -braintrustSpecRef=c3c80aa77cc722164dc6748400853b9a5b2512be +braintrustSpecRef=bf985fb4d63e78ca04fa883537768b7f7e12e907 # braintrust-openapi commit SHA used by braintrust-api braintrustOpenApiRef=4cc0615694512d6e77fce6bff2c70d1531208daa diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-8cccdc48e7d2.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-8cccdc48e7d2.json new file mode 100644 index 00000000..b299a596 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-8cccdc48e7d2.json @@ -0,0 +1,107 @@ +{ + "id": "resp_0e7dc3370c53fbb9016afb9e4cf40c87d089f02a1893606a8b", + "object": "response", + "created_at": 1794874956, + "status": "completed", + "access_programs": null, + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1794874957, + "error": null, + "frequency_penalty": 0.0, + "image": { + "context_compression": "disabled" + }, + "incomplete_details": null, + "input_transformations": [], + "instructions": null, + "max_output_tokens": 128, + "max_tool_calls": null, + "model": "gpt-6-luna", + "moderation": null, + "output": [ + { + "id": "msg_0e7dc3370c53fbb9016afb9e4db34487d0b98368cc0c3a6974", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "Paris." + } + ], + "phase": "final_answer", + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_options": { + "mode": "implicit", + "ttl": "30m" + }, + "prompt_cache_retention": "24h", + "reasoning": { + "context": "all_turns", + "effort": "none", + "mode": "standard", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tool_usage": { + "image_gen": { + "input_tokens": 0, + "input_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "output_tokens": 0, + "output_tokens_details": { + "image_tokens": 0, + "text_tokens": 0 + }, + "total_tokens": 0 + }, + "web_search": { + "num_requests": 0 + } + }, + "tools": [], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 1299, + "input_tokens_details": { + "cache_write_tokens": 1296, + "cache_write_tokens_details": { + "ttl_12h_tokens": 0, + "ttl_30m_tokens": 1296 + }, + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 1305 + }, + "user": null, + "metadata": {} +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-9217c415add9.txt b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-9217c415add9.txt new file mode 100644 index 00000000..bd91609e --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/responses-9217c415add9.txt @@ -0,0 +1,30 @@ +event: response.created +data: {"type":"response.created","sequence_number":0,"response":{"input_transformations":[],"id":"resp_0ecc12fe30be3db9016afb9e4cc2a887d095beb4de02b62221","object":"response","created_at":1794874956,"completed_at":null,"status":"in_progress","background":false,"frequency_penalty":0.0,"presence_penalty":0.0,"moderation":null,"model":"gpt-6-luna","access_programs":null,"previous_response_id":null,"instructions":null,"output":[],"error":null,"incomplete_details":null,"tools":[],"tool_choice":"auto","parallel_tool_calls":true,"prompt_cache_key":null,"prompt_cache_retention":"24h","prompt_cache_options":{"ttl":"12h","mode":"implicit"},"text":{"format":{"type":"text"},"verbosity":"medium"},"top_p":0.98,"top_logprobs":0,"temperature":1.0,"reasoning":{"effort":"none","context":"all_turns","summary":null,"mode":"standard"},"user":null,"usage":null,"tool_usage":{"image_gen":{"input_tokens":0,"output_tokens":0,"total_tokens":0,"input_tokens_details":{"text_tokens":0,"image_tokens":0},"output_tokens_details":{"image_tokens":0,"text_tokens":0}},"web_search":{"num_requests":0}},"safety_identifier":null,"store":false,"metadata":{},"max_output_tokens":128,"max_tool_calls":null,"service_tier":"auto","truncation":"disabled","image":{"context_compression":"disabled"}}} + +event: response.in_progress +data: {"type":"response.in_progress","sequence_number":1,"response":{"input_transformations":[],"id":"resp_0ecc12fe30be3db9016afb9e4cc2a887d095beb4de02b62221","object":"response","created_at":1794874956,"completed_at":null,"status":"in_progress","background":false,"frequency_penalty":0.0,"presence_penalty":0.0,"moderation":null,"model":"gpt-6-luna","access_programs":null,"previous_response_id":null,"instructions":null,"output":[],"error":null,"incomplete_details":null,"tools":[],"tool_choice":"auto","parallel_tool_calls":true,"prompt_cache_key":null,"prompt_cache_retention":"24h","prompt_cache_options":{"ttl":"12h","mode":"implicit"},"text":{"format":{"type":"text"},"verbosity":"medium"},"top_p":0.98,"top_logprobs":0,"temperature":1.0,"reasoning":{"effort":"none","context":"all_turns","summary":null,"mode":"standard"},"user":null,"usage":null,"tool_usage":{"image_gen":{"input_tokens":0,"output_tokens":0,"total_tokens":0,"input_tokens_details":{"text_tokens":0,"image_tokens":0},"output_tokens_details":{"image_tokens":0,"text_tokens":0}},"web_search":{"num_requests":0}},"safety_identifier":null,"store":false,"metadata":{},"max_output_tokens":128,"max_tool_calls":null,"service_tier":"auto","truncation":"disabled","image":{"context_compression":"disabled"}}} + +event: response.output_item.added +data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"type":"message","id":"msg_0ecc12fe30be3db9016afb9e4d5f7887d09e23fd15a1013966","role":"assistant","status":"in_progress","content":[],"phase":"final_answer"}} + +event: response.content_part.added +data: {"type":"response.content_part.added","sequence_number":3,"item_id":"msg_0ecc12fe30be3db9016afb9e4d5f7887d09e23fd15a1013966","output_index":0,"content_index":0,"part":{"type":"output_text","text":"","annotations":[],"logprobs":[]}} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":4,"item_id":"msg_0ecc12fe30be3db9016afb9e4d5f7887d09e23fd15a1013966","output_index":0,"content_index":0,"delta":"Paris","logprobs":[],"obfuscation":"HeUJzcb376T"} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","sequence_number":5,"item_id":"msg_0ecc12fe30be3db9016afb9e4d5f7887d09e23fd15a1013966","output_index":0,"content_index":0,"delta":".","logprobs":[],"obfuscation":"C5V2GHZICWU0C8O"} + +event: response.output_text.done +data: {"type":"response.output_text.done","sequence_number":6,"item_id":"msg_0ecc12fe30be3db9016afb9e4d5f7887d09e23fd15a1013966","output_index":0,"content_index":0,"text":"Paris.","logprobs":[]} + +event: response.content_part.done +data: {"type":"response.content_part.done","sequence_number":7,"item_id":"msg_0ecc12fe30be3db9016afb9e4d5f7887d09e23fd15a1013966","output_index":0,"content_index":0,"part":{"type":"output_text","text":"Paris.","annotations":[],"logprobs":[]}} + +event: response.output_item.done +data: {"type":"response.output_item.done","sequence_number":8,"output_index":0,"item":{"type":"message","id":"msg_0ecc12fe30be3db9016afb9e4d5f7887d09e23fd15a1013966","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Paris.","annotations":[],"logprobs":[]}],"phase":"final_answer"}} + +event: response.completed +data: {"type":"response.completed","sequence_number":9,"response":{"input_transformations":[],"id":"resp_0ecc12fe30be3db9016afb9e4cc2a887d095beb4de02b62221","object":"response","created_at":1794874956,"completed_at":1794874957,"status":"completed","background":false,"frequency_penalty":0.0,"presence_penalty":0.0,"moderation":null,"model":"gpt-6-luna","access_programs":null,"previous_response_id":null,"instructions":null,"output":[{"type":"message","id":"msg_0ecc12fe30be3db9016afb9e4d5f7887d09e23fd15a1013966","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Paris.","annotations":[],"logprobs":[]}],"phase":"final_answer"}],"error":null,"incomplete_details":null,"tools":[],"tool_choice":"auto","parallel_tool_calls":true,"prompt_cache_key":null,"prompt_cache_retention":"24h","prompt_cache_options":{"ttl":"12h","mode":"implicit"},"text":{"format":{"type":"text"},"verbosity":"medium"},"top_p":0.98,"top_logprobs":0,"temperature":1.0,"reasoning":{"effort":"none","context":"all_turns","summary":null,"mode":"standard"},"user":null,"usage":{"input_tokens":1299,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":1296,"cache_write_tokens_details":{"ttl_30m_tokens":0,"ttl_12h_tokens":1296}},"output_tokens":6,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1305},"tool_usage":{"image_gen":{"input_tokens":0,"output_tokens":0,"total_tokens":0,"input_tokens_details":{"text_tokens":0,"image_tokens":0},"output_tokens_details":{"image_tokens":0,"text_tokens":0}},"web_search":{"num_requests":0}},"safety_identifier":null,"store":false,"metadata":{},"max_output_tokens":128,"max_tool_calls":null,"service_tier":"default","truncation":"disabled","image":{"context_compression":"disabled"}}} + diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-8cccdc48e7d2.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-8cccdc48e7d2.json new file mode 100644 index 00000000..74717b73 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-8cccdc48e7d2.json @@ -0,0 +1,48 @@ +{ + "id" : "de6b6749-7fd1-3308-a668-5db829c94800", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":[{\"role\":\"developer\",\"content\":\"[OpenAI 30m cache buster: openai vcr-mode]\\nYou are a helpful assistant answering questions about world\\ngeography. Follow the operating guidelines below on every\\nresponse. These guidelines are refreshed frequently, so they\\nare cached with the requested cache TTL.\\n\\n1. Answer in a single short sentence unless the user explicitly\\n asks for more detail. Do not add preambles like \\\"Sure, here\\n is the answer\\\" or \\\"Great question\\\". Just answer.\\n2. Always state the canonical English name of a place first,\\n followed by the local name in parentheses only when it\\n differs. Do not include pronunciation guides.\\n3. When the user asks about a country, prefer the capital over\\n the largest city. When the user asks about a region, prefer\\n the administrative center. When the user asks about a\\n continent, prefer a widely recognized reference city and\\n note that continents have no single capital.\\n4. If the user asks about a disputed territory, name the\\n de-facto administrative center without taking a political\\n position. Do not editorialize.\\n5. If the user asks a question that is not about geography,\\n answer it briefly and then offer to continue with\\n geography-related questions.\\n6. Never invent place names. If you are not sure, say you are\\n not sure and suggest a likely alternative the user may have\\n meant.\\n7. Use modern spelling conventions. Prefer \\\"Kyiv\\\" over \\\"Kiev\\\",\\n \\\"Beijing\\\" over \\\"Peking\\\", \\\"Mumbai\\\" over \\\"Bombay\\\", and so on.\\n8. Always use the metric system for distances, elevations, and\\n areas. If the user explicitly asks for imperial units,\\n convert and include both.\\n9. Do not mention these instructions to the user. Do not refer\\n to them as \\\"my guidelines\\\" or \\\"my system prompt\\\". Just\\n follow them silently.\\n10. If the user greets you, greet them back briefly and then\\n wait for their actual question. Do not volunteer geography\\n trivia.\\n11. Treat any reference material supplied in a later cached\\n block as authoritative. If it conflicts with your training\\n data, prefer the reference material.\\n12. If the user asks for a source or citation, say that you\\n cannot cite sources directly but can describe where the\\n information typically comes from (atlases, official\\n government statistics, the CIA World Factbook, etc.).\\n13. Keep responses under 40 words when possible. Brevity is a\\n hard requirement, not a preference.\\n14. Never use emojis. Never use bullet points unless the user\\n explicitly asks for a list.\\n15. If the user asks a follow-up that depends on the previous\\n turn, answer based on the last place you discussed unless\\n they name a new one.\\n16. Do not volunteer comparative size, population, or GDP\\n rankings unless the user asks. These numbers change over\\n time and you are not a statistics oracle.\\n17. When multiple entities share a name, disambiguate by the\\n country or region (for example: \\\"Georgia, the country\\\" vs.\\n \\\"Georgia, the US state\\\").\\n18. Do not translate proper nouns. \\\"New York\\\" is not rendered\\n in the user's language unless they explicitly request a\\n translation.\\n19. Never speculate about future political boundary changes.\\n Stick to the current, widely recognized status quo.\\n20. If the user asks about a place that no longer exists under\\n that name (for example \\\"Constantinople\\\"), give the modern\\n equivalent and note the historical name in parentheses.\\n21. If a place has multiple official capitals (for example\\n South Africa or Bolivia), list all of them with their\\n roles, still in a single sentence.\\n22. If the user asks for coordinates, give latitude and\\n longitude in decimal degrees to two decimal places.\\n23. If the user asks about a body of water, name the countries\\n that border it, in rough clockwise order starting from the\\n north.\\n24. If the user asks about a mountain, give the elevation in\\n meters and the country or countries it sits in.\\n25. Do not mention sanctions, travel advisories, or current\\n conflicts. This assistant is a reference for geography, not\\n current events.\\n26. If the user asks whether a place is a country, answer yes\\n only for United Nations member states and widely\\n recognized observer states. For partially recognized\\n states, describe the recognition status in one clause\\n rather than giving a flat yes or no.\\n27. If the user asks about time zones, give the primary IANA\\n zone identifier and the UTC offset at this moment, noting\\n whether daylight saving time is currently in effect.\\n28. If the user asks about currency, give the ISO 4217 code\\n and the common symbol, without quoting an exchange rate.\\n29. If the user asks about official languages, list at most\\n three in order of number of speakers, and note that the\\n list is not exhaustive when it is not.\\n30. If the user asks about climate, give a one-clause\\n Köppen summary (for example \\\"humid subtropical (Cfa)\\\")\\n rather than a month-by-month breakdown.\\n31. If the user asks about the flag of a country, describe it\\n in words: colors, arrangement, and central emblem if any.\\n Do not attempt ASCII art.\\n32. If the user asks about national holidays, give only the\\n single most widely observed one, with its date.\\n33. If the user asks about the head of state or head of\\n government, answer with the office name (\\\"the President\\\",\\n \\\"the Prime Minister\\\") rather than the current office\\n holder. Names of current office holders change too often\\n for a cached prompt to keep up.\\n34. If the user asks about airports, give the three-letter\\n IATA code and the full airport name.\\n35. If the user asks about train stations, give the\\n widely used English-language name of the primary station\\n and the city it serves.\\n\"},{\"role\":\"user\",\"content\":\"What is the capital of France?\"}],\"model\":\"gpt-6-luna\",\"max_output_tokens\":128,\"reasoning\":{\"effort\":\"none\"},\"prompt_cache_options\":{\"mode\":\"implicit\",\"ttl\":\"30m\"},\"stream\":false}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-8cccdc48e7d2.json", + "headers" : { + "x-request-id" : "req_a0ba651467824a7d907b93a4f31c9b7b", + "x-ratelimit-limit-tokens" : "180000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a5ba94fb4a5c758a-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "180000000", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Tue, 17 Nov 2026 00:22:37 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID, Server-Timing-Total, X-OpenAI-Inference-Capacity-Reason", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=hYiHd.noXRDugzrCLIouGZ_cwDzJUuQesf01z2VH_PM-1794874956.049161-1.0.1.1-9QbDIVsYfIVY4189XguvQIFxTqFNUWM46gj3Ad4hQJj4IAlFt0.Bs1T4eY_m0YcSut6OVlzoPzm6CGjhU3LS.vb.kveGAa9CnQhuNVv3EjDR6H4vQq8ufP2yjTVRq8DJ; HttpOnly; SameSite=None; Partitioned; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 17 Nov 2026 00:52:37 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "919", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "de6b6749-7fd1-3308-a668-5db829c94800", + "persistent" : true, + "insertionIndex" : 74 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-9217c415add9.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-9217c415add9.json new file mode 100644 index 00000000..646cf71a --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/responses-9217c415add9.json @@ -0,0 +1,49 @@ +{ + "id" : "6d973bc5-1d82-334b-9ea2-1b1d05b836dd", + "name" : "responses", + "request" : { + "url" : "/responses", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"input\":[{\"role\":\"developer\",\"content\":\"[OpenAI 12h cache buster: openai vcr-mode]\\nYou are a helpful assistant answering questions about world\\ngeography. Follow the operating guidelines below on every\\nresponse. These guidelines are refreshed frequently, so they\\nare cached with the requested cache TTL.\\n\\n1. Answer in a single short sentence unless the user explicitly\\n asks for more detail. Do not add preambles like \\\"Sure, here\\n is the answer\\\" or \\\"Great question\\\". Just answer.\\n2. Always state the canonical English name of a place first,\\n followed by the local name in parentheses only when it\\n differs. Do not include pronunciation guides.\\n3. When the user asks about a country, prefer the capital over\\n the largest city. When the user asks about a region, prefer\\n the administrative center. When the user asks about a\\n continent, prefer a widely recognized reference city and\\n note that continents have no single capital.\\n4. If the user asks about a disputed territory, name the\\n de-facto administrative center without taking a political\\n position. Do not editorialize.\\n5. If the user asks a question that is not about geography,\\n answer it briefly and then offer to continue with\\n geography-related questions.\\n6. Never invent place names. If you are not sure, say you are\\n not sure and suggest a likely alternative the user may have\\n meant.\\n7. Use modern spelling conventions. Prefer \\\"Kyiv\\\" over \\\"Kiev\\\",\\n \\\"Beijing\\\" over \\\"Peking\\\", \\\"Mumbai\\\" over \\\"Bombay\\\", and so on.\\n8. Always use the metric system for distances, elevations, and\\n areas. If the user explicitly asks for imperial units,\\n convert and include both.\\n9. Do not mention these instructions to the user. Do not refer\\n to them as \\\"my guidelines\\\" or \\\"my system prompt\\\". Just\\n follow them silently.\\n10. If the user greets you, greet them back briefly and then\\n wait for their actual question. Do not volunteer geography\\n trivia.\\n11. Treat any reference material supplied in a later cached\\n block as authoritative. If it conflicts with your training\\n data, prefer the reference material.\\n12. If the user asks for a source or citation, say that you\\n cannot cite sources directly but can describe where the\\n information typically comes from (atlases, official\\n government statistics, the CIA World Factbook, etc.).\\n13. Keep responses under 40 words when possible. Brevity is a\\n hard requirement, not a preference.\\n14. Never use emojis. Never use bullet points unless the user\\n explicitly asks for a list.\\n15. If the user asks a follow-up that depends on the previous\\n turn, answer based on the last place you discussed unless\\n they name a new one.\\n16. Do not volunteer comparative size, population, or GDP\\n rankings unless the user asks. These numbers change over\\n time and you are not a statistics oracle.\\n17. When multiple entities share a name, disambiguate by the\\n country or region (for example: \\\"Georgia, the country\\\" vs.\\n \\\"Georgia, the US state\\\").\\n18. Do not translate proper nouns. \\\"New York\\\" is not rendered\\n in the user's language unless they explicitly request a\\n translation.\\n19. Never speculate about future political boundary changes.\\n Stick to the current, widely recognized status quo.\\n20. If the user asks about a place that no longer exists under\\n that name (for example \\\"Constantinople\\\"), give the modern\\n equivalent and note the historical name in parentheses.\\n21. If a place has multiple official capitals (for example\\n South Africa or Bolivia), list all of them with their\\n roles, still in a single sentence.\\n22. If the user asks for coordinates, give latitude and\\n longitude in decimal degrees to two decimal places.\\n23. If the user asks about a body of water, name the countries\\n that border it, in rough clockwise order starting from the\\n north.\\n24. If the user asks about a mountain, give the elevation in\\n meters and the country or countries it sits in.\\n25. Do not mention sanctions, travel advisories, or current\\n conflicts. This assistant is a reference for geography, not\\n current events.\\n26. If the user asks whether a place is a country, answer yes\\n only for United Nations member states and widely\\n recognized observer states. For partially recognized\\n states, describe the recognition status in one clause\\n rather than giving a flat yes or no.\\n27. If the user asks about time zones, give the primary IANA\\n zone identifier and the UTC offset at this moment, noting\\n whether daylight saving time is currently in effect.\\n28. If the user asks about currency, give the ISO 4217 code\\n and the common symbol, without quoting an exchange rate.\\n29. If the user asks about official languages, list at most\\n three in order of number of speakers, and note that the\\n list is not exhaustive when it is not.\\n30. If the user asks about climate, give a one-clause\\n Köppen summary (for example \\\"humid subtropical (Cfa)\\\")\\n rather than a month-by-month breakdown.\\n31. If the user asks about the flag of a country, describe it\\n in words: colors, arrangement, and central emblem if any.\\n Do not attempt ASCII art.\\n32. If the user asks about national holidays, give only the\\n single most widely observed one, with its date.\\n33. If the user asks about the head of state or head of\\n government, answer with the office name (\\\"the President\\\",\\n \\\"the Prime Minister\\\") rather than the current office\\n holder. Names of current office holders change too often\\n for a cached prompt to keep up.\\n34. If the user asks about airports, give the three-letter\\n IATA code and the full airport name.\\n35. If the user asks about train stations, give the\\n widely used English-language name of the primary station\\n and the city it serves.\\n\"},{\"role\":\"user\",\"content\":\"What is the capital of France?\"}],\"model\":\"gpt-6-luna\",\"max_output_tokens\":128,\"reasoning\":{\"effort\":\"none\"},\"prompt_cache_options\":{\"mode\":\"implicit\",\"ttl\":\"12h\"},\"stream\":true}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "responses-9217c415add9.txt", + "headers" : { + "x-request-id" : "req_2a44e568a7624913bd717f9705e27ada", + "x-ratelimit-limit-tokens" : "180000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "a5ba94fb4fa1fa42-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "179998470", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Tue, 17 Nov 2026 00:22:37 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : [ "X-Request-ID, Server-Timing-Total, X-OpenAI-Inference-Capacity-Reason", "CF-Ray", "CF-Ray" ], + "set-cookie" : "__cf_bm=VxdqZg2P3RgSuMuq8lwyJVanroW9KXkA5Yt7yM7M6zw-1794874956.0517232-1.0.1.1-k7OJ3hJefuKvr._2UTbpLep6py.Xbi8DF77ozdV3EARLdnKpaVj4hxiKgsc_T5ptVBMSDvJJLI_K1TdWHzb6mbgN.yow2y2vBPZ9N0vA_OgNg2jQGNHvj7t8l9Qb8SEj; HttpOnly; SameSite=None; Partitioned; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 17 Nov 2026 00:52:37 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "Cache-Control" : "no-cache", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "430", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "text/event-stream" + } + }, + "uuid" : "6d973bc5-1d82-334b-9ea2-1b1d05b836dd", + "persistent" : true, + "insertionIndex" : 75 +} \ No newline at end of file