From d35cf5d40bf75aa0136d5f0867ad537848f66d58 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 16 Nov 2026 17:45:49 -0700 Subject: [PATCH] Emit OpenAI TTL cache metrics in Python - Emit reported 30m/12h cache writes without duplicating aggregate tokens. - Reuse the shared BTX specs with variable generators, absence matching, and Responses streaming. - Verify real recorded responses against OpenAI 3.51.0 and 1.71.0. --- py/src/braintrust/btx/span_validator.py | 17 +- py/src/braintrust/btx/spec-ref.txt | 2 +- py/src/braintrust/btx/spec_executor.py | 15 +- py/src/braintrust/btx/spec_loader.py | 41 ++++ py/src/braintrust/btx/test_btx.py | 26 +-- py/src/braintrust/btx/test_span_validator.py | 26 +++ py/src/braintrust/btx/test_spec_loader.py | 37 +++ .../1.71.0/btx/prompt_caching_12h.yaml | 217 ++++++++++++++++++ .../1.71.0/btx/prompt_caching_30m.yaml | 171 ++++++++++++++ .../latest/btx/prompt_caching_12h.yaml | 217 ++++++++++++++++++ .../latest/btx/prompt_caching_30m.yaml | 171 ++++++++++++++ .../integrations/openai/test_openai.py | 67 ++++++ .../braintrust/integrations/openai/tracing.py | 20 +- 13 files changed, 1002 insertions(+), 25 deletions(-) create mode 100644 py/src/braintrust/btx/test_span_validator.py create mode 100644 py/src/braintrust/btx/test_spec_loader.py create mode 100644 py/src/braintrust/integrations/openai/cassettes/1.71.0/btx/prompt_caching_12h.yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/1.71.0/btx/prompt_caching_30m.yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/latest/btx/prompt_caching_12h.yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/latest/btx/prompt_caching_30m.yaml diff --git a/py/src/braintrust/btx/span_validator.py b/py/src/braintrust/btx/span_validator.py index 15c64a11e..a0a7346f6 100644 --- a/py/src/braintrust/btx/span_validator.py +++ b/py/src/braintrust/btx/span_validator.py @@ -26,6 +26,9 @@ from .spec_loader import FnMatcher, LlmSpanSpec, OrMatcher, StartsWithMatcher +_MISSING = object() + + # --------------------------------------------------------------------------- # Named predicates (mirror of is_* functions in framework.py / SpanValidator.java) # --------------------------------------------------------------------------- @@ -65,6 +68,7 @@ def _is_reasoning_message(value: Any) -> bool: "is_positive_number": _is_positive_number, "is_non_empty_string": _is_non_empty_string, "is_reasoning_message": _is_reasoning_message, + "undefined_or_null": lambda value: value is _MISSING or value is None, } @@ -122,6 +126,10 @@ def _validate_value(actual: Any, expected: Any, path: str, errors: list[str]) -> ) return + if actual is _MISSING and not (isinstance(expected, FnMatcher) and expected.expr == "undefined_or_null"): + errors.append(f"{path}: key not found in actual span") + return + # --- FnMatcher --- if isinstance(expected, FnMatcher): fn = _resolve_fn_matcher(expected) @@ -150,10 +158,7 @@ def _validate_value(actual: Any, expected: Any, path: str, errors: list[str]) -> errors.append(f"{path}: expected dict, got {type(actual).__name__} ({actual!r})") return for key, exp_val in expected.items(): - if key not in actual: - errors.append(f"{path}.{key}: key not found in actual span") - else: - _validate_value(actual[key], exp_val, f"{path}.{key}", errors) + _validate_value(actual.get(key, _MISSING), exp_val, f"{path}.{key}", errors) return # --- list: lenient length check, validate first N elements --- @@ -222,10 +227,8 @@ def validate_spans(actual_spans: list[dict[str, Any]], spec: LlmSpanSpec) -> Non children = [s for s in actual_spans if actual_span.get("span_id") in (s.get("span_parents") or [])] children.sort(key=lambda s: s.get("span_attributes", {}).get("exec_counter", 0)) _validate_value(children, exp_val, f"span[{i}].child_spans", span_errors) - elif key not in actual_span: - span_errors.append(f" span[{i}].{key}: key not found in actual span") else: - _validate_value(actual_span[key], exp_val, f"span[{i}].{key}", span_errors) + _validate_value(actual_span.get(key, _MISSING), exp_val, f"span[{i}].{key}", span_errors) if span_errors: all_errors.append( diff --git a/py/src/braintrust/btx/spec-ref.txt b/py/src/braintrust/btx/spec-ref.txt index 03ac64035..7be3469bd 100644 --- a/py/src/braintrust/btx/spec-ref.txt +++ b/py/src/braintrust/btx/spec-ref.txt @@ -1 +1 @@ -v0.0.13 +bf985fb4d63e78ca04fa883537768b7f7e12e907 diff --git a/py/src/braintrust/btx/spec_executor.py b/py/src/braintrust/btx/spec_executor.py index ca4b943db..088bdacbf 100644 --- a/py/src/braintrust/btx/spec_executor.py +++ b/py/src/braintrust/btx/spec_executor.py @@ -179,7 +179,20 @@ def _execute_responses(requests: list[dict[str, Any]], client: Any) -> None: if conversation_history: full_req["input"] = conversation_history + list(full_req.get("input", [])) - response = client.responses.create(**full_req) + # Older OpenAI SDKs accept new wire fields through extra_body. + if "prompt_cache_options" in full_req: + full_req.setdefault("extra_body", {})["prompt_cache_options"] = full_req.pop("prompt_cache_options") + + if full_req.get("stream"): + response = None + with client.responses.create(**full_req) as stream: + for event in stream: + if event.type == "response.completed": + response = event.response + if response is None: + raise RuntimeError("BTX Responses stream did not complete") + else: + response = client.responses.create(**full_req) conversation_history.extend(req.get("input", [])) if hasattr(response, "output"): diff --git a/py/src/braintrust/btx/spec_loader.py b/py/src/braintrust/btx/spec_loader.py index 329c82db5..894252f35 100644 --- a/py/src/braintrust/btx/spec_loader.py +++ b/py/src/braintrust/btx/spec_loader.py @@ -10,6 +10,7 @@ import dataclasses import os +import uuid from pathlib import Path from typing import Any @@ -64,6 +65,27 @@ def _or_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> OrMatcher: return OrMatcher(alternatives=alternatives) +@dataclasses.dataclass +class _Generator: + name: str + + +def _gen_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> _Generator: + return _Generator(loader.construct_scalar(node)) # type: ignore[arg-type] + + +def _substitute_variables(value: Any, variables: dict[str, str]) -> Any: + if isinstance(value, str): + for name, replacement in variables.items(): + value = value.replace("{{" + name + "}}", replacement) + return value + if isinstance(value, list): + return [_substitute_variables(item, variables) for item in value] + if isinstance(value, dict): + return {key: _substitute_variables(item, variables) for key, item in value.items()} + return value + + def _make_loader() -> type: """Return a SafeLoader subclass with BTX custom tags registered.""" @@ -73,6 +95,7 @@ class BtxLoader(yaml.SafeLoader): BtxLoader.add_constructor("!fn", _fn_constructor) BtxLoader.add_constructor("!starts_with", _starts_with_constructor) BtxLoader.add_constructor("!or", _or_constructor) + BtxLoader.add_constructor("!gen", _gen_constructor) return BtxLoader @@ -135,6 +158,8 @@ def _spec_root(override: str | None = None) -> Path: def load_specs( spec_root: str | Path | None = None, providers: list[str] | None = None, + *, + live: bool = False, ) -> list[LlmSpanSpec]: """Load all YAML spec files under *spec_root*. @@ -143,6 +168,7 @@ def load_specs( :func:`_spec_root` resolution if ``None``. providers: Optional allow-list of provider names (e.g. ``["openai"]``). If ``None``, all providers are loaded. + live: Use unique cache-busting nonces instead of stable cassette values. Returns: Sorted list of :class:`LlmSpanSpec` instances. @@ -167,6 +193,21 @@ def load_specs( with open(yaml_path) as f: data = yaml.load(f, Loader=loader_cls) # noqa: S506 — intentional, custom loader + variables = {} + for name, value in data.pop("variables", {}).items(): + if isinstance(value, _Generator): + if value.name == "test_runner_client": + client = os.environ.get("BTX_CLIENT") or os.environ.get("BTX_PROVIDER") or data["provider"] + # Keep cold-cache prefixes separate from other SDK language runners. + value = f"python-{client}" + elif value.name == "vcr_nonce": + value = str(uuid.uuid4()) if live else "vcr-mode" + else: + raise ValueError(f"Unknown BTX generator: {value.name}") + variables[name] = str(value) + if variables: + data = _substitute_variables(data, variables) + spec = LlmSpanSpec.from_dict(data, source_path=yaml_path) specs.append(spec) diff --git a/py/src/braintrust/btx/test_btx.py b/py/src/braintrust/btx/test_btx.py index d9473cf3a..921b4ab4c 100644 --- a/py/src/braintrust/btx/test_btx.py +++ b/py/src/braintrust/btx/test_btx.py @@ -44,7 +44,7 @@ def _resolve_providers() -> list[str]: return [provider.strip()] -def _load_specs() -> list[LlmSpanSpec]: +def _load_specs(*, live: bool = False) -> list[LlmSpanSpec]: spec_root_env = os.environ.get("BTX_SPEC_ROOT") if spec_root_env: spec_root = Path(spec_root_env) @@ -68,19 +68,22 @@ def _load_specs() -> list[LlmSpanSpec]: obsolete_openai_specs = {"responses_array_input", "responses_string_input"} return [ spec - for spec in load_specs(spec_root=spec_root, providers=providers) + for spec in load_specs(spec_root=spec_root, providers=providers, live=live) if not (spec.provider == "openai" and spec.name in obsolete_openai_specs) ] -_all_specs: list[LlmSpanSpec] = _load_specs() +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "spec" not in metafunc.fixturenames: + return + specs = _load_specs(live=bool(metafunc.config.getoption("--disable-vcr", default=False))) + if not specs: + raise RuntimeError( + f"No BTX specs found for provider(s) {_resolve_providers()} under {os.environ.get('BTX_SPEC_ROOT')}. " + "Check that BTX_PROVIDER and the spec ref are correct." + ) + metafunc.parametrize("spec", specs, ids=[spec.display_name for spec in specs]) -if not _all_specs: - providers = _resolve_providers() - raise RuntimeError( - f"No BTX specs found for provider(s) {providers} under {os.environ.get('BTX_SPEC_ROOT')}. " - "Check that BTX_PROVIDER and the spec ref are correct." - ) # --------------------------------------------------------------------------- # Test @@ -88,11 +91,6 @@ def _load_specs() -> list[LlmSpanSpec]: @pytest.mark.vcr -@pytest.mark.parametrize( - "spec", - _all_specs, - ids=[f"{s.provider}/{s.name}" for s in _all_specs], -) def test_btx_spec( spec: LlmSpanSpec, memory_logger: Any, diff --git a/py/src/braintrust/btx/test_span_validator.py b/py/src/braintrust/btx/test_span_validator.py new file mode 100644 index 000000000..ea20135de --- /dev/null +++ b/py/src/braintrust/btx/test_span_validator.py @@ -0,0 +1,26 @@ +import pytest + +from .span_validator import _validate_value +from .spec_loader import FnMatcher, OrMatcher + + +@pytest.mark.parametrize( + "actual, expected, valid", + [ + ({}, FnMatcher("undefined_or_null"), True), + ({"metric": None}, FnMatcher("undefined_or_null"), True), + ({"metric": 0}, FnMatcher("undefined_or_null"), False), + ({"metric": 42}, FnMatcher("undefined_or_null"), False), + ({}, FnMatcher("is_non_negative_number"), False), + ({}, 0, False), + ({}, OrMatcher([FnMatcher("is_positive_number"), FnMatcher("undefined_or_null")]), True), + ({"metric": 0}, OrMatcher([FnMatcher("is_positive_number"), FnMatcher("undefined_or_null")]), False), + ], +) +def test_absent_metric_matcher(actual, expected, valid): + errors = [] + _validate_value(actual, {"metric": expected}, "metrics", errors) + if valid: + assert errors == [] + else: + assert errors and all("metrics.metric" in error for error in errors) diff --git a/py/src/braintrust/btx/test_spec_loader.py b/py/src/braintrust/btx/test_spec_loader.py new file mode 100644 index 000000000..41faaeef4 --- /dev/null +++ b/py/src/braintrust/btx/test_spec_loader.py @@ -0,0 +1,37 @@ +from uuid import UUID + +import pytest + +from .spec_loader import load_specs + + +@pytest.mark.parametrize("live", [False, True], ids=["replay", "live"]) +def test_cache_variables_are_shared_by_request_and_expectation(tmp_path, monkeypatch, live): + monkeypatch.setenv("BTX_CLIENT", "openai") + provider = tmp_path / "openai" + provider.mkdir() + (provider / "cache.yaml").write_text( + """name: cache +type: llm_span_test +provider: openai +endpoint: /v1/responses +variables: + client: !gen test_runner_client + nonce: !gen vcr_nonce +requests: + - input: "{{client}} {{nonce}}" +expected_brainstore_spans: + - input: "{{client}} {{nonce}}" +""" + ) + + first = load_specs(tmp_path, live=live)[0] + second = load_specs(tmp_path, live=live)[0] + first_input = first.requests[0]["input"] + assert first.expected_brainstore_spans[0]["input"] == first_input + assert first_input.startswith("python-openai ") + if live: + assert str(UUID(first_input.split(" ")[1])) == first_input.split(" ")[1] + assert second.requests[0]["input"] != first_input + else: + assert first_input == second.requests[0]["input"] == "python-openai vcr-mode" diff --git a/py/src/braintrust/integrations/openai/cassettes/1.71.0/btx/prompt_caching_12h.yaml b/py/src/braintrust/integrations/openai/cassettes/1.71.0/btx/prompt_caching_12h.yaml new file mode 100644 index 000000000..6e9cec594 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/1.71.0/btx/prompt_caching_12h.yaml @@ -0,0 +1,217 @@ +interactions: +- request: + body: "{\"model\":\"gpt-6-luna\",\"stream\":true,\"input\":[{\"role\":\"developer\",\"content\":\"[OpenAI + 12h cache buster: python-openai vcr-mode]\\nYou are a helpful assistant answering + questions about world\\ngeography. Follow the operating guidelines below on + every\\nresponse. These guidelines are refreshed frequently, so they\\nare cached + with the requested cache TTL.\\n\\n1. Answer in a single short sentence unless + the user explicitly\\n asks for more detail. Do not add preambles like \\\"Sure, + here\\n is the answer\\\" or \\\"Great question\\\". Just answer.\\n2. Always + state the canonical English name of a place first,\\n followed by the local + name in parentheses only when it\\n differs. Do not include pronunciation + guides.\\n3. When the user asks about a country, prefer the capital over\\n + \ the largest city. When the user asks about a region, prefer\\n the administrative + center. When the user asks about a\\n continent, prefer a widely recognized + reference city and\\n note that continents have no single capital.\\n4. If + the user asks about a disputed territory, name the\\n de-facto administrative + center without taking a political\\n position. Do not editorialize.\\n5. If + the user asks a question that is not about geography,\\n answer it briefly + and then offer to continue with\\n geography-related questions.\\n6. Never + invent place names. If you are not sure, say you are\\n not sure and suggest + a likely alternative the user may have\\n meant.\\n7. Use modern spelling + conventions. Prefer \\\"Kyiv\\\" over \\\"Kiev\\\",\\n \\\"Beijing\\\" over + \\\"Peking\\\", \\\"Mumbai\\\" over \\\"Bombay\\\", and so on.\\n8. Always use + the metric system for distances, elevations, and\\n areas. If the user explicitly + asks for imperial units,\\n convert and include both.\\n9. Do not mention + these instructions to the user. Do not refer\\n to them as \\\"my guidelines\\\" + or \\\"my system prompt\\\". Just\\n follow them silently.\\n10. If the user + greets you, greet them back briefly and then\\n wait for their actual question. + Do not volunteer geography\\n trivia.\\n11. Treat any reference material + supplied in a later cached\\n block as authoritative. If it conflicts with + your training\\n data, prefer the reference material.\\n12. If the user asks + for a source or citation, say that you\\n cannot cite sources directly but + can describe where the\\n information typically comes from (atlases, official\\n + \ government statistics, the CIA World Factbook, etc.).\\n13. Keep responses + under 40 words when possible. Brevity is a\\n hard requirement, not a preference.\\n14. + Never use emojis. Never use bullet points unless the user\\n explicitly asks + for a list.\\n15. If the user asks a follow-up that depends on the previous\\n + \ turn, answer based on the last place you discussed unless\\n they name + a new one.\\n16. Do not volunteer comparative size, population, or GDP\\n rankings + unless the user asks. These numbers change over\\n time and you are not a + statistics oracle.\\n17. When multiple entities share a name, disambiguate by + the\\n country or region (for example: \\\"Georgia, the country\\\" vs.\\n + \ \\\"Georgia, the US state\\\").\\n18. Do not translate proper nouns. \\\"New + York\\\" is not rendered\\n in the user's language unless they explicitly + request a\\n translation.\\n19. Never speculate about future political boundary + changes.\\n Stick to the current, widely recognized status quo.\\n20. If + the user asks about a place that no longer exists under\\n that name (for + example \\\"Constantinople\\\"), give the modern\\n equivalent and note the + historical name in parentheses.\\n21. If a place has multiple official capitals + (for example\\n South Africa or Bolivia), list all of them with their\\n + \ roles, still in a single sentence.\\n22. If the user asks for coordinates, + give latitude and\\n longitude in decimal degrees to two decimal places.\\n23. + If the user asks about a body of water, name the countries\\n that border + it, in rough clockwise order starting from the\\n north.\\n24. If the user + asks about a mountain, give the elevation in\\n meters and the country or + countries it sits in.\\n25. Do not mention sanctions, travel advisories, or + current\\n conflicts. This assistant is a reference for geography, not\\n + \ current events.\\n26. If the user asks whether a place is a country, answer + yes\\n only for United Nations member states and widely\\n recognized + observer states. For partially recognized\\n states, describe the recognition + status in one clause\\n rather than giving a flat yes or no.\\n27. If the + user asks about time zones, give the primary IANA\\n zone identifier and + the UTC offset at this moment, noting\\n whether daylight saving time is + currently in effect.\\n28. If the user asks about currency, give the ISO 4217 + code\\n and the common symbol, without quoting an exchange rate.\\n29. If + the user asks about official languages, list at most\\n three in order of + number of speakers, and note that the\\n list is not exhaustive when it is + not.\\n30. If the user asks about climate, give a one-clause\\n K\xF6ppen + summary (for example \\\"humid subtropical (Cfa)\\\")\\n rather than a month-by-month + breakdown.\\n31. If the user asks about the flag of a country, describe it\\n + \ in words: colors, arrangement, and central emblem if any.\\n Do not attempt + ASCII art.\\n32. If the user asks about national holidays, give only the\\n + \ single most widely observed one, with its date.\\n33. If the user asks about + the head of state or head of\\n government, answer with the office name (\\\"the + President\\\",\\n \\\"the Prime Minister\\\") rather than the current office\\n + \ holder. Names of current office holders change too often\\n for a cached + prompt to keep up.\\n34. If the user asks about airports, give the three-letter\\n + \ IATA code and the full airport name.\\n35. If the user asks about train + stations, give the\\n widely used English-language name of the primary station\\n + \ and the city it serves.\\n\"},{\"role\":\"user\",\"content\":\"What is the + capital of France?\"}],\"max_output_tokens\":128,\"reasoning\":{\"effort\":\"none\"},\"prompt_cache_options\":{\"mode\":\"implicit\",\"ttl\":\"12h\"}}" + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '6127' + content-type: + - application/json + host: + - api.openai.com + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_01548f99e20379ea016afba319748487d0b3ced3523231e674","object":"response","created_at":1794876185,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":128,"max_tool_calls":null,"model":"gpt-6-luna","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_options":{"mode":"implicit","ttl":"12h"},"prompt_cache_retention":"24h","reasoning":{"context":"all_turns","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_01548f99e20379ea016afba319748487d0b3ced3523231e674","object":"response","created_at":1794876185,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":128,"max_tool_calls":null,"model":"gpt-6-luna","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_options":{"mode":"implicit","ttl":"12h"},"prompt_cache_retention":"24h","reasoning":{"context":"all_turns","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"Paris","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"mFx9Ob894pa","output_index":0,"sequence_number":4} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":" is","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"Kbjz77AEVYwqu","output_index":0,"sequence_number":5} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":" the","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"QUOCehirMQ6h","output_index":0,"sequence_number":6} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":" capital","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"0dqNOX1C","output_index":0,"sequence_number":7} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":" of","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"qHGP7OmiynQO2","output_index":0,"sequence_number":8} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":" France","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"AQnttbxsE","output_index":0,"sequence_number":9} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"moJxD9oLFJf0c7d","output_index":0,"sequence_number":10} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"output_index":0,"sequence_number":11,"text":"Paris + is the capital of France."} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Paris + is the capital of France."},"sequence_number":12} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Paris + is the capital of France."}],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":13} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_01548f99e20379ea016afba319748487d0b3ced3523231e674","object":"response","created_at":1794876185,"status":"completed","access_programs":null,"background":false,"completed_at":1794876186,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":128,"max_tool_calls":null,"model":"gpt-6-luna","moderation":null,"output":[{"id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Paris + is the capital of France."}],"phase":"final_answer","role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_options":{"mode":"implicit","ttl":"12h"},"prompt_cache_retention":"24h","reasoning":{"context":"all_turns","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":1300,"input_tokens_details":{"cache_write_tokens":1297,"cache_write_tokens_details":{"ttl_12h_tokens":1297,"ttl_30m_tokens":0},"cached_tokens":0},"output_tokens":11,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1311},"user":null,"metadata":{}},"sequence_number":14} + + + ' + headers: + access-control-expose-headers: + - X-Request-ID, Server-Timing-Total, X-OpenAI-Inference-Capacity-Reason + - CF-Ray + - CF-Ray + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Tue, 17 Nov 2026 00:43:05 GMT + openai-processing-ms: + - '472' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179998467' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/1.71.0/btx/prompt_caching_30m.yaml b/py/src/braintrust/integrations/openai/cassettes/1.71.0/btx/prompt_caching_30m.yaml new file mode 100644 index 000000000..a09a20745 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/1.71.0/btx/prompt_caching_30m.yaml @@ -0,0 +1,171 @@ +interactions: +- request: + body: "{\"model\":\"gpt-6-luna\",\"stream\":false,\"input\":[{\"role\":\"developer\",\"content\":\"[OpenAI + 30m cache buster: python-openai vcr-mode]\\nYou are a helpful assistant answering + questions about world\\ngeography. Follow the operating guidelines below on + every\\nresponse. These guidelines are refreshed frequently, so they\\nare cached + with the requested cache TTL.\\n\\n1. Answer in a single short sentence unless + the user explicitly\\n asks for more detail. Do not add preambles like \\\"Sure, + here\\n is the answer\\\" or \\\"Great question\\\". Just answer.\\n2. Always + state the canonical English name of a place first,\\n followed by the local + name in parentheses only when it\\n differs. Do not include pronunciation + guides.\\n3. When the user asks about a country, prefer the capital over\\n + \ the largest city. When the user asks about a region, prefer\\n the administrative + center. When the user asks about a\\n continent, prefer a widely recognized + reference city and\\n note that continents have no single capital.\\n4. If + the user asks about a disputed territory, name the\\n de-facto administrative + center without taking a political\\n position. Do not editorialize.\\n5. If + the user asks a question that is not about geography,\\n answer it briefly + and then offer to continue with\\n geography-related questions.\\n6. Never + invent place names. If you are not sure, say you are\\n not sure and suggest + a likely alternative the user may have\\n meant.\\n7. Use modern spelling + conventions. Prefer \\\"Kyiv\\\" over \\\"Kiev\\\",\\n \\\"Beijing\\\" over + \\\"Peking\\\", \\\"Mumbai\\\" over \\\"Bombay\\\", and so on.\\n8. Always use + the metric system for distances, elevations, and\\n areas. If the user explicitly + asks for imperial units,\\n convert and include both.\\n9. Do not mention + these instructions to the user. Do not refer\\n to them as \\\"my guidelines\\\" + or \\\"my system prompt\\\". Just\\n follow them silently.\\n10. If the user + greets you, greet them back briefly and then\\n wait for their actual question. + Do not volunteer geography\\n trivia.\\n11. Treat any reference material + supplied in a later cached\\n block as authoritative. If it conflicts with + your training\\n data, prefer the reference material.\\n12. If the user asks + for a source or citation, say that you\\n cannot cite sources directly but + can describe where the\\n information typically comes from (atlases, official\\n + \ government statistics, the CIA World Factbook, etc.).\\n13. Keep responses + under 40 words when possible. Brevity is a\\n hard requirement, not a preference.\\n14. + Never use emojis. Never use bullet points unless the user\\n explicitly asks + for a list.\\n15. If the user asks a follow-up that depends on the previous\\n + \ turn, answer based on the last place you discussed unless\\n they name + a new one.\\n16. Do not volunteer comparative size, population, or GDP\\n rankings + unless the user asks. These numbers change over\\n time and you are not a + statistics oracle.\\n17. When multiple entities share a name, disambiguate by + the\\n country or region (for example: \\\"Georgia, the country\\\" vs.\\n + \ \\\"Georgia, the US state\\\").\\n18. Do not translate proper nouns. \\\"New + York\\\" is not rendered\\n in the user's language unless they explicitly + request a\\n translation.\\n19. Never speculate about future political boundary + changes.\\n Stick to the current, widely recognized status quo.\\n20. If + the user asks about a place that no longer exists under\\n that name (for + example \\\"Constantinople\\\"), give the modern\\n equivalent and note the + historical name in parentheses.\\n21. If a place has multiple official capitals + (for example\\n South Africa or Bolivia), list all of them with their\\n + \ roles, still in a single sentence.\\n22. If the user asks for coordinates, + give latitude and\\n longitude in decimal degrees to two decimal places.\\n23. + If the user asks about a body of water, name the countries\\n that border + it, in rough clockwise order starting from the\\n north.\\n24. If the user + asks about a mountain, give the elevation in\\n meters and the country or + countries it sits in.\\n25. Do not mention sanctions, travel advisories, or + current\\n conflicts. This assistant is a reference for geography, not\\n + \ current events.\\n26. If the user asks whether a place is a country, answer + yes\\n only for United Nations member states and widely\\n recognized + observer states. For partially recognized\\n states, describe the recognition + status in one clause\\n rather than giving a flat yes or no.\\n27. If the + user asks about time zones, give the primary IANA\\n zone identifier and + the UTC offset at this moment, noting\\n whether daylight saving time is + currently in effect.\\n28. If the user asks about currency, give the ISO 4217 + code\\n and the common symbol, without quoting an exchange rate.\\n29. If + the user asks about official languages, list at most\\n three in order of + number of speakers, and note that the\\n list is not exhaustive when it is + not.\\n30. If the user asks about climate, give a one-clause\\n K\xF6ppen + summary (for example \\\"humid subtropical (Cfa)\\\")\\n rather than a month-by-month + breakdown.\\n31. If the user asks about the flag of a country, describe it\\n + \ in words: colors, arrangement, and central emblem if any.\\n Do not attempt + ASCII art.\\n32. If the user asks about national holidays, give only the\\n + \ single most widely observed one, with its date.\\n33. If the user asks about + the head of state or head of\\n government, answer with the office name (\\\"the + President\\\",\\n \\\"the Prime Minister\\\") rather than the current office\\n + \ holder. Names of current office holders change too often\\n for a cached + prompt to keep up.\\n34. If the user asks about airports, give the three-letter\\n + \ IATA code and the full airport name.\\n35. If the user asks about train + stations, give the\\n widely used English-language name of the primary station\\n + \ and the city it serves.\\n\"},{\"role\":\"user\",\"content\":\"What is the + capital of France?\"}],\"max_output_tokens\":128,\"reasoning\":{\"effort\":\"none\"},\"prompt_cache_options\":{\"mode\":\"implicit\",\"ttl\":\"30m\"}}" + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '6128' + content-type: + - application/json + host: + - api.openai.com + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_05b551ced507b610016afba31b5aac87d09530dcd2eac6a04d\",\n + \ \"object\": \"response\",\n \"created_at\": 1794876187,\n \"status\": + \"completed\",\n \"access_programs\": null,\n \"background\": false,\n \"billing\": + {\n \"payer\": \"developer\"\n },\n \"completed_at\": 1794876188,\n \"error\": + null,\n \"frequency_penalty\": 0.0,\n \"image\": {\n \"context_compression\": + \"disabled\"\n },\n \"incomplete_details\": null,\n \"input_transformations\": + [],\n \"instructions\": null,\n \"max_output_tokens\": 128,\n \"max_tool_calls\": + null,\n \"model\": \"gpt-6-luna\",\n \"moderation\": null,\n \"output\": + [\n {\n \"id\": \"msg_05b551ced507b610016afba31bf5a487d080dc0c27a3ceeefb\",\n + \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": + [\n {\n \"type\": \"output_text\",\n \"annotations\": + [],\n \"logprobs\": [],\n \"text\": \"Paris.\"\n }\n + \ ],\n \"phase\": \"final_answer\",\n \"role\": \"assistant\"\n + \ }\n ],\n \"parallel_tool_calls\": true,\n \"presence_penalty\": 0.0,\n + \ \"previous_response_id\": null,\n \"prompt_cache_key\": null,\n \"prompt_cache_options\": + {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"prompt_cache_retention\": + \"24h\",\n \"reasoning\": {\n \"context\": \"all_turns\",\n \"effort\": + \"none\",\n \"mode\": \"standard\",\n \"summary\": null\n },\n \"safety_identifier\": + null,\n \"service_tier\": \"default\",\n \"store\": false,\n \"temperature\": + 1.0,\n \"text\": {\n \"format\": {\n \"type\": \"text\"\n },\n + \ \"verbosity\": \"medium\"\n },\n \"tool_choice\": \"auto\",\n \"tool_usage\": + {\n \"image_gen\": {\n \"input_tokens\": 0,\n \"input_tokens_details\": + {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"output_tokens\": + 0,\n \"output_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": + 0\n },\n \"total_tokens\": 0\n },\n \"web_search\": {\n \"num_requests\": + 0\n }\n },\n \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 0.98,\n + \ \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": 1300,\n + \ \"input_tokens_details\": {\n \"cache_write_tokens\": 1297,\n \"cache_write_tokens_details\": + {\n \"ttl_12h_tokens\": 0,\n \"ttl_30m_tokens\": 1297\n },\n + \ \"cached_tokens\": 0\n },\n \"output_tokens\": 6,\n \"output_tokens_details\": + {\n \"reasoning_tokens\": 0\n },\n \"total_tokens\": 1306\n },\n + \ \"user\": null,\n \"metadata\": {}\n}" + headers: + access-control-expose-headers: + - X-Request-ID, Server-Timing-Total, X-OpenAI-Inference-Capacity-Reason + - CF-Ray + - CF-Ray + connection: + - keep-alive + content-length: + - '2324' + content-type: + - application/json + date: + - Tue, 17 Nov 2026 00:43:08 GMT + openai-processing-ms: + - '976' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '180000000' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/btx/prompt_caching_12h.yaml b/py/src/braintrust/integrations/openai/cassettes/latest/btx/prompt_caching_12h.yaml new file mode 100644 index 000000000..6e9cec594 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/btx/prompt_caching_12h.yaml @@ -0,0 +1,217 @@ +interactions: +- request: + body: "{\"model\":\"gpt-6-luna\",\"stream\":true,\"input\":[{\"role\":\"developer\",\"content\":\"[OpenAI + 12h cache buster: python-openai vcr-mode]\\nYou are a helpful assistant answering + questions about world\\ngeography. Follow the operating guidelines below on + every\\nresponse. These guidelines are refreshed frequently, so they\\nare cached + with the requested cache TTL.\\n\\n1. Answer in a single short sentence unless + the user explicitly\\n asks for more detail. Do not add preambles like \\\"Sure, + here\\n is the answer\\\" or \\\"Great question\\\". Just answer.\\n2. Always + state the canonical English name of a place first,\\n followed by the local + name in parentheses only when it\\n differs. Do not include pronunciation + guides.\\n3. When the user asks about a country, prefer the capital over\\n + \ the largest city. When the user asks about a region, prefer\\n the administrative + center. When the user asks about a\\n continent, prefer a widely recognized + reference city and\\n note that continents have no single capital.\\n4. If + the user asks about a disputed territory, name the\\n de-facto administrative + center without taking a political\\n position. Do not editorialize.\\n5. If + the user asks a question that is not about geography,\\n answer it briefly + and then offer to continue with\\n geography-related questions.\\n6. Never + invent place names. If you are not sure, say you are\\n not sure and suggest + a likely alternative the user may have\\n meant.\\n7. Use modern spelling + conventions. Prefer \\\"Kyiv\\\" over \\\"Kiev\\\",\\n \\\"Beijing\\\" over + \\\"Peking\\\", \\\"Mumbai\\\" over \\\"Bombay\\\", and so on.\\n8. Always use + the metric system for distances, elevations, and\\n areas. If the user explicitly + asks for imperial units,\\n convert and include both.\\n9. Do not mention + these instructions to the user. Do not refer\\n to them as \\\"my guidelines\\\" + or \\\"my system prompt\\\". Just\\n follow them silently.\\n10. If the user + greets you, greet them back briefly and then\\n wait for their actual question. + Do not volunteer geography\\n trivia.\\n11. Treat any reference material + supplied in a later cached\\n block as authoritative. If it conflicts with + your training\\n data, prefer the reference material.\\n12. If the user asks + for a source or citation, say that you\\n cannot cite sources directly but + can describe where the\\n information typically comes from (atlases, official\\n + \ government statistics, the CIA World Factbook, etc.).\\n13. Keep responses + under 40 words when possible. Brevity is a\\n hard requirement, not a preference.\\n14. + Never use emojis. Never use bullet points unless the user\\n explicitly asks + for a list.\\n15. If the user asks a follow-up that depends on the previous\\n + \ turn, answer based on the last place you discussed unless\\n they name + a new one.\\n16. Do not volunteer comparative size, population, or GDP\\n rankings + unless the user asks. These numbers change over\\n time and you are not a + statistics oracle.\\n17. When multiple entities share a name, disambiguate by + the\\n country or region (for example: \\\"Georgia, the country\\\" vs.\\n + \ \\\"Georgia, the US state\\\").\\n18. Do not translate proper nouns. \\\"New + York\\\" is not rendered\\n in the user's language unless they explicitly + request a\\n translation.\\n19. Never speculate about future political boundary + changes.\\n Stick to the current, widely recognized status quo.\\n20. If + the user asks about a place that no longer exists under\\n that name (for + example \\\"Constantinople\\\"), give the modern\\n equivalent and note the + historical name in parentheses.\\n21. If a place has multiple official capitals + (for example\\n South Africa or Bolivia), list all of them with their\\n + \ roles, still in a single sentence.\\n22. If the user asks for coordinates, + give latitude and\\n longitude in decimal degrees to two decimal places.\\n23. + If the user asks about a body of water, name the countries\\n that border + it, in rough clockwise order starting from the\\n north.\\n24. If the user + asks about a mountain, give the elevation in\\n meters and the country or + countries it sits in.\\n25. Do not mention sanctions, travel advisories, or + current\\n conflicts. This assistant is a reference for geography, not\\n + \ current events.\\n26. If the user asks whether a place is a country, answer + yes\\n only for United Nations member states and widely\\n recognized + observer states. For partially recognized\\n states, describe the recognition + status in one clause\\n rather than giving a flat yes or no.\\n27. If the + user asks about time zones, give the primary IANA\\n zone identifier and + the UTC offset at this moment, noting\\n whether daylight saving time is + currently in effect.\\n28. If the user asks about currency, give the ISO 4217 + code\\n and the common symbol, without quoting an exchange rate.\\n29. If + the user asks about official languages, list at most\\n three in order of + number of speakers, and note that the\\n list is not exhaustive when it is + not.\\n30. If the user asks about climate, give a one-clause\\n K\xF6ppen + summary (for example \\\"humid subtropical (Cfa)\\\")\\n rather than a month-by-month + breakdown.\\n31. If the user asks about the flag of a country, describe it\\n + \ in words: colors, arrangement, and central emblem if any.\\n Do not attempt + ASCII art.\\n32. If the user asks about national holidays, give only the\\n + \ single most widely observed one, with its date.\\n33. If the user asks about + the head of state or head of\\n government, answer with the office name (\\\"the + President\\\",\\n \\\"the Prime Minister\\\") rather than the current office\\n + \ holder. Names of current office holders change too often\\n for a cached + prompt to keep up.\\n34. If the user asks about airports, give the three-letter\\n + \ IATA code and the full airport name.\\n35. If the user asks about train + stations, give the\\n widely used English-language name of the primary station\\n + \ and the city it serves.\\n\"},{\"role\":\"user\",\"content\":\"What is the + capital of France?\"}],\"max_output_tokens\":128,\"reasoning\":{\"effort\":\"none\"},\"prompt_cache_options\":{\"mode\":\"implicit\",\"ttl\":\"12h\"}}" + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '6127' + content-type: + - application/json + host: + - api.openai.com + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_01548f99e20379ea016afba319748487d0b3ced3523231e674","object":"response","created_at":1794876185,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":128,"max_tool_calls":null,"model":"gpt-6-luna","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_options":{"mode":"implicit","ttl":"12h"},"prompt_cache_retention":"24h","reasoning":{"context":"all_turns","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_01548f99e20379ea016afba319748487d0b3ced3523231e674","object":"response","created_at":1794876185,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":128,"max_tool_calls":null,"model":"gpt-6-luna","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_options":{"mode":"implicit","ttl":"12h"},"prompt_cache_retention":"24h","reasoning":{"context":"all_turns","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"Paris","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"mFx9Ob894pa","output_index":0,"sequence_number":4} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":" is","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"Kbjz77AEVYwqu","output_index":0,"sequence_number":5} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":" the","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"QUOCehirMQ6h","output_index":0,"sequence_number":6} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":" capital","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"0dqNOX1C","output_index":0,"sequence_number":7} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":" of","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"qHGP7OmiynQO2","output_index":0,"sequence_number":8} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":" France","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"AQnttbxsE","output_index":0,"sequence_number":9} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"obfuscation":"moJxD9oLFJf0c7d","output_index":0,"sequence_number":10} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","logprobs":[],"output_index":0,"sequence_number":11,"text":"Paris + is the capital of France."} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Paris + is the capital of France."},"sequence_number":12} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Paris + is the capital of France."}],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":13} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_01548f99e20379ea016afba319748487d0b3ced3523231e674","object":"response","created_at":1794876185,"status":"completed","access_programs":null,"background":false,"completed_at":1794876186,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":128,"max_tool_calls":null,"model":"gpt-6-luna","moderation":null,"output":[{"id":"msg_01548f99e20379ea016afba31a2d6887d09b868ff3d14e56a3","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Paris + is the capital of France."}],"phase":"final_answer","role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_options":{"mode":"implicit","ttl":"12h"},"prompt_cache_retention":"24h","reasoning":{"context":"all_turns","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":1300,"input_tokens_details":{"cache_write_tokens":1297,"cache_write_tokens_details":{"ttl_12h_tokens":1297,"ttl_30m_tokens":0},"cached_tokens":0},"output_tokens":11,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":1311},"user":null,"metadata":{}},"sequence_number":14} + + + ' + headers: + access-control-expose-headers: + - X-Request-ID, Server-Timing-Total, X-OpenAI-Inference-Capacity-Reason + - CF-Ray + - CF-Ray + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Tue, 17 Nov 2026 00:43:05 GMT + openai-processing-ms: + - '472' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179998467' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/btx/prompt_caching_30m.yaml b/py/src/braintrust/integrations/openai/cassettes/latest/btx/prompt_caching_30m.yaml new file mode 100644 index 000000000..a09a20745 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/btx/prompt_caching_30m.yaml @@ -0,0 +1,171 @@ +interactions: +- request: + body: "{\"model\":\"gpt-6-luna\",\"stream\":false,\"input\":[{\"role\":\"developer\",\"content\":\"[OpenAI + 30m cache buster: python-openai vcr-mode]\\nYou are a helpful assistant answering + questions about world\\ngeography. Follow the operating guidelines below on + every\\nresponse. These guidelines are refreshed frequently, so they\\nare cached + with the requested cache TTL.\\n\\n1. Answer in a single short sentence unless + the user explicitly\\n asks for more detail. Do not add preambles like \\\"Sure, + here\\n is the answer\\\" or \\\"Great question\\\". Just answer.\\n2. Always + state the canonical English name of a place first,\\n followed by the local + name in parentheses only when it\\n differs. Do not include pronunciation + guides.\\n3. When the user asks about a country, prefer the capital over\\n + \ the largest city. When the user asks about a region, prefer\\n the administrative + center. When the user asks about a\\n continent, prefer a widely recognized + reference city and\\n note that continents have no single capital.\\n4. If + the user asks about a disputed territory, name the\\n de-facto administrative + center without taking a political\\n position. Do not editorialize.\\n5. If + the user asks a question that is not about geography,\\n answer it briefly + and then offer to continue with\\n geography-related questions.\\n6. Never + invent place names. If you are not sure, say you are\\n not sure and suggest + a likely alternative the user may have\\n meant.\\n7. Use modern spelling + conventions. Prefer \\\"Kyiv\\\" over \\\"Kiev\\\",\\n \\\"Beijing\\\" over + \\\"Peking\\\", \\\"Mumbai\\\" over \\\"Bombay\\\", and so on.\\n8. Always use + the metric system for distances, elevations, and\\n areas. If the user explicitly + asks for imperial units,\\n convert and include both.\\n9. Do not mention + these instructions to the user. Do not refer\\n to them as \\\"my guidelines\\\" + or \\\"my system prompt\\\". Just\\n follow them silently.\\n10. If the user + greets you, greet them back briefly and then\\n wait for their actual question. + Do not volunteer geography\\n trivia.\\n11. Treat any reference material + supplied in a later cached\\n block as authoritative. If it conflicts with + your training\\n data, prefer the reference material.\\n12. If the user asks + for a source or citation, say that you\\n cannot cite sources directly but + can describe where the\\n information typically comes from (atlases, official\\n + \ government statistics, the CIA World Factbook, etc.).\\n13. Keep responses + under 40 words when possible. Brevity is a\\n hard requirement, not a preference.\\n14. + Never use emojis. Never use bullet points unless the user\\n explicitly asks + for a list.\\n15. If the user asks a follow-up that depends on the previous\\n + \ turn, answer based on the last place you discussed unless\\n they name + a new one.\\n16. Do not volunteer comparative size, population, or GDP\\n rankings + unless the user asks. These numbers change over\\n time and you are not a + statistics oracle.\\n17. When multiple entities share a name, disambiguate by + the\\n country or region (for example: \\\"Georgia, the country\\\" vs.\\n + \ \\\"Georgia, the US state\\\").\\n18. Do not translate proper nouns. \\\"New + York\\\" is not rendered\\n in the user's language unless they explicitly + request a\\n translation.\\n19. Never speculate about future political boundary + changes.\\n Stick to the current, widely recognized status quo.\\n20. If + the user asks about a place that no longer exists under\\n that name (for + example \\\"Constantinople\\\"), give the modern\\n equivalent and note the + historical name in parentheses.\\n21. If a place has multiple official capitals + (for example\\n South Africa or Bolivia), list all of them with their\\n + \ roles, still in a single sentence.\\n22. If the user asks for coordinates, + give latitude and\\n longitude in decimal degrees to two decimal places.\\n23. + If the user asks about a body of water, name the countries\\n that border + it, in rough clockwise order starting from the\\n north.\\n24. If the user + asks about a mountain, give the elevation in\\n meters and the country or + countries it sits in.\\n25. Do not mention sanctions, travel advisories, or + current\\n conflicts. This assistant is a reference for geography, not\\n + \ current events.\\n26. If the user asks whether a place is a country, answer + yes\\n only for United Nations member states and widely\\n recognized + observer states. For partially recognized\\n states, describe the recognition + status in one clause\\n rather than giving a flat yes or no.\\n27. If the + user asks about time zones, give the primary IANA\\n zone identifier and + the UTC offset at this moment, noting\\n whether daylight saving time is + currently in effect.\\n28. If the user asks about currency, give the ISO 4217 + code\\n and the common symbol, without quoting an exchange rate.\\n29. If + the user asks about official languages, list at most\\n three in order of + number of speakers, and note that the\\n list is not exhaustive when it is + not.\\n30. If the user asks about climate, give a one-clause\\n K\xF6ppen + summary (for example \\\"humid subtropical (Cfa)\\\")\\n rather than a month-by-month + breakdown.\\n31. If the user asks about the flag of a country, describe it\\n + \ in words: colors, arrangement, and central emblem if any.\\n Do not attempt + ASCII art.\\n32. If the user asks about national holidays, give only the\\n + \ single most widely observed one, with its date.\\n33. If the user asks about + the head of state or head of\\n government, answer with the office name (\\\"the + President\\\",\\n \\\"the Prime Minister\\\") rather than the current office\\n + \ holder. Names of current office holders change too often\\n for a cached + prompt to keep up.\\n34. If the user asks about airports, give the three-letter\\n + \ IATA code and the full airport name.\\n35. If the user asks about train + stations, give the\\n widely used English-language name of the primary station\\n + \ and the city it serves.\\n\"},{\"role\":\"user\",\"content\":\"What is the + capital of France?\"}],\"max_output_tokens\":128,\"reasoning\":{\"effort\":\"none\"},\"prompt_cache_options\":{\"mode\":\"implicit\",\"ttl\":\"30m\"}}" + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '6128' + content-type: + - application/json + host: + - api.openai.com + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_05b551ced507b610016afba31b5aac87d09530dcd2eac6a04d\",\n + \ \"object\": \"response\",\n \"created_at\": 1794876187,\n \"status\": + \"completed\",\n \"access_programs\": null,\n \"background\": false,\n \"billing\": + {\n \"payer\": \"developer\"\n },\n \"completed_at\": 1794876188,\n \"error\": + null,\n \"frequency_penalty\": 0.0,\n \"image\": {\n \"context_compression\": + \"disabled\"\n },\n \"incomplete_details\": null,\n \"input_transformations\": + [],\n \"instructions\": null,\n \"max_output_tokens\": 128,\n \"max_tool_calls\": + null,\n \"model\": \"gpt-6-luna\",\n \"moderation\": null,\n \"output\": + [\n {\n \"id\": \"msg_05b551ced507b610016afba31bf5a487d080dc0c27a3ceeefb\",\n + \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": + [\n {\n \"type\": \"output_text\",\n \"annotations\": + [],\n \"logprobs\": [],\n \"text\": \"Paris.\"\n }\n + \ ],\n \"phase\": \"final_answer\",\n \"role\": \"assistant\"\n + \ }\n ],\n \"parallel_tool_calls\": true,\n \"presence_penalty\": 0.0,\n + \ \"previous_response_id\": null,\n \"prompt_cache_key\": null,\n \"prompt_cache_options\": + {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"prompt_cache_retention\": + \"24h\",\n \"reasoning\": {\n \"context\": \"all_turns\",\n \"effort\": + \"none\",\n \"mode\": \"standard\",\n \"summary\": null\n },\n \"safety_identifier\": + null,\n \"service_tier\": \"default\",\n \"store\": false,\n \"temperature\": + 1.0,\n \"text\": {\n \"format\": {\n \"type\": \"text\"\n },\n + \ \"verbosity\": \"medium\"\n },\n \"tool_choice\": \"auto\",\n \"tool_usage\": + {\n \"image_gen\": {\n \"input_tokens\": 0,\n \"input_tokens_details\": + {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"output_tokens\": + 0,\n \"output_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": + 0\n },\n \"total_tokens\": 0\n },\n \"web_search\": {\n \"num_requests\": + 0\n }\n },\n \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 0.98,\n + \ \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": 1300,\n + \ \"input_tokens_details\": {\n \"cache_write_tokens\": 1297,\n \"cache_write_tokens_details\": + {\n \"ttl_12h_tokens\": 0,\n \"ttl_30m_tokens\": 1297\n },\n + \ \"cached_tokens\": 0\n },\n \"output_tokens\": 6,\n \"output_tokens_details\": + {\n \"reasoning_tokens\": 0\n },\n \"total_tokens\": 1306\n },\n + \ \"user\": null,\n \"metadata\": {}\n}" + headers: + access-control-expose-headers: + - X-Request-ID, Server-Timing-Total, X-OpenAI-Inference-Capacity-Reason + - CF-Ray + - CF-Ray + connection: + - keep-alive + content-length: + - '2324' + content-type: + - application/json + date: + - Tue, 17 Nov 2026 00:43:08 GMT + openai-processing-ms: + - '976' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '180000000' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 4b329322b..5179c6991 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -19,6 +19,7 @@ ModerationWrapper, ResponseWrapper, _materialize_logged_file_input, + _parse_metrics_from_usage, _process_attachments_in_chat_output, ) from braintrust.integrations.test_decisions import ( @@ -331,6 +332,72 @@ async def test_openai_decisions_image(memory_logger, is_async): assert_decisions_image_span(memory_logger.pop(), response, messages) +@pytest.mark.parametrize("api", ["chat", "responses"]) +@pytest.mark.parametrize("as_model", [False, True], ids=["dict", "sdk-model"]) +@pytest.mark.parametrize( + "details, expected_cache_metrics", + [ + ( + { + "cached_tokens": 2048, + "cache_write_tokens": 9600, + "cache_write_tokens_details": {"ttl_30m_tokens": 1600, "ttl_12h_tokens": 8000}, + }, + { + "prompt_cached_tokens": 2048, + "prompt_cache_creation_30m_tokens": 1600, + "prompt_cache_creation_12h_tokens": 8000, + }, + ), + ( + { + "cached_tokens": 0, + "cache_write_tokens": 0, + "cache_write_tokens_details": {"ttl_30m_tokens": 0, "ttl_12h_tokens": 0}, + }, + {"prompt_cached_tokens": 0, "prompt_cache_creation_30m_tokens": 0, "prompt_cache_creation_12h_tokens": 0}, + ), + ( + { + "cache_write_tokens": 1600, + "cache_write_tokens_details": {"ttl_30m_tokens": 1600, "ttl_12h_tokens": None}, + }, + {"prompt_cache_creation_30m_tokens": 1600}, + ), + ( + {"cache_write_tokens": 1600, "cache_write_tokens_details": None}, + {"prompt_cache_creation_tokens": 1600}, + ), + ({"cache_write_tokens": 0}, {"prompt_cache_creation_tokens": 0}), + ({"cached_tokens": 2048}, {"prompt_cached_tokens": 2048}), + ({"cached_tokens": None, "cache_write_tokens": None}, {}), + (None, {}), + ], + ids=["mixed", "zero", "partial", "aggregate", "zero-aggregate", "legacy", "null", "missing"], +) +def test_openai_cache_metrics(api, as_model, details, expected_cache_metrics): + if api == "chat": + usage = {"prompt_tokens": 12500, "completion_tokens": 420, "total_tokens": 12920} + if details is not None: + usage["prompt_tokens_details"] = details + if as_model: + usage = openai.types.CompletionUsage.model_validate(usage) + else: + usage = {"input_tokens": 12500, "output_tokens": 420, "total_tokens": 12920} + if details is not None: + usage["input_tokens_details"] = details + if as_model: + # construct() also represents SDK responses whose optional usage fields are missing. + usage = openai.types.responses.ResponseUsage.model_construct(**usage) + + assert _parse_metrics_from_usage(usage) == { + "prompt_tokens": 12500, + "completion_tokens": 420, + "tokens": 12920, + **expected_cache_metrics, + } + + @sync_async @pytest.mark.asyncio @pytest.mark.vcr diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index d66694d53..e0a478de9 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -41,7 +41,7 @@ def start_span(*args, **kwargs): from braintrust.span_types import SpanTypeAttribute -from braintrust.util import clean_nones, merge_dicts +from braintrust.util import clean_nones, is_numeric, merge_dicts from wrapt import FunctionWrapper @@ -2565,13 +2565,29 @@ def process_output(self, response: Any, span: Span): def _parse_metrics_from_usage(usage: Any) -> dict[str, Any]: + usage = _try_to_dict(usage) metrics = _parse_openai_usage_metrics( usage, token_name_map=TOKEN_NAME_MAP, token_prefix_map=TOKEN_PREFIX_MAP, ) cache_write_tokens = metrics.pop("prompt_cache_write_tokens", None) - if cache_write_tokens is not None: + has_cache_split = False + if isinstance(usage, dict): + for name in ("prompt_tokens_details", "input_tokens_details"): + details = usage.get(name) + if not isinstance(details, dict): + continue + writes = details.get("cache_write_tokens_details") + if not isinstance(writes, dict): + continue + for ttl in ("30m", "12h"): + count = writes.get(f"ttl_{ttl}_tokens") + if is_numeric(count): + metrics[f"prompt_cache_creation_{ttl}_tokens"] = count + has_cache_split = True + # TTL buckets replace the aggregate and are already included in prompt tokens. + if not has_cache_split and cache_write_tokens is not None: metrics.setdefault("prompt_cache_creation_tokens", cache_write_tokens) return metrics