Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions py/src/braintrust/btx/span_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
from .spec_loader import FnMatcher, LlmSpanSpec, OrMatcher, StartsWithMatcher


_MISSING = object()


# ---------------------------------------------------------------------------
# Named predicates (mirror of is_* functions in framework.py / SpanValidator.java)
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -65,6 +68,7 @@ def _is_reasoning_message(value: Any) -> bool:
"is_positive_number": _is_positive_number,
"is_non_empty_string": _is_non_empty_string,
"is_reasoning_message": _is_reasoning_message,
"undefined_or_null": lambda value: value is _MISSING or value is None,
}


Expand Down Expand Up @@ -122,6 +126,10 @@ def _validate_value(actual: Any, expected: Any, path: str, errors: list[str]) ->
)
return

if actual is _MISSING and not (isinstance(expected, FnMatcher) and expected.expr == "undefined_or_null"):
errors.append(f"{path}: key not found in actual span")
return

# --- FnMatcher ---
if isinstance(expected, FnMatcher):
fn = _resolve_fn_matcher(expected)
Expand Down Expand Up @@ -150,10 +158,7 @@ def _validate_value(actual: Any, expected: Any, path: str, errors: list[str]) ->
errors.append(f"{path}: expected dict, got {type(actual).__name__} ({actual!r})")
return
for key, exp_val in expected.items():
if key not in actual:
errors.append(f"{path}.{key}: key not found in actual span")
else:
_validate_value(actual[key], exp_val, f"{path}.{key}", errors)
_validate_value(actual.get(key, _MISSING), exp_val, f"{path}.{key}", errors)
return

# --- list: lenient length check, validate first N elements ---
Expand Down Expand Up @@ -222,10 +227,8 @@ def validate_spans(actual_spans: list[dict[str, Any]], spec: LlmSpanSpec) -> Non
children = [s for s in actual_spans if actual_span.get("span_id") in (s.get("span_parents") or [])]
children.sort(key=lambda s: s.get("span_attributes", {}).get("exec_counter", 0))
_validate_value(children, exp_val, f"span[{i}].child_spans", span_errors)
elif key not in actual_span:
span_errors.append(f" span[{i}].{key}: key not found in actual span")
else:
_validate_value(actual_span[key], exp_val, f"span[{i}].{key}", span_errors)
_validate_value(actual_span.get(key, _MISSING), exp_val, f"span[{i}].{key}", span_errors)

if span_errors:
all_errors.append(
Expand Down
2 changes: 1 addition & 1 deletion py/src/braintrust/btx/spec-ref.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
v0.0.13
bf985fb4d63e78ca04fa883537768b7f7e12e907
15 changes: 14 additions & 1 deletion py/src/braintrust/btx/spec_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,20 @@ def _execute_responses(requests: list[dict[str, Any]], client: Any) -> None:
if conversation_history:
full_req["input"] = conversation_history + list(full_req.get("input", []))

response = client.responses.create(**full_req)
# Older OpenAI SDKs accept new wire fields through extra_body.
if "prompt_cache_options" in full_req:
full_req.setdefault("extra_body", {})["prompt_cache_options"] = full_req.pop("prompt_cache_options")

if full_req.get("stream"):
response = None
with client.responses.create(**full_req) as stream:
for event in stream:
if event.type == "response.completed":
response = event.response
if response is None:
raise RuntimeError("BTX Responses stream did not complete")
else:
response = client.responses.create(**full_req)

conversation_history.extend(req.get("input", []))
if hasattr(response, "output"):
Expand Down
41 changes: 41 additions & 0 deletions py/src/braintrust/btx/spec_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import dataclasses
import os
import uuid
from pathlib import Path
from typing import Any

Expand Down Expand Up @@ -64,6 +65,27 @@ def _or_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> OrMatcher:
return OrMatcher(alternatives=alternatives)


@dataclasses.dataclass
class _Generator:
name: str


def _gen_constructor(loader: yaml.SafeLoader, node: yaml.Node) -> _Generator:
return _Generator(loader.construct_scalar(node)) # type: ignore[arg-type]


def _substitute_variables(value: Any, variables: dict[str, str]) -> Any:
if isinstance(value, str):
for name, replacement in variables.items():
value = value.replace("{{" + name + "}}", replacement)
return value
if isinstance(value, list):
return [_substitute_variables(item, variables) for item in value]
if isinstance(value, dict):
return {key: _substitute_variables(item, variables) for key, item in value.items()}
return value


def _make_loader() -> type:
"""Return a SafeLoader subclass with BTX custom tags registered."""

Expand All @@ -73,6 +95,7 @@ class BtxLoader(yaml.SafeLoader):
BtxLoader.add_constructor("!fn", _fn_constructor)
BtxLoader.add_constructor("!starts_with", _starts_with_constructor)
BtxLoader.add_constructor("!or", _or_constructor)
BtxLoader.add_constructor("!gen", _gen_constructor)
return BtxLoader


Expand Down Expand Up @@ -135,6 +158,8 @@ def _spec_root(override: str | None = None) -> Path:
def load_specs(
spec_root: str | Path | None = None,
providers: list[str] | None = None,
*,
live: bool = False,
) -> list[LlmSpanSpec]:
"""Load all YAML spec files under *spec_root*.

Expand All @@ -143,6 +168,7 @@ def load_specs(
:func:`_spec_root` resolution if ``None``.
providers: Optional allow-list of provider names (e.g. ``["openai"]``).
If ``None``, all providers are loaded.
live: Use unique cache-busting nonces instead of stable cassette values.

Returns:
Sorted list of :class:`LlmSpanSpec` instances.
Expand All @@ -167,6 +193,21 @@ def load_specs(
with open(yaml_path) as f:
data = yaml.load(f, Loader=loader_cls) # noqa: S506 — intentional, custom loader

variables = {}
for name, value in data.pop("variables", {}).items():
if isinstance(value, _Generator):
if value.name == "test_runner_client":
client = os.environ.get("BTX_CLIENT") or os.environ.get("BTX_PROVIDER") or data["provider"]
# Keep cold-cache prefixes separate from other SDK language runners.
value = f"python-{client}"
elif value.name == "vcr_nonce":
value = str(uuid.uuid4()) if live else "vcr-mode"
else:
raise ValueError(f"Unknown BTX generator: {value.name}")
variables[name] = str(value)
if variables:
data = _substitute_variables(data, variables)

spec = LlmSpanSpec.from_dict(data, source_path=yaml_path)
specs.append(spec)

Expand Down
26 changes: 12 additions & 14 deletions py/src/braintrust/btx/test_btx.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def _resolve_providers() -> list[str]:
return [provider.strip()]


def _load_specs() -> list[LlmSpanSpec]:
def _load_specs(*, live: bool = False) -> list[LlmSpanSpec]:
spec_root_env = os.environ.get("BTX_SPEC_ROOT")
if spec_root_env:
spec_root = Path(spec_root_env)
Expand All @@ -68,31 +68,29 @@ def _load_specs() -> list[LlmSpanSpec]:
obsolete_openai_specs = {"responses_array_input", "responses_string_input"}
return [
spec
for spec in load_specs(spec_root=spec_root, providers=providers)
for spec in load_specs(spec_root=spec_root, providers=providers, live=live)
if not (spec.provider == "openai" and spec.name in obsolete_openai_specs)
]


_all_specs: list[LlmSpanSpec] = _load_specs()
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
if "spec" not in metafunc.fixturenames:
return
specs = _load_specs(live=bool(metafunc.config.getoption("--disable-vcr", default=False)))
if not specs:
raise RuntimeError(
f"No BTX specs found for provider(s) {_resolve_providers()} under {os.environ.get('BTX_SPEC_ROOT')}. "
"Check that BTX_PROVIDER and the spec ref are correct."
)
metafunc.parametrize("spec", specs, ids=[spec.display_name for spec in specs])

if not _all_specs:
providers = _resolve_providers()
raise RuntimeError(
f"No BTX specs found for provider(s) {providers} under {os.environ.get('BTX_SPEC_ROOT')}. "
"Check that BTX_PROVIDER and the spec ref are correct."
)

# ---------------------------------------------------------------------------
# Test
# ---------------------------------------------------------------------------


@pytest.mark.vcr
@pytest.mark.parametrize(
"spec",
_all_specs,
ids=[f"{s.provider}/{s.name}" for s in _all_specs],
)
def test_btx_spec(
spec: LlmSpanSpec,
memory_logger: Any,
Expand Down
26 changes: 26 additions & 0 deletions py/src/braintrust/btx/test_span_validator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import pytest

from .span_validator import _validate_value
from .spec_loader import FnMatcher, OrMatcher


@pytest.mark.parametrize(
"actual, expected, valid",
[
({}, FnMatcher("undefined_or_null"), True),
({"metric": None}, FnMatcher("undefined_or_null"), True),
({"metric": 0}, FnMatcher("undefined_or_null"), False),
({"metric": 42}, FnMatcher("undefined_or_null"), False),
({}, FnMatcher("is_non_negative_number"), False),
({}, 0, False),
({}, OrMatcher([FnMatcher("is_positive_number"), FnMatcher("undefined_or_null")]), True),
({"metric": 0}, OrMatcher([FnMatcher("is_positive_number"), FnMatcher("undefined_or_null")]), False),
],
)
def test_absent_metric_matcher(actual, expected, valid):
errors = []
_validate_value(actual, {"metric": expected}, "metrics", errors)
if valid:
assert errors == []
else:
assert errors and all("metrics.metric" in error for error in errors)
37 changes: 37 additions & 0 deletions py/src/braintrust/btx/test_spec_loader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
from uuid import UUID

import pytest

from .spec_loader import load_specs


@pytest.mark.parametrize("live", [False, True], ids=["replay", "live"])
def test_cache_variables_are_shared_by_request_and_expectation(tmp_path, monkeypatch, live):
monkeypatch.setenv("BTX_CLIENT", "openai")
provider = tmp_path / "openai"
provider.mkdir()
(provider / "cache.yaml").write_text(
"""name: cache
type: llm_span_test
provider: openai
endpoint: /v1/responses
variables:
client: !gen test_runner_client
nonce: !gen vcr_nonce
requests:
- input: "{{client}} {{nonce}}"
expected_brainstore_spans:
- input: "{{client}} {{nonce}}"
"""
)

first = load_specs(tmp_path, live=live)[0]
second = load_specs(tmp_path, live=live)[0]
first_input = first.requests[0]["input"]
assert first.expected_brainstore_spans[0]["input"] == first_input
assert first_input.startswith("python-openai ")
if live:
assert str(UUID(first_input.split(" ")[1])) == first_input.split(" ")[1]
assert second.requests[0]["input"] != first_input
else:
assert first_input == second.requests[0]["input"] == "python-openai vcr-mode"
Loading
Loading