From 3b85c900b9a9aa312890049f820a42c8a080d4cc Mon Sep 17 00:00:00 2001 From: Ritchie Cotton <243725+infinityrobot@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:50:09 +1100 Subject: [PATCH 1/6] Fixed: RubyLLM 2.x tool-calling turns dropping the final answer ruby_llm 2.x runs the whole tool loop inside Chat#complete (step until complete?), so wrapping complete gave one span per turn. On tool-calling turns that span recorded the tool-call request as its output and only the last request's tokens. On 2.x, trace Chat#generate instead, which step calls once per model request. 1.x keeps wrapping complete, which recurses after tool calls. The variant is picked when the patch is applied, by whether the class defines a public generate. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../contrib/ruby_llm/instrumentation/chat.rb | 46 +++++++++----- .../ruby_llm/instrumentation/chat_test.rb | 60 +++++++++++++++++++ 2 files changed, 90 insertions(+), 16 deletions(-) diff --git a/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb b/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb index bdba507..0e6497e 100644 --- a/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb +++ b/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb @@ -13,43 +13,57 @@ module Contrib module RubyLLM module Instrumentation # Chat instrumentation for RubyLLM. - # Wraps complete() and execute_tool() methods to create spans. + # Wraps the method that makes one chat request, and execute_tool(), to create spans. module Chat def self.included(base) # Guard against double-wrapping - base.prepend(InstanceMethods) unless applied?(base) + return if applied?(base) + + base.prepend(InstanceMethods) + base.prepend(base.public_method_defined?(:generate) ? GenerateTracing : CompleteTracing) end def self.applied?(base) base.ancestors.include?(InstanceMethods) end - module InstanceMethods - # Wrap complete() to trace chat completions. - # Each call creates a span - recursive calls from tool execution - # create nested spans (each is a separate API call). + # ruby_llm 1.x: each complete() call makes one request, and recursive calls from + # tool execution create nested spans. + module CompleteTracing def complete(&block) return block ? super : super() unless tracing_enabled? + trace_chat_request(block) { |wrapper| super(&wrapper) } + end + end + + # ruby_llm 2.x: complete() loops over step(), which calls generate() once per + # request, so each generate() call gets its own span. + module GenerateTracing + def generate(&block) + return block ? super : super() unless tracing_enabled? + + trace_chat_request(block) { |wrapper| super(&wrapper) } + end + end + + module InstanceMethods + private + + # Traces one chat request. Yields the streaming wrapper block, or nil when not + # streaming, for the caller to pass to super. + def trace_chat_request(block) tracer = Braintrust::Contrib.tracer_for(self) tracer.in_span("ruby_llm.chat") do |span| if block - # Streaming: pass a block that calls super() with the wrapper - handle_streaming_complete(span, block) do |&wrapper| - super(&wrapper) - end + handle_streaming_complete(span, block) { |&wrapper| yield wrapper } else - # Non-streaming: pass a block that calls super() - handle_non_streaming_complete(span) do - super() - end + handle_non_streaming_complete(span) { yield nil } end end end - private - # Wrap execute_tool() to trace tool executions. # This is a private method in RubyLLM - wrapping it avoids # conflicting with user-registered on_tool_call/on_tool_result callbacks. diff --git a/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb b/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb index 9e46f96..17eecc4 100644 --- a/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb +++ b/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb @@ -14,6 +14,7 @@ def test_included_prepends_instance_methods mock = Minitest::Mock.new mock.expect(:include?, false, [Chat::InstanceMethods]) mock.expect(:prepend, nil, [Chat::InstanceMethods]) + mock.expect(:prepend, nil, [Chat::CompleteTracing]) base.define_singleton_method(:ancestors) { mock } base.define_singleton_method(:prepend) { |mod| mock.prepend(mod) } @@ -23,6 +24,28 @@ def test_included_prepends_instance_methods mock.verify end + def test_included_traces_complete_when_generate_is_not_defined + base = Class.new { def complete = nil } + + Chat.included(base) + + assert_includes base.ancestors, Chat::CompleteTracing + refute_includes base.ancestors, Chat::GenerateTracing + end + + def test_included_traces_generate_when_defined + base = Class.new do + def complete = nil + + def generate = nil + end + + Chat.included(base) + + assert_includes base.ancestors, Chat::GenerateTracing + refute_includes base.ancestors, Chat::CompleteTracing + end + def test_included_skips_prepend_when_already_applied base = Class.new do include Braintrust::Contrib::RubyLLM::Instrumentation::Chat @@ -290,6 +313,43 @@ def test_tool_calling_creates_nested_spans end end + def test_tool_calling_creates_a_chat_span_per_request + skip "ruby_llm 1.x nests a chat span per request inside complete()" if ruby_llm_major < 2 + + with_weather_test_tool do |tool_class| + VCR.use_cassette(ruby_llm_cassette("tool_calling")) do + rig = setup_otel_test_rig + + configure_ruby_llm_for_vcr + + chat = RubyLLM.chat(model: "gpt-4o-mini") + Braintrust.instrument!(:ruby_llm, target: chat, tracer_provider: rig.tracer_provider) + chat.with_tools(tool_class) + + response = chat.ask("What's the weather like in San Francisco?") + + spans = rig.drain + chat_spans = spans.select { |s| s.name == "ruby_llm.chat" }.sort_by(&:start_timestamp) + + # 2.x runs the whole tool loop inside complete(): one request for the tool call, one for the answer. + assert_equal 2, chat_spans.length + + tool_call_output, answer_output = chat_spans.map { |s| JSON.parse(s.attributes["braintrust.output_json"]).first } + + assert_equal "tool_calls", tool_call_output["finish_reason"] + assert_equal "weather_test", tool_call_output.dig("message", "tool_calls", 0, "function", "name") + + assert_equal "stop", answer_output["finish_reason"] + assert_equal response.content, answer_output.dig("message", "content") + + # Each span carries its own request's usage, as recorded in the cassette. + metrics = chat_spans.map { |s| JSON.parse(s.attributes["braintrust.metrics"]) } + assert_equal [74, 124], metrics.map { |m| m["prompt_tokens"] } + assert_equal [23, 19], metrics.map { |m| m["completion_tokens"] } + end + end + end + # --- Direct complete() --- def test_direct_complete_creates_span From b4e94b67c124b608624ea3bd30b628751912d240 Mon Sep 17 00:00:00 2001 From: Ritchie Cotton <243725+infinityrobot@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:01:51 +1100 Subject: [PATCH 2/6] Changed: Group RubyLLM 2.x turns under a ruby_llm.complete span Tracing generate() alone made a tool-calling turn's chat and tool spans siblings, so a caller without its own span got one trace per request. On 2.x, complete() now opens a task span that parents them. It carries the turn's input and final output but no metrics, since each request's tokens are on its ruby_llm.chat span. 1.x is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../contrib/ruby_llm/instrumentation/chat.rb | 21 +++++++++++++++- lib/braintrust/contrib/ruby_llm/patcher.rb | 2 +- .../contrib/ruby_llm/deprecated_test.rb | 3 +-- .../ruby_llm/instrumentation/chat_test.rb | 24 +++++++++++++------ .../contrib/ruby_llm/integration_helper.rb | 13 ++++++++++ 5 files changed, 52 insertions(+), 11 deletions(-) diff --git a/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb b/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb index 0e6497e..3c6ed09 100644 --- a/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb +++ b/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb @@ -38,13 +38,32 @@ def complete(&block) end # ruby_llm 2.x: complete() loops over step(), which calls generate() once per - # request, so each generate() call gets its own span. + # request, so each generate() call gets its own span. complete() gets a parent + # span so a turn's chat and tool spans share one trace when the caller has no span. module GenerateTracing def generate(&block) return block ? super : super() unless tracing_enabled? trace_chat_request(block) { |wrapper| super(&wrapper) } end + + def complete(&block) + return block ? super : super() unless tracing_enabled? + + tracer = Braintrust::Contrib.tracer_for(self) + + tracer.in_span("ruby_llm.complete") do |span| + Support::OTel.set_json_attr(span, "braintrust.span_attributes", {type: "task"}) + input_messages = build_input_messages + Support::OTel.set_json_attr(span, "braintrust.input_json", input_messages) if input_messages.any? + + response = block ? super : super() + + # No metrics here: each request's tokens live on its ruby_llm.chat child span. + Support::OTel.set_json_attr(span, "braintrust.output_json", [format_message_for_input(response)]) if response + response + end + end end module InstanceMethods diff --git a/lib/braintrust/contrib/ruby_llm/patcher.rb b/lib/braintrust/contrib/ruby_llm/patcher.rb index a4fde88..a044ebb 100644 --- a/lib/braintrust/contrib/ruby_llm/patcher.rb +++ b/lib/braintrust/contrib/ruby_llm/patcher.rb @@ -7,7 +7,7 @@ module Braintrust module Contrib module RubyLLM # Patcher for RubyLLM chat completions. - # Instruments RubyLLM::Chat#complete and #execute_tool methods. + # Instruments RubyLLM::Chat#complete (plus #generate on 2.x) and #execute_tool. class ChatPatcher < Braintrust::Contrib::Patcher class << self def applicable? diff --git a/test/braintrust/contrib/ruby_llm/deprecated_test.rb b/test/braintrust/contrib/ruby_llm/deprecated_test.rb index 643e73b..84f4cd2 100644 --- a/test/braintrust/contrib/ruby_llm/deprecated_test.rb +++ b/test/braintrust/contrib/ruby_llm/deprecated_test.rb @@ -120,8 +120,7 @@ def test_unwrap_disables_instrumentation_for_subsequent_requests # First request should create a span chat.ask("Say 'test'") - spans_before = rig.drain - assert_equal 1, spans_before.length, "Expected 1 span before unwrap" + drain_chat_span(rig) # Now unwrap the chat instance suppress_logs { Braintrust::Trace::Contrib::Github::Crmne::RubyLLM.unwrap(chat) } diff --git a/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb b/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb index 17eecc4..715d3a2 100644 --- a/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb +++ b/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb @@ -156,7 +156,7 @@ def test_complete_creates_span_with_correct_attributes refute_nil response.content assert response.content.length > 0 - span = rig.drain_one + span = drain_chat_span(rig) assert_equal "ruby_llm.chat", span.name @@ -208,7 +208,7 @@ def test_streaming_complete_creates_span refute_empty chunks - span = rig.drain_one + span = drain_chat_span(rig) assert_equal "ruby_llm.chat", span.name @@ -243,7 +243,7 @@ def test_streaming_complete_creates_span # --- Tool calling --- - def test_tool_calling_creates_nested_spans + def test_tool_calling_creates_chat_and_tool_spans skip "Tool calling test requires ruby_llm >= 1.9" unless self.class.supports_tool_calling? with_weather_test_tool do |tool_class| @@ -346,6 +346,16 @@ def test_tool_calling_creates_a_chat_span_per_request metrics = chat_spans.map { |s| JSON.parse(s.attributes["braintrust.metrics"]) } assert_equal [74, 124], metrics.map { |m| m["prompt_tokens"] } assert_equal [23, 19], metrics.map { |m| m["completion_tokens"] } + + # complete() is a task span parenting the turn's chat and tool spans, so they share one trace. + complete_span = spans.find { |s| s.name == "ruby_llm.complete" } + assert_equal "task", JSON.parse(complete_span.attributes["braintrust.span_attributes"])["type"] + assert_equal response.content, JSON.parse(complete_span.attributes["braintrust.output_json"]).first["content"] + refute complete_span.attributes.key?("braintrust.metrics"), "Tokens belong on the chat spans only" + + children = spans.reject { |s| s.equal?(complete_span) } + assert_equal 3, children.length + assert(children.all? { |s| s.parent_span_id == complete_span.span_id }) end end end @@ -369,7 +379,7 @@ def test_direct_complete_creates_span refute_nil response.content assert response.content.length > 0 - span = rig.drain_one + span = drain_chat_span(rig) assert_equal "ruby_llm.chat", span.name @@ -402,8 +412,8 @@ def test_wrapping_is_idempotent refute_nil response - # Drain and verify - should only have ONE span (not double-wrapped) - span = rig.drain_one + # Drain and verify - should only have ONE chat span (not double-wrapped) + span = drain_chat_span(rig) assert_equal "ruby_llm.chat", span.name end @@ -434,7 +444,7 @@ def test_class_level_instrumentation_traces_new_instances refute_nil response.content # Verify span was created (proving class-level instrumentation works) - span = rig.drain_one + span = drain_chat_span(rig) assert_equal "ruby_llm.chat", span.name assert span.attributes.key?("braintrust.input_json") diff --git a/test/braintrust/contrib/ruby_llm/integration_helper.rb b/test/braintrust/contrib/ruby_llm/integration_helper.rb index 317121b..08a9756 100644 --- a/test/braintrust/contrib/ruby_llm/integration_helper.rb +++ b/test/braintrust/contrib/ruby_llm/integration_helper.rb @@ -34,6 +34,19 @@ def configure_ruby_llm_for_vcr end end + # Major version of the loaded ruby_llm gem. + def ruby_llm_major + Gem.loaded_specs["ruby_llm"]&.version&.segments&.first || 1 + end + + # Drains the rig and returns its single ruby_llm.chat span. On 2.x, complete() + # also emits a ruby_llm.complete parent span. + def drain_chat_span(rig) + chat_spans = rig.drain.select { |s| s.name == "ruby_llm.chat" } + assert_equal 1, chat_spans.length, "Expected exactly 1 ruby_llm.chat span" + chat_spans.first + end + # Cassette path for a ruby_llm interaction, for the loaded ruby_llm version. def ruby_llm_cassette(name) versioned_cassette("ruby_llm", name) From 74725575d16909de2cd85366ff9608f2786e71a3 Mon Sep 17 00:00:00 2001 From: Ritchie Cotton <243725+infinityrobot@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:16:20 +1100 Subject: [PATCH 3/6] Changed: Assert the ruby_llm.complete span count in drain_chat_span Filtering to ruby_llm.chat alone let a double-wrapped complete() pass the idempotency test. Also expect exactly one ruby_llm.complete span on 2.x and none on 1.x. Co-Authored-By: Claude Opus 5.5 (1M context) --- test/braintrust/contrib/ruby_llm/integration_helper.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/braintrust/contrib/ruby_llm/integration_helper.rb b/test/braintrust/contrib/ruby_llm/integration_helper.rb index 08a9756..521393c 100644 --- a/test/braintrust/contrib/ruby_llm/integration_helper.rb +++ b/test/braintrust/contrib/ruby_llm/integration_helper.rb @@ -39,11 +39,15 @@ def ruby_llm_major Gem.loaded_specs["ruby_llm"]&.version&.segments&.first || 1 end - # Drains the rig and returns its single ruby_llm.chat span. On 2.x, complete() - # also emits a ruby_llm.complete parent span. + # Drains the rig and returns its single ruby_llm.chat span, checking that 2.x also + # emitted exactly one ruby_llm.complete parent and 1.x none. def drain_chat_span(rig) - chat_spans = rig.drain.select { |s| s.name == "ruby_llm.chat" } + spans = rig.drain + chat_spans = spans.select { |s| s.name == "ruby_llm.chat" } + complete_spans = spans.select { |s| s.name == "ruby_llm.complete" } + assert_equal 1, chat_spans.length, "Expected exactly 1 ruby_llm.chat span" + assert_equal (ruby_llm_major >= 2) ? 1 : 0, complete_spans.length, "Unexpected ruby_llm.complete span count" chat_spans.first end From f5743ac974a9294a9f2747876ca30584c4c4df51 Mon Sep 17 00:00:00 2001 From: Ritchie Cotton <243725+infinityrobot@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:42:13 +1100 Subject: [PATCH 4/6] Fixed: RubyLLM 2.x cache tokens missing from span metrics ruby_llm 2.x renamed the cache counts to cache_read_tokens and cache_write_tokens, so cached input never reached prompt_tokens and a cached request reported only its few uncached tokens. Read either name in one helper shared by the streaming and non-streaming paths. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../contrib/ruby_llm/instrumentation/chat.rb | 30 +----------- .../ruby_llm/instrumentation/common.rb | 18 ++++++++ .../ruby_llm/instrumentation/common_test.rb | 46 +++++++++++++++++++ 3 files changed, 66 insertions(+), 28 deletions(-) create mode 100644 test/braintrust/contrib/ruby_llm/instrumentation/common_test.rb diff --git a/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb b/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb index 3c6ed09..bee3deb 100644 --- a/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb +++ b/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb @@ -457,20 +457,7 @@ def capture_streaming_output(span, aggregated_chunks, result, time_to_first_toke # Set metrics (token usage + time_to_first_token) # RubyLLM stores tokens directly in the response hash, not in a nested usage object - metrics = {} - if result.respond_to?(:to_h) - result_hash = result.to_h - usage = { - "input_tokens" => result_hash[:input_tokens], - "output_tokens" => result_hash[:output_tokens], - "cached_tokens" => result_hash[:cached_tokens], - "cache_creation_tokens" => result_hash[:cache_creation_tokens] - }.compact - - unless usage.empty? - metrics = Common.parse_usage_tokens(usage) - end - end + metrics = Common.message_usage_metrics(result) metrics["time_to_first_token"] = time_to_first_token if time_to_first_token Support::OTel.set_json_attr(span, "braintrust.metrics", metrics) unless metrics.empty? end @@ -513,20 +500,7 @@ def capture_non_streaming_output(span, response, messages_before_count, time_to_ Support::OTel.set_json_attr(span, "braintrust.output_json", output) # Set metrics (token usage + time_to_first_token) - metrics = {} - if response.respond_to?(:to_h) - response_hash = response.to_h - usage = { - "input_tokens" => response_hash[:input_tokens], - "output_tokens" => response_hash[:output_tokens], - "cached_tokens" => response_hash[:cached_tokens], - "cache_creation_tokens" => response_hash[:cache_creation_tokens] - }.compact - - unless usage.empty? - metrics = Common.parse_usage_tokens(usage) - end - end + metrics = Common.message_usage_metrics(response) metrics["time_to_first_token"] = time_to_first_token if time_to_first_token Support::OTel.set_json_attr(span, "braintrust.metrics", metrics) unless metrics.empty? end diff --git a/lib/braintrust/contrib/ruby_llm/instrumentation/common.rb b/lib/braintrust/contrib/ruby_llm/instrumentation/common.rb index e1df858..391c4e2 100644 --- a/lib/braintrust/contrib/ruby_llm/instrumentation/common.rb +++ b/lib/braintrust/contrib/ruby_llm/instrumentation/common.rb @@ -6,6 +6,24 @@ module RubyLLM module Instrumentation # Common utilities for RubyLLM instrumentation. module Common + # Braintrust metrics for a RubyLLM message's token counts. ruby_llm 2.x renamed the cache + # counts from cached_tokens/cache_creation_tokens to cache_read_tokens/cache_write_tokens. + # @param message [Object] a RubyLLM message (anything responding to to_h) + # @return [Hash] normalized metrics, empty when no counts are reported + def self.message_usage_metrics(message) + return {} unless message.respond_to?(:to_h) + + counts = message.to_h + usage = { + "input_tokens" => counts[:input_tokens], + "output_tokens" => counts[:output_tokens], + "cached_tokens" => counts[:cached_tokens] || counts[:cache_read_tokens], + "cache_creation_tokens" => counts[:cache_creation_tokens] || counts[:cache_write_tokens] + }.compact + + usage.empty? ? {} : parse_usage_tokens(usage) + end + # Parse RubyLLM usage tokens into normalized Braintrust metrics. # RubyLLM normalizes token fields from all providers (OpenAI, Anthropic, etc.) # into a consistent format: diff --git a/test/braintrust/contrib/ruby_llm/instrumentation/common_test.rb b/test/braintrust/contrib/ruby_llm/instrumentation/common_test.rb new file mode 100644 index 0000000..6663c13 --- /dev/null +++ b/test/braintrust/contrib/ruby_llm/instrumentation/common_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require "test_helper" +require "braintrust/contrib/ruby_llm/instrumentation/common" + +class Braintrust::Contrib::RubyLLM::Instrumentation::CommonTest < Minitest::Test + Common = Braintrust::Contrib::RubyLLM::Instrumentation::Common + Message = Struct.new(:counts) do + def to_h = counts + end + + # ======================= + # message_usage_metrics + # ======================= + + def test_reads_ruby_llm_2_cache_keys + message = Message.new({input_tokens: 3, output_tokens: 26, cache_read_tokens: 2763, cache_write_tokens: 366}) + + metrics = Common.message_usage_metrics(message) + + assert_equal 2763, metrics["prompt_cached_tokens"] + assert_equal 366, metrics["prompt_cache_creation_tokens"] + assert_equal 3132, metrics["prompt_tokens"] + assert_equal 26, metrics["completion_tokens"] + assert_equal 3158, metrics["tokens"] + end + + def test_reads_ruby_llm_1_cache_keys + message = Message.new({input_tokens: 10, output_tokens: 5, cached_tokens: 40, cache_creation_tokens: 50}) + + metrics = Common.message_usage_metrics(message) + + assert_equal 40, metrics["prompt_cached_tokens"] + assert_equal 50, metrics["prompt_cache_creation_tokens"] + assert_equal 100, metrics["prompt_tokens"] + assert_equal 105, metrics["tokens"] + end + + def test_returns_empty_without_counts + assert_empty Common.message_usage_metrics(Message.new({role: :assistant})) + end + + def test_returns_empty_when_message_has_no_to_h + assert_empty Common.message_usage_metrics(Object.new) + end +end From e3cc75e1f806c677381c2c8ae5c748c2e103d8ca Mon Sep 17 00:00:00 2001 From: Ritchie Cotton <243725+infinityrobot@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:51:31 +1100 Subject: [PATCH 5/6] Fixed: RubyLLM tool schemas mutated by span metadata format_tool_schema deleted strict and additionalProperties from the tool's parameters hash in place unless it was frozen. That hash is the tool's cached schema, so instrumenting a chat stripped the constraints from the requests RubyLLM sends to the provider, for every later instance and even for untraced chats. Build trimmed copies instead. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../contrib/ruby_llm/instrumentation/chat.rb | 24 ++++++++----------- .../ruby_llm/instrumentation/chat_test.rb | 18 ++++++++++++++ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb b/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb index bee3deb..0cf620c 100644 --- a/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb +++ b/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb @@ -232,22 +232,18 @@ def format_tool_schema(tool, provider) tool_schema = tool_params_schema(tool) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool) end - # Strip RubyLLM-specific fields to match native OpenAI format + # Strip RubyLLM-specific fields to match native OpenAI format. Build copies: the + # parameters hash is the tool's cached schema, which RubyLLM sends to the provider. function_key = tool_schema&.key?(:function) ? :function : "function" - if tool_schema && tool_schema[function_key] - tool_params = tool_schema[function_key][:parameters] || tool_schema[function_key]["parameters"] - if tool_params.is_a?(Hash) - tool_params = tool_params.dup if tool_params.frozen? - tool_params.delete("strict") - tool_params.delete(:strict) - tool_params.delete("additionalProperties") - tool_params.delete(:additionalProperties) - params_key = tool_schema[function_key].key?(:parameters) ? :parameters : "parameters" - tool_schema[function_key][params_key] = tool_params - end - end + function = tool_schema && tool_schema[function_key] + return tool_schema unless function + + params_key = function.key?(:parameters) ? :parameters : "parameters" + tool_params = function[params_key] + return tool_schema unless tool_params.is_a?(Hash) - tool_schema + stripped = tool_params.reject { |key, _| %w[strict additionalProperties].include?(key.to_s) } + tool_schema.merge(function_key => function.merge(params_key => stripped)) end # A tool's JSON Schema, across ruby_llm versions. diff --git a/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb b/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb index 715d3a2..1dd74bc 100644 --- a/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb +++ b/test/braintrust/contrib/ruby_llm/instrumentation/chat_test.rb @@ -360,6 +360,24 @@ def test_tool_calling_creates_a_chat_span_per_request end end + def test_tool_metadata_leaves_the_tool_schema_sent_to_the_provider_intact + skip "Tool calling test requires ruby_llm >= 1.9" unless self.class.supports_tool_calling? + + with_weather_test_tool do |tool_class| + configure_ruby_llm_for_vcr + schema_of = ->(tool) { tool.respond_to?(:parameters_schema) ? tool.parameters_schema : tool.params_schema } + expected = Marshal.load(Marshal.dump(schema_of.call(tool_class.new))) + + chat = RubyLLM.chat(model: "gpt-4o-mini") + Braintrust.instrument!(:ruby_llm, target: chat) + chat.with_tools(tool_class) + chat.send(:extract_tools_metadata) + + assert_equal expected, schema_of.call(chat.tools.values.first) + assert_equal expected, schema_of.call(tool_class.new) + end + end + # --- Direct complete() --- def test_direct_complete_creates_span From 44d16e55f6da7d01e88a6abadc33d57a008177c2 Mon Sep 17 00:00:00 2001 From: Ritchie Cotton <243725+infinityrobot@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:18:35 +1100 Subject: [PATCH 6/6] Fixed: CI resolving faraday 2.0.0 for the ruby_llm appraisals faraday-net_http 3.5.0 (2026-10-11) is past faraday 2.14.4's `< 3.5` cap, so the ruby_llm appraisals backtracked to faraday 2.0.0, which cannot load the net_http adapter, and every recorded RubyLLM test errored. Require faraday >= 2.14 there. Co-Authored-By: Claude Opus 5.5 (1M context) --- Appraisals | 12 ++++++------ gemfiles/ruby_llm.gemfile | 2 +- gemfiles/ruby_llm_1_8.gemfile | 2 +- gemfiles/ruby_llm_1_9.gemfile | 2 +- gemfiles/ruby_llm_2_0.gemfile | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Appraisals b/Appraisals index 20ceaac..5ac366e 100644 --- a/Appraisals +++ b/Appraisals @@ -23,13 +23,13 @@ OPTIONAL_GEMS = { "8.0" => {constraint: "~> 8.0", deps: {}}, "latest" => {constraint: ">= 8.0", deps: {}} }, - # faraday 2.0.0 doesn't depend on faraday-net_http, so the resolver can pair it with an - # adapter newer faraday excludes, leaving :net_http unregistered. + # faraday >= 2.14 stops resolution backtracking to faraday 2.0.0, which cannot load the net_http + # adapter, once faraday-net_http releases past the newest faraday's cap. "ruby_llm" => { - "1.8" => {constraint: "~> 1.8.0", deps: {"faraday" => ">= 2.0.1"}}, - "1.9" => {constraint: "~> 1.9.0", deps: {"faraday" => ">= 2.0.1"}}, - "2.0" => {constraint: "~> 2.0.0", deps: {"faraday" => ">= 2.0.1"}}, - "latest" => {constraint: ">= 2.0", deps: {"faraday" => ">= 2.0.1"}} + "1.8" => {constraint: "~> 1.8.0", deps: {"faraday" => ">= 2.14"}}, + "1.9" => {constraint: "~> 1.9.0", deps: {"faraday" => ">= 2.14"}}, + "2.0" => {constraint: "~> 2.0.0", deps: {"faraday" => ">= 2.14"}}, + "latest" => {constraint: ">= 2.0", deps: {"faraday" => ">= 2.14"}} } } diff --git a/gemfiles/ruby_llm.gemfile b/gemfiles/ruby_llm.gemfile index dd5df86..1bc19c0 100644 --- a/gemfiles/ruby_llm.gemfile +++ b/gemfiles/ruby_llm.gemfile @@ -15,6 +15,6 @@ gem "vcr", "~> 6.0" gem "webmock", "~> 3.0" gem "yard", "~> 0.9" gem "ruby_llm", ">= 2.0" -gem "faraday", ">= 2.0.1" +gem "faraday", ">= 2.14" gemspec path: "../" diff --git a/gemfiles/ruby_llm_1_8.gemfile b/gemfiles/ruby_llm_1_8.gemfile index 32a44b9..7f7e040 100644 --- a/gemfiles/ruby_llm_1_8.gemfile +++ b/gemfiles/ruby_llm_1_8.gemfile @@ -15,6 +15,6 @@ gem "vcr", "~> 6.0" gem "webmock", "~> 3.0" gem "yard", "~> 0.9" gem "ruby_llm", "~> 1.8.0" -gem "faraday", ">= 2.0.1" +gem "faraday", ">= 2.14" gemspec path: "../" diff --git a/gemfiles/ruby_llm_1_9.gemfile b/gemfiles/ruby_llm_1_9.gemfile index c4f755a..342cbe4 100644 --- a/gemfiles/ruby_llm_1_9.gemfile +++ b/gemfiles/ruby_llm_1_9.gemfile @@ -15,6 +15,6 @@ gem "vcr", "~> 6.0" gem "webmock", "~> 3.0" gem "yard", "~> 0.9" gem "ruby_llm", "~> 1.9.0" -gem "faraday", ">= 2.0.1" +gem "faraday", ">= 2.14" gemspec path: "../" diff --git a/gemfiles/ruby_llm_2_0.gemfile b/gemfiles/ruby_llm_2_0.gemfile index ce39692..3d7bfd9 100644 --- a/gemfiles/ruby_llm_2_0.gemfile +++ b/gemfiles/ruby_llm_2_0.gemfile @@ -15,6 +15,6 @@ gem "vcr", "~> 6.0" gem "webmock", "~> 3.0" gem "yard", "~> 0.9" gem "ruby_llm", "~> 2.0.0" -gem "faraday", ">= 2.0.1" +gem "faraday", ">= 2.14" gemspec path: "../"