Skip to content

Commit 25b3242

Browse files
noobHappylifenoobHappylife
authored
Fix Responses API streaming for multiple auto tool calls (vllm-project#39626)
Signed-off-by: noobhappylife <aratar1991@hotmail.com>
1 parent b075604 commit 25b3242

3 files changed

Lines changed: 329 additions & 21 deletions

File tree

‎tests/entrypoints/openai/responses/test_function_call.py‎

Lines changed: 50 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -249,40 +249,74 @@ async def test_function_calling_with_streaming_expected_arguments(
249249
"additionalProperties": False,
250250
},
251251
"strict": True,
252-
}
252+
},
253+
{
254+
"type": "function",
255+
"name": "get_time",
256+
"description": "Get current local time for provided location.",
257+
"parameters": {
258+
"type": "object",
259+
"properties": {
260+
"location": {"type": "string"},
261+
},
262+
"required": ["location"],
263+
"additionalProperties": False,
264+
},
265+
"strict": True,
266+
},
253267
]
254268

255269
stream_response = await client.responses.create(
256270
model=model_name,
257-
input="Can you tell me what the current weather is in Berlin?",
271+
input=(
272+
"Use tools only. Call get_weather for Berlin and get_time for Tokyo. "
273+
"Do not answer directly."
274+
),
258275
tools=tools,
259276
stream=True,
260277
)
261278

262-
tool_call_item = None
263-
completed_event = None
279+
tool_call_items = {}
280+
arguments_done_events = {}
281+
completed_events = {}
264282
async for event in stream_response:
265283
if (
266284
event.type == "response.output_item.added"
267285
and event.item.type == "function_call"
268286
):
269-
tool_call_item = event.item
270-
elif event.type == "response.function_call_arguments.delta" and tool_call_item:
287+
tool_call_items[event.output_index] = event.item
288+
elif event.type == "response.function_call_arguments.delta":
289+
tool_call_item = tool_call_items[event.output_index]
271290
tool_call_item.arguments += event.delta
291+
elif event.type == "response.function_call_arguments.done":
292+
arguments_done_events[event.output_index] = event
272293
elif (
273294
event.type == "response.output_item.done"
274295
and event.item.type == "function_call"
275296
):
276-
completed_event = event
277-
assert tool_call_item is not None
278-
assert tool_call_item.type == "function_call"
279-
assert tool_call_item.name == "get_weather"
280-
assert completed_event is not None
281-
assert tool_call_item.arguments == completed_event.item.arguments
282-
assert tool_call_item.name == completed_event.item.name
283-
args = json.loads(tool_call_item.arguments)
284-
assert "location" in args
285-
assert args["location"] is not None
297+
completed_events[event.output_index] = event
298+
assert len(tool_call_items) >= 2
299+
assert len(arguments_done_events) >= 2
300+
assert len(completed_events) >= 2
301+
302+
tool_calls_by_name = {
303+
event.item.name: (
304+
tool_call_items[output_index],
305+
arguments_done_events[output_index],
306+
event.item,
307+
)
308+
for output_index, event in completed_events.items()
309+
}
310+
assert {"get_weather", "get_time"}.issubset(tool_calls_by_name)
311+
for added_item, arguments_done_event, completed_item in tool_calls_by_name.values():
312+
assert added_item.type == "function_call"
313+
assert added_item.arguments == arguments_done_event.arguments
314+
assert added_item.arguments == completed_item.arguments
315+
assert added_item.name == arguments_done_event.name
316+
assert added_item.name == completed_item.name
317+
args = json.loads(added_item.arguments)
318+
assert "location" in args
319+
assert args["location"] is not None
286320

287321

288322
@pytest.mark.asyncio

‎tests/entrypoints/openai/responses/test_serving_responses.py‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@
2727
import vllm.envs as envs
2828
from vllm.entrypoints.mcp.tool_server import ToolServer
2929
from vllm.entrypoints.openai.engine.protocol import (
30+
DeltaFunctionCall,
3031
DeltaMessage,
32+
DeltaToolCall,
3133
ErrorResponse,
3234
RequestResponseMetadata,
3335
)
@@ -928,3 +930,197 @@ async def result_generator():
928930
]
929931
assert len(item_done_events) == 1
930932
assert isinstance(item_done_events[0].item, ResponseReasoningItem)
933+
934+
935+
class TestAutoToolStreaming:
936+
@staticmethod
937+
async def _collect_events(delta_sequence: list[DeltaMessage]):
938+
serving = _make_serving_instance_with_reasoning()
939+
_mock_parser_with_reasoning(serving, delta_sequence)
940+
941+
contexts = [
942+
_make_simple_context_with_output("chunk", [i])
943+
for i in range(len(delta_sequence))
944+
]
945+
946+
async def result_generator():
947+
for ctx in contexts:
948+
yield ctx
949+
950+
request = ResponsesRequest(
951+
input="hi",
952+
tools=[
953+
{
954+
"type": "function",
955+
"name": "get_weather",
956+
"description": "Get weather.",
957+
"parameters": {
958+
"type": "object",
959+
"properties": {"location": {"type": "string"}},
960+
"required": ["location"],
961+
"additionalProperties": False,
962+
},
963+
}
964+
],
965+
tool_choice="auto",
966+
stream=True,
967+
)
968+
sampling_params = SamplingParams(max_tokens=64)
969+
metadata = RequestResponseMetadata(request_id="req")
970+
_identity_increment._counter = 0 # type: ignore
971+
972+
events = []
973+
async for event in serving._process_simple_streaming_events(
974+
request=request,
975+
sampling_params=sampling_params,
976+
result_generator=result_generator(),
977+
context=SimpleContext(),
978+
model_name="test-model",
979+
tokenizer=MagicMock(),
980+
request_metadata=metadata,
981+
created_time=0,
982+
_increment_sequence_number_and_return=_identity_increment,
983+
):
984+
events.append(event)
985+
return events
986+
987+
@pytest.mark.skip_global_cleanup
988+
@pytest.mark.asyncio
989+
async def test_auto_multi_tool_streaming_opens_one_item_per_tool(self, monkeypatch):
990+
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
991+
992+
delta_sequence = [
993+
DeltaMessage(
994+
tool_calls=[
995+
DeltaToolCall(
996+
id="call_vienna",
997+
type="function",
998+
index=0,
999+
function=DeltaFunctionCall(
1000+
name="get_weather",
1001+
arguments="",
1002+
),
1003+
)
1004+
]
1005+
),
1006+
DeltaMessage(
1007+
tool_calls=[
1008+
DeltaToolCall(
1009+
index=0,
1010+
function=DeltaFunctionCall(
1011+
arguments='{"location":"Vienna"}',
1012+
),
1013+
)
1014+
]
1015+
),
1016+
DeltaMessage(
1017+
tool_calls=[
1018+
DeltaToolCall(
1019+
id="call_berlin",
1020+
type="function",
1021+
index=1,
1022+
function=DeltaFunctionCall(
1023+
name="get_weather",
1024+
arguments='{"location":"Berlin"}',
1025+
),
1026+
)
1027+
]
1028+
),
1029+
]
1030+
events = await self._collect_events(delta_sequence)
1031+
1032+
function_items = [
1033+
event
1034+
for event in events
1035+
if event.type == "response.output_item.added"
1036+
and getattr(event.item, "type", None) == "function_call"
1037+
]
1038+
assert len(function_items) == 2
1039+
assert [event.item.name for event in function_items] == [
1040+
"get_weather",
1041+
"get_weather",
1042+
]
1043+
assert [event.output_index for event in function_items] == [0, 1]
1044+
1045+
argument_deltas = [
1046+
event.delta
1047+
for event in events
1048+
if event.type == "response.function_call_arguments.delta"
1049+
]
1050+
assert argument_deltas == [
1051+
'{"location":"Vienna"}',
1052+
'{"location":"Berlin"}',
1053+
]
1054+
1055+
argument_done = [
1056+
event
1057+
for event in events
1058+
if event.type == "response.function_call_arguments.done"
1059+
]
1060+
assert [event.arguments for event in argument_done] == [
1061+
'{"location":"Vienna"}',
1062+
'{"location":"Berlin"}',
1063+
]
1064+
assert [event.output_index for event in argument_done] == [0, 1]
1065+
1066+
function_done = [
1067+
event
1068+
for event in events
1069+
if event.type == "response.output_item.done"
1070+
and getattr(event.item, "type", None) == "function_call"
1071+
]
1072+
assert [event.item.arguments for event in function_done] == [
1073+
'{"location":"Vienna"}',
1074+
'{"location":"Berlin"}',
1075+
]
1076+
assert [event.output_index for event in function_done] == [0, 1]
1077+
1078+
@pytest.mark.skip_global_cleanup
1079+
@pytest.mark.asyncio
1080+
async def test_auto_tool_choice_first_delta_tool_call_does_not_duplicate_item(
1081+
self, monkeypatch
1082+
):
1083+
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
1084+
1085+
delta_sequence = [
1086+
DeltaMessage(
1087+
tool_calls=[
1088+
DeltaToolCall(
1089+
id="call_test",
1090+
type="function",
1091+
index=0,
1092+
function=DeltaFunctionCall(
1093+
name="get_weather",
1094+
arguments="",
1095+
),
1096+
)
1097+
]
1098+
),
1099+
DeltaMessage(
1100+
tool_calls=[
1101+
DeltaToolCall(
1102+
index=0,
1103+
function=DeltaFunctionCall(
1104+
arguments='{"location":"Berlin"}',
1105+
),
1106+
)
1107+
]
1108+
),
1109+
]
1110+
events = await self._collect_events(delta_sequence)
1111+
1112+
function_items = [
1113+
event
1114+
for event in events
1115+
if event.type == "response.output_item.added"
1116+
and getattr(event.item, "type", None) == "function_call"
1117+
]
1118+
assert len(function_items) == 1
1119+
assert function_items[0].item.name == "get_weather"
1120+
1121+
argument_deltas = [
1122+
event.delta
1123+
for event in events
1124+
if event.type == "response.function_call_arguments.delta"
1125+
]
1126+
assert "".join(argument_deltas) == '{"location":"Berlin"}'

0 commit comments

Comments
 (0)