[Bugfix][Frontend] streaming tool-call serializer drops first args chunk when name and args share a DeltaMessage (#42683)

Signed-off-by: ignaciosica <mignacio.sica@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Ignacio Sica
2026-05-28 05:20:55 +00:00
committed by GitHub
co-authored by sfeng33
parent 626fa9bba5
commit 7909f82a45
4 changed files with 241 additions and 39 deletions
@@ -1124,3 +1124,66 @@ class TestAutoToolStreaming:
if event.type == "response.function_call_arguments.delta"
]
assert "".join(argument_deltas) == '{"location":"Berlin"}'
@pytest.mark.skip_global_cleanup
@pytest.mark.asyncio
async def test_compound_content_and_tool_name_args_same_delta(self, monkeypatch):
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
tool_args = '{"location":"Berlin"}'
delta_sequence = [
DeltaMessage(
content="Let me check.",
tool_calls=[
DeltaToolCall(
id="call_weather",
type="function",
index=0,
function=DeltaFunctionCall(name="get_weather"),
),
DeltaToolCall(
index=0,
function=DeltaFunctionCall(arguments=tool_args),
),
],
)
]
events = await self._collect_events(delta_sequence)
text_deltas = [
event.delta
for event in events
if event.type == "response.output_text.delta"
]
assert text_deltas == ["Let me check."]
argument_deltas = [
event.delta
for event in events
if event.type == "response.function_call_arguments.delta"
]
assert argument_deltas == [tool_args]
types = [event.type for event in events]
assert types.index("response.output_text.delta") < types.index(
"response.function_call_arguments.delta"
)
argument_done = [
event
for event in events
if event.type == "response.function_call_arguments.done"
]
assert [event.arguments for event in argument_done] == [tool_args]
function_done = [
event
for event in events
if event.type == "response.output_item.done"
and getattr(event.item, "type", None) == "function_call"
]
assert len(function_done) == 1
assert function_done[0].item.name == "get_weather"
assert function_done[0].item.arguments == tool_args
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
)
from vllm.entrypoints.openai.responses.streaming_events import (
SimpleStreamingEventProcessor,
_StateType,
split_delta,
)
def _make_tool_call(
index: int, name: str | None = None, arguments: str | None = None
) -> DeltaToolCall:
fn = DeltaFunctionCall(name=name, arguments=arguments)
return DeltaToolCall(index=index, function=fn)
class TestSplitDelta:
def test_all_three_fields(self):
tc = _make_tool_call(0, name="f")
delta = DeltaMessage(reasoning="r", content="c", tool_calls=[tc])
result = split_delta(delta)
assert len(result) == 3
assert result[0].reasoning == "r" and result[0].content is None
assert result[1].content == "c" and result[1].reasoning is None
assert len(result[2].tool_calls) == 1 and result[2].content is None
def test_tool_calls_grouped_by_index(self):
tc0 = _make_tool_call(0, name="f1")
tc1 = _make_tool_call(1, name="f2")
tc0b = _make_tool_call(0, arguments='{"a":1}')
# Different indices → split
result = split_delta(DeltaMessage(tool_calls=[tc0, tc1]))
assert len(result) == 2
assert result[0].tool_calls == [tc0]
assert result[1].tool_calls == [tc1]
# Same index → stays together
delta = DeltaMessage(tool_calls=[tc0, tc0b])
result = split_delta(delta)
assert len(result) == 1
assert result[0] is delta
def _run_through_processor(
processor: SimpleStreamingEventProcessor,
delta_message: DeltaMessage,
) -> list:
"""Simulate the streaming loop from serving.py for a single delta."""
events = []
for dm in split_delta(delta_message):
target_state, tool_call = processor.resolve_target_state(dm)
if target_state == _StateType.NONE:
continue
if processor.needs_transition(target_state, tool_call):
events.extend(processor.close_current())
events.extend(processor.open(target_state, tool_call))
events.extend(processor.emit_delta(dm, None))
return events
class TestProcessorCompoundDeltas:
def test_all_three_states(self):
tc = _make_tool_call(0, name="f", arguments="{}")
delta = DeltaMessage(reasoning="r", content="c", tool_calls=[tc])
processor = SimpleStreamingEventProcessor()
events = _run_through_processor(processor, delta)
types = [e.type for e in events]
r_idx = types.index("response.reasoning_text.delta")
c_idx = types.index("response.output_text.delta")
fc_idx = types.index("response.function_call_arguments.delta")
assert r_idx < c_idx < fc_idx
def test_parallel_tool_calls(self):
tc0 = _make_tool_call(0, name="f1", arguments='{"a":1}')
tc1 = _make_tool_call(1, name="f2", arguments='{"b":2}')
delta = DeltaMessage(tool_calls=[tc0, tc1])
processor = SimpleStreamingEventProcessor()
events = _run_through_processor(processor, delta)
added = [e for e in events if e.type == "response.output_item.added"]
deltas = [
e for e in events if e.type == "response.function_call_arguments.delta"
]
assert len(added) == 2
assert len(deltas) == 2
def test_split_name_and_args_same_index(self):
"""Regression: parsers like KimiK2 emit name and args as separate
DeltaToolCalls at the same index within one DeltaMessage."""
tc_name = _make_tool_call(0, name="get_weather")
tc_args = _make_tool_call(0, arguments='{"city":"SF"}')
delta = DeltaMessage(tool_calls=[tc_name, tc_args])
processor = SimpleStreamingEventProcessor()
events = _run_through_processor(processor, delta)
deltas = [
e for e in events if e.type == "response.function_call_arguments.delta"
]
assert len(deltas) == 1
assert deltas[0].delta == '{"city":"SF"}'
def test_reasoning_to_content_transition(self):
"""Regression: the old special case in emit_delta handled this;
now split_delta handles it generically."""
processor = SimpleStreamingEventProcessor()
_run_through_processor(processor, DeltaMessage(reasoning="think"))
assert processor.state.current_state == _StateType.REASONING
events = _run_through_processor(
processor, DeltaMessage(reasoning="more", content="answer")
)
types = [e.type for e in events]
assert "response.reasoning_text.delta" in types
assert "response.output_text.delta" in types
+12 -10
View File
@@ -85,6 +85,7 @@ from vllm.entrypoints.openai.responses.streaming_events import (
emit_content_delta_events,
emit_previous_item_done_events,
emit_tool_action_events,
split_delta,
)
from vllm.entrypoints.openai.responses.utils import (
construct_input_messages,
@@ -1414,18 +1415,19 @@ class OpenAIServingResponses(OpenAIServing):
if not delta_message:
continue
target_state, tool_call = processor.resolve_target_state(delta_message)
if target_state == _StateType.NONE:
continue
for dm in split_delta(delta_message):
target_state, tool_call = processor.resolve_target_state(dm)
if target_state == _StateType.NONE:
continue
if processor.needs_transition(target_state, tool_call):
for event in processor.close_current():
yield _increment_sequence_number_and_return(event)
for event in processor.open(target_state, tool_call):
yield _increment_sequence_number_and_return(event)
if processor.needs_transition(target_state, tool_call):
for event in processor.close_current():
yield _increment_sequence_number_and_return(event)
for event in processor.open(target_state, tool_call):
yield _increment_sequence_number_and_return(event)
for event in processor.emit_delta(delta_message, output, _get_logprobs):
yield _increment_sequence_number_and_return(event)
for event in processor.emit_delta(dm, output, _get_logprobs):
yield _increment_sequence_number_and_return(event)
for event in processor.close_current():
yield _increment_sequence_number_and_return(event)
@@ -61,7 +61,7 @@ from openai.types.responses.response_reasoning_item import (
from openai_harmony import Message as HarmonyMessage
from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, DeltaToolCall
from vllm.entrypoints.openai.parser.harmony_utils import (
extract_function_from_recipient,
is_function_recipient,
@@ -1118,6 +1118,38 @@ class _StateHandlers(NamedTuple):
done_fn: Callable[..., list[StreamingResponsesResponse]]
def split_delta(delta: DeltaMessage) -> list[DeltaMessage]:
"""Decompose a DeltaMessage with multiple fields into atomic deltas.
The Responses API emits typed SSE events (one type per event), so a
compound DeltaMessage must be split before entering the state machine.
Order: reasoning -> content -> tool_calls (grouped by index).
"""
has_reasoning = delta.reasoning is not None
has_content = delta.content is not None
has_tools = bool(delta.tool_calls)
parts = int(has_reasoning) + int(has_content) + int(has_tools)
if parts <= 1 and (
not has_tools
or len({tc.index for tc in delta.tool_calls if tc.index is not None}) <= 1
):
return [delta]
deltas: list[DeltaMessage] = []
if has_reasoning:
deltas.append(DeltaMessage(reasoning=delta.reasoning))
if has_content:
deltas.append(DeltaMessage(content=delta.content))
if has_tools:
groups: dict[int | None, list[DeltaToolCall]] = {}
for tc in delta.tool_calls:
groups.setdefault(tc.index, []).append(tc)
for tcs in groups.values():
deltas.append(DeltaMessage(tool_calls=tcs))
return deltas or [delta]
class SimpleStreamingEventProcessor:
"""
State-machine processor for the simple (non-Harmony) streaming path.
@@ -1223,38 +1255,17 @@ class SimpleStreamingEventProcessor:
]
| None = None,
) -> list[StreamingResponsesResponse]:
"""
Emit incremental events for the current state from the delta.
Special case: when already in REASONING and the same delta also
carries content, we emit the reasoning delta, close reasoning,
open content, and then emit the content delta.
"""
"""Emit incremental events for the current state from the delta."""
handlers = self._STATE_HANDLERS[self.state.current_state]
events: list[StreamingResponsesResponse] = []
# Special case: reasoning -> content inside a single delta.
if (
self.state.current_state == _StateType.REASONING
and delta_message.reasoning is not None
and delta_message.content is not None
):
events.extend(handlers.delta_fn(self.state, delta_message.reasoning))
events.extend(self.close_current())
events.extend(self.open(_StateType.CONTENT))
content_handlers = self._STATE_HANDLERS[_StateType.CONTENT]
logprobs = get_logprobs(output) if get_logprobs else []
events.extend(
content_handlers.delta_fn(self.state, delta_message.content, logprobs)
)
return events
if self.state.current_state == _StateType.TOOL_CALL:
assert delta_message.tool_calls is not None
tool_call_function = delta_message.tool_calls[0].function
assert tool_call_function is not None
if tool_call_function.arguments:
return handlers.delta_fn(self.state, tool_call_function.arguments)
combined_args = ""
for tc in delta_message.tool_calls:
if tc.function is not None and tc.function.arguments:
combined_args += tc.function.arguments
if combined_args:
return handlers.delta_fn(self.state, combined_args)
return []
elif self.state.current_state == _StateType.REASONING:
assert delta_message.reasoning is not None