[Bugfix][Frontend] streaming tool-call serializer drops first args chunk when name and args share a DeltaMessage (#42683)
Signed-off-by: ignaciosica <mignacio.sica@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -1124,3 +1124,66 @@ class TestAutoToolStreaming:
|
||||
if event.type == "response.function_call_arguments.delta"
|
||||
]
|
||||
assert "".join(argument_deltas) == '{"location":"Berlin"}'
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.asyncio
|
||||
async def test_compound_content_and_tool_name_args_same_delta(self, monkeypatch):
|
||||
monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False)
|
||||
|
||||
tool_args = '{"location":"Berlin"}'
|
||||
|
||||
delta_sequence = [
|
||||
DeltaMessage(
|
||||
content="Let me check.",
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
id="call_weather",
|
||||
type="function",
|
||||
index=0,
|
||||
function=DeltaFunctionCall(name="get_weather"),
|
||||
),
|
||||
DeltaToolCall(
|
||||
index=0,
|
||||
function=DeltaFunctionCall(arguments=tool_args),
|
||||
),
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
events = await self._collect_events(delta_sequence)
|
||||
|
||||
text_deltas = [
|
||||
event.delta
|
||||
for event in events
|
||||
if event.type == "response.output_text.delta"
|
||||
]
|
||||
assert text_deltas == ["Let me check."]
|
||||
|
||||
argument_deltas = [
|
||||
event.delta
|
||||
for event in events
|
||||
if event.type == "response.function_call_arguments.delta"
|
||||
]
|
||||
assert argument_deltas == [tool_args]
|
||||
|
||||
types = [event.type for event in events]
|
||||
assert types.index("response.output_text.delta") < types.index(
|
||||
"response.function_call_arguments.delta"
|
||||
)
|
||||
|
||||
argument_done = [
|
||||
event
|
||||
for event in events
|
||||
if event.type == "response.function_call_arguments.done"
|
||||
]
|
||||
assert [event.arguments for event in argument_done] == [tool_args]
|
||||
|
||||
function_done = [
|
||||
event
|
||||
for event in events
|
||||
if event.type == "response.output_item.done"
|
||||
and getattr(event.item, "type", None) == "function_call"
|
||||
]
|
||||
assert len(function_done) == 1
|
||||
assert function_done[0].item.name == "get_weather"
|
||||
assert function_done[0].item.arguments == tool_args
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.streaming_events import (
|
||||
SimpleStreamingEventProcessor,
|
||||
_StateType,
|
||||
split_delta,
|
||||
)
|
||||
|
||||
|
||||
def _make_tool_call(
|
||||
index: int, name: str | None = None, arguments: str | None = None
|
||||
) -> DeltaToolCall:
|
||||
fn = DeltaFunctionCall(name=name, arguments=arguments)
|
||||
return DeltaToolCall(index=index, function=fn)
|
||||
|
||||
|
||||
class TestSplitDelta:
|
||||
def test_all_three_fields(self):
|
||||
tc = _make_tool_call(0, name="f")
|
||||
delta = DeltaMessage(reasoning="r", content="c", tool_calls=[tc])
|
||||
result = split_delta(delta)
|
||||
|
||||
assert len(result) == 3
|
||||
assert result[0].reasoning == "r" and result[0].content is None
|
||||
assert result[1].content == "c" and result[1].reasoning is None
|
||||
assert len(result[2].tool_calls) == 1 and result[2].content is None
|
||||
|
||||
def test_tool_calls_grouped_by_index(self):
|
||||
tc0 = _make_tool_call(0, name="f1")
|
||||
tc1 = _make_tool_call(1, name="f2")
|
||||
tc0b = _make_tool_call(0, arguments='{"a":1}')
|
||||
|
||||
# Different indices → split
|
||||
result = split_delta(DeltaMessage(tool_calls=[tc0, tc1]))
|
||||
assert len(result) == 2
|
||||
assert result[0].tool_calls == [tc0]
|
||||
assert result[1].tool_calls == [tc1]
|
||||
|
||||
# Same index → stays together
|
||||
delta = DeltaMessage(tool_calls=[tc0, tc0b])
|
||||
result = split_delta(delta)
|
||||
assert len(result) == 1
|
||||
assert result[0] is delta
|
||||
|
||||
|
||||
def _run_through_processor(
|
||||
processor: SimpleStreamingEventProcessor,
|
||||
delta_message: DeltaMessage,
|
||||
) -> list:
|
||||
"""Simulate the streaming loop from serving.py for a single delta."""
|
||||
events = []
|
||||
for dm in split_delta(delta_message):
|
||||
target_state, tool_call = processor.resolve_target_state(dm)
|
||||
if target_state == _StateType.NONE:
|
||||
continue
|
||||
if processor.needs_transition(target_state, tool_call):
|
||||
events.extend(processor.close_current())
|
||||
events.extend(processor.open(target_state, tool_call))
|
||||
events.extend(processor.emit_delta(dm, None))
|
||||
return events
|
||||
|
||||
|
||||
class TestProcessorCompoundDeltas:
|
||||
def test_all_three_states(self):
|
||||
tc = _make_tool_call(0, name="f", arguments="{}")
|
||||
delta = DeltaMessage(reasoning="r", content="c", tool_calls=[tc])
|
||||
|
||||
processor = SimpleStreamingEventProcessor()
|
||||
events = _run_through_processor(processor, delta)
|
||||
|
||||
types = [e.type for e in events]
|
||||
r_idx = types.index("response.reasoning_text.delta")
|
||||
c_idx = types.index("response.output_text.delta")
|
||||
fc_idx = types.index("response.function_call_arguments.delta")
|
||||
assert r_idx < c_idx < fc_idx
|
||||
|
||||
def test_parallel_tool_calls(self):
|
||||
tc0 = _make_tool_call(0, name="f1", arguments='{"a":1}')
|
||||
tc1 = _make_tool_call(1, name="f2", arguments='{"b":2}')
|
||||
delta = DeltaMessage(tool_calls=[tc0, tc1])
|
||||
|
||||
processor = SimpleStreamingEventProcessor()
|
||||
events = _run_through_processor(processor, delta)
|
||||
|
||||
added = [e for e in events if e.type == "response.output_item.added"]
|
||||
deltas = [
|
||||
e for e in events if e.type == "response.function_call_arguments.delta"
|
||||
]
|
||||
assert len(added) == 2
|
||||
assert len(deltas) == 2
|
||||
|
||||
def test_split_name_and_args_same_index(self):
|
||||
"""Regression: parsers like KimiK2 emit name and args as separate
|
||||
DeltaToolCalls at the same index within one DeltaMessage."""
|
||||
tc_name = _make_tool_call(0, name="get_weather")
|
||||
tc_args = _make_tool_call(0, arguments='{"city":"SF"}')
|
||||
delta = DeltaMessage(tool_calls=[tc_name, tc_args])
|
||||
|
||||
processor = SimpleStreamingEventProcessor()
|
||||
events = _run_through_processor(processor, delta)
|
||||
|
||||
deltas = [
|
||||
e for e in events if e.type == "response.function_call_arguments.delta"
|
||||
]
|
||||
assert len(deltas) == 1
|
||||
assert deltas[0].delta == '{"city":"SF"}'
|
||||
|
||||
def test_reasoning_to_content_transition(self):
|
||||
"""Regression: the old special case in emit_delta handled this;
|
||||
now split_delta handles it generically."""
|
||||
processor = SimpleStreamingEventProcessor()
|
||||
_run_through_processor(processor, DeltaMessage(reasoning="think"))
|
||||
assert processor.state.current_state == _StateType.REASONING
|
||||
|
||||
events = _run_through_processor(
|
||||
processor, DeltaMessage(reasoning="more", content="answer")
|
||||
)
|
||||
types = [e.type for e in events]
|
||||
assert "response.reasoning_text.delta" in types
|
||||
assert "response.output_text.delta" in types
|
||||
@@ -85,6 +85,7 @@ from vllm.entrypoints.openai.responses.streaming_events import (
|
||||
emit_content_delta_events,
|
||||
emit_previous_item_done_events,
|
||||
emit_tool_action_events,
|
||||
split_delta,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.utils import (
|
||||
construct_input_messages,
|
||||
@@ -1414,18 +1415,19 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
if not delta_message:
|
||||
continue
|
||||
|
||||
target_state, tool_call = processor.resolve_target_state(delta_message)
|
||||
if target_state == _StateType.NONE:
|
||||
continue
|
||||
for dm in split_delta(delta_message):
|
||||
target_state, tool_call = processor.resolve_target_state(dm)
|
||||
if target_state == _StateType.NONE:
|
||||
continue
|
||||
|
||||
if processor.needs_transition(target_state, tool_call):
|
||||
for event in processor.close_current():
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
for event in processor.open(target_state, tool_call):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
if processor.needs_transition(target_state, tool_call):
|
||||
for event in processor.close_current():
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
for event in processor.open(target_state, tool_call):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
|
||||
for event in processor.emit_delta(delta_message, output, _get_logprobs):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
for event in processor.emit_delta(dm, output, _get_logprobs):
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
|
||||
for event in processor.close_current():
|
||||
yield _increment_sequence_number_and_return(event)
|
||||
|
||||
@@ -61,7 +61,7 @@ from openai.types.responses.response_reasoning_item import (
|
||||
from openai_harmony import Message as HarmonyMessage
|
||||
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage, DeltaToolCall
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
extract_function_from_recipient,
|
||||
is_function_recipient,
|
||||
@@ -1118,6 +1118,38 @@ class _StateHandlers(NamedTuple):
|
||||
done_fn: Callable[..., list[StreamingResponsesResponse]]
|
||||
|
||||
|
||||
def split_delta(delta: DeltaMessage) -> list[DeltaMessage]:
|
||||
"""Decompose a DeltaMessage with multiple fields into atomic deltas.
|
||||
|
||||
The Responses API emits typed SSE events (one type per event), so a
|
||||
compound DeltaMessage must be split before entering the state machine.
|
||||
Order: reasoning -> content -> tool_calls (grouped by index).
|
||||
"""
|
||||
has_reasoning = delta.reasoning is not None
|
||||
has_content = delta.content is not None
|
||||
has_tools = bool(delta.tool_calls)
|
||||
parts = int(has_reasoning) + int(has_content) + int(has_tools)
|
||||
|
||||
if parts <= 1 and (
|
||||
not has_tools
|
||||
or len({tc.index for tc in delta.tool_calls if tc.index is not None}) <= 1
|
||||
):
|
||||
return [delta]
|
||||
|
||||
deltas: list[DeltaMessage] = []
|
||||
if has_reasoning:
|
||||
deltas.append(DeltaMessage(reasoning=delta.reasoning))
|
||||
if has_content:
|
||||
deltas.append(DeltaMessage(content=delta.content))
|
||||
if has_tools:
|
||||
groups: dict[int | None, list[DeltaToolCall]] = {}
|
||||
for tc in delta.tool_calls:
|
||||
groups.setdefault(tc.index, []).append(tc)
|
||||
for tcs in groups.values():
|
||||
deltas.append(DeltaMessage(tool_calls=tcs))
|
||||
return deltas or [delta]
|
||||
|
||||
|
||||
class SimpleStreamingEventProcessor:
|
||||
"""
|
||||
State-machine processor for the simple (non-Harmony) streaming path.
|
||||
@@ -1223,38 +1255,17 @@ class SimpleStreamingEventProcessor:
|
||||
]
|
||||
| None = None,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
"""
|
||||
Emit incremental events for the current state from the delta.
|
||||
|
||||
Special case: when already in REASONING and the same delta also
|
||||
carries content, we emit the reasoning delta, close reasoning,
|
||||
open content, and then emit the content delta.
|
||||
"""
|
||||
"""Emit incremental events for the current state from the delta."""
|
||||
handlers = self._STATE_HANDLERS[self.state.current_state]
|
||||
events: list[StreamingResponsesResponse] = []
|
||||
|
||||
# Special case: reasoning -> content inside a single delta.
|
||||
if (
|
||||
self.state.current_state == _StateType.REASONING
|
||||
and delta_message.reasoning is not None
|
||||
and delta_message.content is not None
|
||||
):
|
||||
events.extend(handlers.delta_fn(self.state, delta_message.reasoning))
|
||||
events.extend(self.close_current())
|
||||
events.extend(self.open(_StateType.CONTENT))
|
||||
content_handlers = self._STATE_HANDLERS[_StateType.CONTENT]
|
||||
logprobs = get_logprobs(output) if get_logprobs else []
|
||||
events.extend(
|
||||
content_handlers.delta_fn(self.state, delta_message.content, logprobs)
|
||||
)
|
||||
return events
|
||||
|
||||
if self.state.current_state == _StateType.TOOL_CALL:
|
||||
assert delta_message.tool_calls is not None
|
||||
tool_call_function = delta_message.tool_calls[0].function
|
||||
assert tool_call_function is not None
|
||||
if tool_call_function.arguments:
|
||||
return handlers.delta_fn(self.state, tool_call_function.arguments)
|
||||
combined_args = ""
|
||||
for tc in delta_message.tool_calls:
|
||||
if tc.function is not None and tc.function.arguments:
|
||||
combined_args += tc.function.arguments
|
||||
if combined_args:
|
||||
return handlers.delta_fn(self.state, combined_args)
|
||||
return []
|
||||
elif self.state.current_state == _StateType.REASONING:
|
||||
assert delta_message.reasoning is not None
|
||||
|
||||
Reference in New Issue
Block a user