diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 90c3183939..25b00ff192 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -1124,3 +1124,66 @@ class TestAutoToolStreaming: if event.type == "response.function_call_arguments.delta" ] assert "".join(argument_deltas) == '{"location":"Berlin"}' + + @pytest.mark.skip_global_cleanup + @pytest.mark.asyncio + async def test_compound_content_and_tool_name_args_same_delta(self, monkeypatch): + monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) + + tool_args = '{"location":"Berlin"}' + + delta_sequence = [ + DeltaMessage( + content="Let me check.", + tool_calls=[ + DeltaToolCall( + id="call_weather", + type="function", + index=0, + function=DeltaFunctionCall(name="get_weather"), + ), + DeltaToolCall( + index=0, + function=DeltaFunctionCall(arguments=tool_args), + ), + ], + ) + ] + + events = await self._collect_events(delta_sequence) + + text_deltas = [ + event.delta + for event in events + if event.type == "response.output_text.delta" + ] + assert text_deltas == ["Let me check."] + + argument_deltas = [ + event.delta + for event in events + if event.type == "response.function_call_arguments.delta" + ] + assert argument_deltas == [tool_args] + + types = [event.type for event in events] + assert types.index("response.output_text.delta") < types.index( + "response.function_call_arguments.delta" + ) + + argument_done = [ + event + for event in events + if event.type == "response.function_call_arguments.done" + ] + assert [event.arguments for event in argument_done] == [tool_args] + + function_done = [ + event + for event in events + if event.type == "response.output_item.done" + and getattr(event.item, "type", None) == "function_call" + ] + assert len(function_done) == 1 + assert function_done[0].item.name == "get_weather" + assert function_done[0].item.arguments == tool_args diff --git a/tests/entrypoints/openai/responses/test_streaming_events.py b/tests/entrypoints/openai/responses/test_streaming_events.py new file mode 100644 index 0000000000..1f5fb8862a --- /dev/null +++ b/tests/entrypoints/openai/responses/test_streaming_events.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.entrypoints.openai.engine.protocol import ( + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, +) +from vllm.entrypoints.openai.responses.streaming_events import ( + SimpleStreamingEventProcessor, + _StateType, + split_delta, +) + + +def _make_tool_call( + index: int, name: str | None = None, arguments: str | None = None +) -> DeltaToolCall: + fn = DeltaFunctionCall(name=name, arguments=arguments) + return DeltaToolCall(index=index, function=fn) + + +class TestSplitDelta: + def test_all_three_fields(self): + tc = _make_tool_call(0, name="f") + delta = DeltaMessage(reasoning="r", content="c", tool_calls=[tc]) + result = split_delta(delta) + + assert len(result) == 3 + assert result[0].reasoning == "r" and result[0].content is None + assert result[1].content == "c" and result[1].reasoning is None + assert len(result[2].tool_calls) == 1 and result[2].content is None + + def test_tool_calls_grouped_by_index(self): + tc0 = _make_tool_call(0, name="f1") + tc1 = _make_tool_call(1, name="f2") + tc0b = _make_tool_call(0, arguments='{"a":1}') + + # Different indices → split + result = split_delta(DeltaMessage(tool_calls=[tc0, tc1])) + assert len(result) == 2 + assert result[0].tool_calls == [tc0] + assert result[1].tool_calls == [tc1] + + # Same index → stays together + delta = DeltaMessage(tool_calls=[tc0, tc0b]) + result = split_delta(delta) + assert len(result) == 1 + assert result[0] is delta + + +def _run_through_processor( + processor: SimpleStreamingEventProcessor, + delta_message: DeltaMessage, +) -> list: + """Simulate the streaming loop from serving.py for a single delta.""" + events = [] + for dm in split_delta(delta_message): + target_state, tool_call = processor.resolve_target_state(dm) + if target_state == _StateType.NONE: + continue + if processor.needs_transition(target_state, tool_call): + events.extend(processor.close_current()) + events.extend(processor.open(target_state, tool_call)) + events.extend(processor.emit_delta(dm, None)) + return events + + +class TestProcessorCompoundDeltas: + def test_all_three_states(self): + tc = _make_tool_call(0, name="f", arguments="{}") + delta = DeltaMessage(reasoning="r", content="c", tool_calls=[tc]) + + processor = SimpleStreamingEventProcessor() + events = _run_through_processor(processor, delta) + + types = [e.type for e in events] + r_idx = types.index("response.reasoning_text.delta") + c_idx = types.index("response.output_text.delta") + fc_idx = types.index("response.function_call_arguments.delta") + assert r_idx < c_idx < fc_idx + + def test_parallel_tool_calls(self): + tc0 = _make_tool_call(0, name="f1", arguments='{"a":1}') + tc1 = _make_tool_call(1, name="f2", arguments='{"b":2}') + delta = DeltaMessage(tool_calls=[tc0, tc1]) + + processor = SimpleStreamingEventProcessor() + events = _run_through_processor(processor, delta) + + added = [e for e in events if e.type == "response.output_item.added"] + deltas = [ + e for e in events if e.type == "response.function_call_arguments.delta" + ] + assert len(added) == 2 + assert len(deltas) == 2 + + def test_split_name_and_args_same_index(self): + """Regression: parsers like KimiK2 emit name and args as separate + DeltaToolCalls at the same index within one DeltaMessage.""" + tc_name = _make_tool_call(0, name="get_weather") + tc_args = _make_tool_call(0, arguments='{"city":"SF"}') + delta = DeltaMessage(tool_calls=[tc_name, tc_args]) + + processor = SimpleStreamingEventProcessor() + events = _run_through_processor(processor, delta) + + deltas = [ + e for e in events if e.type == "response.function_call_arguments.delta" + ] + assert len(deltas) == 1 + assert deltas[0].delta == '{"city":"SF"}' + + def test_reasoning_to_content_transition(self): + """Regression: the old special case in emit_delta handled this; + now split_delta handles it generically.""" + processor = SimpleStreamingEventProcessor() + _run_through_processor(processor, DeltaMessage(reasoning="think")) + assert processor.state.current_state == _StateType.REASONING + + events = _run_through_processor( + processor, DeltaMessage(reasoning="more", content="answer") + ) + types = [e.type for e in events] + assert "response.reasoning_text.delta" in types + assert "response.output_text.delta" in types diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 68f71bd5d4..3c80e3d015 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -85,6 +85,7 @@ from vllm.entrypoints.openai.responses.streaming_events import ( emit_content_delta_events, emit_previous_item_done_events, emit_tool_action_events, + split_delta, ) from vllm.entrypoints.openai.responses.utils import ( construct_input_messages, @@ -1414,18 +1415,19 @@ class OpenAIServingResponses(OpenAIServing): if not delta_message: continue - target_state, tool_call = processor.resolve_target_state(delta_message) - if target_state == _StateType.NONE: - continue + for dm in split_delta(delta_message): + target_state, tool_call = processor.resolve_target_state(dm) + if target_state == _StateType.NONE: + continue - if processor.needs_transition(target_state, tool_call): - for event in processor.close_current(): - yield _increment_sequence_number_and_return(event) - for event in processor.open(target_state, tool_call): - yield _increment_sequence_number_and_return(event) + if processor.needs_transition(target_state, tool_call): + for event in processor.close_current(): + yield _increment_sequence_number_and_return(event) + for event in processor.open(target_state, tool_call): + yield _increment_sequence_number_and_return(event) - for event in processor.emit_delta(delta_message, output, _get_logprobs): - yield _increment_sequence_number_and_return(event) + for event in processor.emit_delta(dm, output, _get_logprobs): + yield _increment_sequence_number_and_return(event) for event in processor.close_current(): yield _increment_sequence_number_and_return(event) diff --git a/vllm/entrypoints/openai/responses/streaming_events.py b/vllm/entrypoints/openai/responses/streaming_events.py index b49c46fc53..9c463b3d5b 100644 --- a/vllm/entrypoints/openai/responses/streaming_events.py +++ b/vllm/entrypoints/openai/responses/streaming_events.py @@ -61,7 +61,7 @@ from openai.types.responses.response_reasoning_item import ( from openai_harmony import Message as HarmonyMessage from vllm.entrypoints.mcp.tool_server import ToolServer -from vllm.entrypoints.openai.engine.protocol import DeltaMessage +from vllm.entrypoints.openai.engine.protocol import DeltaMessage, DeltaToolCall from vllm.entrypoints.openai.parser.harmony_utils import ( extract_function_from_recipient, is_function_recipient, @@ -1118,6 +1118,38 @@ class _StateHandlers(NamedTuple): done_fn: Callable[..., list[StreamingResponsesResponse]] +def split_delta(delta: DeltaMessage) -> list[DeltaMessage]: + """Decompose a DeltaMessage with multiple fields into atomic deltas. + + The Responses API emits typed SSE events (one type per event), so a + compound DeltaMessage must be split before entering the state machine. + Order: reasoning -> content -> tool_calls (grouped by index). + """ + has_reasoning = delta.reasoning is not None + has_content = delta.content is not None + has_tools = bool(delta.tool_calls) + parts = int(has_reasoning) + int(has_content) + int(has_tools) + + if parts <= 1 and ( + not has_tools + or len({tc.index for tc in delta.tool_calls if tc.index is not None}) <= 1 + ): + return [delta] + + deltas: list[DeltaMessage] = [] + if has_reasoning: + deltas.append(DeltaMessage(reasoning=delta.reasoning)) + if has_content: + deltas.append(DeltaMessage(content=delta.content)) + if has_tools: + groups: dict[int | None, list[DeltaToolCall]] = {} + for tc in delta.tool_calls: + groups.setdefault(tc.index, []).append(tc) + for tcs in groups.values(): + deltas.append(DeltaMessage(tool_calls=tcs)) + return deltas or [delta] + + class SimpleStreamingEventProcessor: """ State-machine processor for the simple (non-Harmony) streaming path. @@ -1223,38 +1255,17 @@ class SimpleStreamingEventProcessor: ] | None = None, ) -> list[StreamingResponsesResponse]: - """ - Emit incremental events for the current state from the delta. - - Special case: when already in REASONING and the same delta also - carries content, we emit the reasoning delta, close reasoning, - open content, and then emit the content delta. - """ + """Emit incremental events for the current state from the delta.""" handlers = self._STATE_HANDLERS[self.state.current_state] - events: list[StreamingResponsesResponse] = [] - - # Special case: reasoning -> content inside a single delta. - if ( - self.state.current_state == _StateType.REASONING - and delta_message.reasoning is not None - and delta_message.content is not None - ): - events.extend(handlers.delta_fn(self.state, delta_message.reasoning)) - events.extend(self.close_current()) - events.extend(self.open(_StateType.CONTENT)) - content_handlers = self._STATE_HANDLERS[_StateType.CONTENT] - logprobs = get_logprobs(output) if get_logprobs else [] - events.extend( - content_handlers.delta_fn(self.state, delta_message.content, logprobs) - ) - return events if self.state.current_state == _StateType.TOOL_CALL: assert delta_message.tool_calls is not None - tool_call_function = delta_message.tool_calls[0].function - assert tool_call_function is not None - if tool_call_function.arguments: - return handlers.delta_fn(self.state, tool_call_function.arguments) + combined_args = "" + for tc in delta_message.tool_calls: + if tc.function is not None and tc.function.arguments: + combined_args += tc.function.arguments + if combined_args: + return handlers.delta_fn(self.state, combined_args) return [] elif self.state.current_state == _StateType.REASONING: assert delta_message.reasoning is not None