From b32fe416ea1a036cda7373fc701b12b75bf86ac9 Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Sat, 23 May 2026 19:18:30 -0400 Subject: [PATCH] [Bugfix] Fix reasoning dropped on streaming boundary deltas (#42691) Signed-off-by: sfeng33 <4florafeng@gmail.com> --- tests/parser/test_streaming.py | 72 ++++++++++++++++++++++++++++++++++ vllm/parser/abstract_parser.py | 30 ++++++++++---- 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/tests/parser/test_streaming.py b/tests/parser/test_streaming.py index d9194d48ed..c4409117ad 100644 --- a/tests/parser/test_streaming.py +++ b/tests/parser/test_streaming.py @@ -137,6 +137,78 @@ def test_parse_delta_both_parsers(tokenizer, request_obj): assert json.loads(tool_args) == {"city": "Dallas"} +def stream_chunks(parser, tokenizer, chunks, request_obj): + """Stream pre-split token-ID chunks through the parser.""" + results: list[DeltaMessage | None] = [] + prompt_token_ids: list[int] | None = [] + for chunk in chunks: + delta_text = tokenizer.decode(chunk) + result = parser.parse_delta( + delta_text, chunk, request_obj, prompt_token_ids=prompt_token_ids + ) + prompt_token_ids = None + results.append(result) + return results + + +def _boundary_chunks(tokenizer, parser): + """Split MODEL_OUTPUT into 3 chunks that straddle the boundary.""" + token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False) + end_token_id = parser._reasoning_parser.end_token_id + end_idx = token_ids.index(end_token_id) + return [ + token_ids[: end_idx - 1], + token_ids[end_idx - 1 : end_idx + 2], + token_ids[end_idx + 2 :], + ] + + +def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj): + """Regression: reasoning must not be lost when a multi-token delta + spans the reasoning/tool-call boundary.""" + parser = make_parser(tokenizer, reasoning=True, tool=True) + chunks = _boundary_chunks(tokenizer, parser) + results = stream_chunks(parser, tokenizer, chunks, request_obj) + reasoning, content, tool_calls = collect_fields(results) + + assert "think about this" in reasoning + assert content == "" + assert len(tool_calls) > 0 + assert tool_calls[0].function.name == "get_weather" + tool_args = "".join( + tc.function.arguments for tc in tool_calls if tc.function.arguments + ) + assert json.loads(tool_args) == {"city": "Dallas"} + + +def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj): + """When no tool parser is active, boundary-spanning chunks must still + preserve reasoning and pass post- text as content.""" + parser = make_parser(tokenizer, reasoning=True, tool=False) + chunks = _boundary_chunks(tokenizer, parser) + results = stream_chunks(parser, tokenizer, chunks, request_obj) + reasoning, content, tool_calls = collect_fields(results) + + assert "think about this" in reasoning + assert len(tool_calls) == 0 + assert "" in content + assert "get_weather" in content + + +def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj): + """Regression: must not leak into content when streaming + token-by-token with reasoning=True, tool=False.""" + parser = make_parser(tokenizer, reasoning=True, tool=False) + results = stream_text( + parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[] + ) + reasoning, content, tool_calls = collect_fields(results) + + assert "let me think about this" in reasoning + assert "" not in content + assert "" not in content + + def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj): """Regression test for vllm-project/vllm#40466. diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 46acc7fb15..2a13f13860 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -634,6 +634,13 @@ class DelegatingParser(Parser): return False return self._reasoning_parser.is_reasoning_end(input_ids) + def is_reasoning_end_streaming( + self, input_ids: list[int], delta_ids: list[int] + ) -> bool: + if self._reasoning_parser is None: + return False + return self._reasoning_parser.is_reasoning_end_streaming(input_ids, delta_ids) + def extract_content_ids(self, input_ids: list[int]) -> list[int]: if self._reasoning_parser is None: return input_ids @@ -679,15 +686,16 @@ class DelegatingParser(Parser): current_token_ids=current_token_ids, delta_token_ids=delta_token_ids, ) - # Hand off remaining content to tool parser - if self._tool_parser and self.is_reasoning_end(delta_token_ids): + if self.is_reasoning_end_streaming(current_token_ids, delta_token_ids): state.reasoning_ended = True current_token_ids = self.extract_content_ids(delta_token_ids) - if delta_message and delta_message.content: - current_text = delta_message.content - delta_message.content = None - else: - current_text = "" + current_text = ( + delta_message.content + if delta_message and delta_message.content + else "" + ) + delta_text = current_text + delta_token_ids = current_token_ids # Tool call extraction if self._in_tool_call_phase(state): @@ -698,6 +706,9 @@ class DelegatingParser(Parser): delta_text = current_text delta_token_ids = current_token_ids + # A boundary delta may carry both reasoning and tool call, + # save it before the tool parser overwrites delta_message. + reasoning = delta_message.reasoning if delta_message else None delta_message, state.function_name_returned = ( self._extract_tool_calls_streaming( previous_text=state.previous_text, @@ -712,6 +723,11 @@ class DelegatingParser(Parser): function_name_returned=state.function_name_returned, ) ) + if reasoning: + if not delta_message: + delta_message = DeltaMessage() + delta_message.reasoning = reasoning + if ( delta_message and delta_message.tool_calls