From ae8f41238cc7316781ef4359890940749735b209 Mon Sep 17 00:00:00 2001 From: alberto Date: Fri, 16 Oct 2026 17:54:42 +0100 Subject: [PATCH] [Bugfix][Parser] Honor a prompt-closed reasoning block in non-streaming parse and reasoning token counts (#61857) Signed-off-by: Alberto Perdomo --- tests/parser/engine/test_qwen3_reasoning.py | 34 ++++ tests/parser/test_parse.py | 179 ++++++++++++++++++++ vllm/parser/abstract_parser.py | 5 + vllm/parser/engine/adapters.py | 8 + vllm/parser/engine/parser_engine.py | 20 ++- vllm/reasoning/abs_reasoning_parsers.py | 9 + 6 files changed, 253 insertions(+), 2 deletions(-) diff --git a/tests/parser/engine/test_qwen3_reasoning.py b/tests/parser/engine/test_qwen3_reasoning.py index 3f285e4d03..e9c751c3cb 100644 --- a/tests/parser/engine/test_qwen3_reasoning.py +++ b/tests/parser/engine/test_qwen3_reasoning.py @@ -380,6 +380,40 @@ class TestDelegatingPromptDetection: assert delta.reasoning is None assert delta.content == "answer" + def test_prior_turn_think_end_keeps_non_streaming_reasoning( + self, mock_tokenizer, mock_request + ): + parser = _Qwen3DelegatingParser(mock_tokenizer) + parser.set_prompt_token_ids([_THINK_END_ID, _IM_END_ID, _IM_START_ID, _TEXT_ID]) + output_ids = [ord("a"), _THINK_END_ID, ord("b")] + + reasoning, content, _ = parser.parse( + mock_tokenizer.decode(output_ids), + mock_request, + model_output_token_ids=output_ids, + ) + + assert reasoning == "a" + assert content == "b" + assert parser.count_reasoning_tokens(output_ids) > 0 + + def test_think_end_in_current_turn_parses_output_as_content( + self, mock_tokenizer, mock_request + ): + parser = _Qwen3DelegatingParser(mock_tokenizer) + parser.set_prompt_token_ids([_IM_START_ID, _THINK_START_ID, _THINK_END_ID]) + output_ids = [ord("a"), ord("b")] + + reasoning, content, _ = parser.parse( + mock_tokenizer.decode(output_ids), + mock_request, + model_output_token_ids=output_ids, + ) + + assert reasoning is None + assert content == "ab" + assert parser.count_reasoning_tokens(output_ids) == 0 + class TestStreaming: def test_basic_streaming(self, parser): diff --git a/tests/parser/test_parse.py b/tests/parser/test_parse.py index 2379b8a4c2..4eea8db73e 100644 --- a/tests/parser/test_parse.py +++ b/tests/parser/test_parse.py @@ -9,6 +9,9 @@ import pytest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.parser_manager import ParserManager +from vllm.parser.qwen3 import qwen3_config from vllm.parser.utils import count_history_tool_calls from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser @@ -417,3 +420,179 @@ def test_parse_auto_tools_no_calls_returns_none(tokenizer): assert reasoning is None assert content == PLAIN_TEXT assert tool_calls is None + + +# -------------------------------------------------------------------------- +# Prompt that already closes the reasoning block +# -------------------------------------------------------------------------- + +CONTINUE_MESSAGES = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "Sure,"}, +] +REASONED_OUTPUT = "\nlet me think\n\n\n" + PLAIN_TEXT +TOOL_TURN_MESSAGES = [ + {"role": "user", "content": "Weather in Paris?"}, + { + "role": "assistant", + "content": "\nneed a tool\n\n\n", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, +] + + +def reasoning_parser(tokenizer, name="qwen3"): + parser_cls = ParserManager.get_parser( + reasoning_parser_name=name, tokenizer=tokenizer + ) + return parser_cls(tokenizer, None) + + +def encode(tokenizer, text): + return tokenizer.encode(text, add_special_tokens=False) + + +def prompt_case(tokenizer, case): + """(rendered prompt ids, request, model output) for each prompt shape.""" + if case == "continue_final_message": + # The template renders an empty, closed think block before "Sure,". + prompt = tokenizer.apply_chat_template( + CONTINUE_MESSAGES, + tokenize=False, + add_generation_prompt=False, + continue_final_message=True, + ) + request = make_request( + messages=CONTINUE_MESSAGES, + add_generation_prompt=False, + continue_final_message=True, + ) + return encode(tokenizer, prompt), request, " here is the answer" + if case == "template_closed_thinking": + # The template switches thinking off, but the parser is not told + # through chat_template_kwargs (#53284). + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": "hi"}], + tokenize=False, + add_generation_prompt=True, + enable_thinking=False, + ) + return encode(tokenizer, prompt), make_request(), PLAIN_TEXT + if case == "earlier_turn_closed_reasoning": + # Only a previous assistant turn closes reasoning; this one is open. + prompt = tokenizer.apply_chat_template( + TOOL_TURN_MESSAGES, + tools=[{"type": "function", "function": {"name": "get_weather"}}], + tokenize=False, + add_generation_prompt=True, + ) + return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT + assert case == "reasoning_open" + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": "hi"}], + tokenize=False, + add_generation_prompt=True, + ) + return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT + + +@pytest.mark.parametrize("case", ["continue_final_message", "template_closed_thinking"]) +def test_parse_prompt_closed_reasoning_is_content(tokenizer, case): + prompt_ids, request, output = prompt_case(tokenizer, case) + parser = reasoning_parser(tokenizer) + parser.set_prompt_token_ids(prompt_ids) + + reasoning, content, _ = parser.parse( + output, request, model_output_token_ids=encode(tokenizer, output) + ) + + assert reasoning is None + assert content == output + assert parser.count_reasoning_tokens(encode(tokenizer, output)) == 0 + + +@pytest.mark.parametrize( + ("parser_name", "case"), + [ + ("qwen3", "reasoning_open"), + ("qwen3", "earlier_turn_closed_reasoning"), + # BaseThinkingReasoningParser scans the prompt without turn + # boundaries, so it must not decide non-streaming classification. + ("deepseek_r1", "earlier_turn_closed_reasoning"), + ], +) +def test_parse_prompt_open_reasoning_still_extracted(tokenizer, parser_name, case): + prompt_ids, request, output = prompt_case(tokenizer, case) + parser = reasoning_parser(tokenizer, parser_name) + parser.set_prompt_token_ids(prompt_ids) + + reasoning, content, _ = parser.parse( + output, request, model_output_token_ids=encode(tokenizer, output) + ) + + assert reasoning.strip() == "let me think" + assert content.strip() == PLAIN_TEXT + assert parser.count_reasoning_tokens(encode(tokenizer, output)) > 0 + + +def test_parse_prompt_ignored_without_turn_boundaries(tokenizer): + """Without turn boundaries, an earlier turn's looks like this + turn's, so the engine must not start the output in content.""" + prompt_ids, request, output = prompt_case( + tokenizer, "earlier_turn_closed_reasoning" + ) + parser = ParserEngine(tokenizer, parser_engine_config=qwen3_config(thinking=True)) + parser.set_prompt_token_ids(prompt_ids) + + reasoning, content, _ = parser.parse( + output, request, model_output_token_ids=encode(tokenizer, output) + ) + + assert reasoning.strip() == "let me think" + assert content.strip() == PLAIN_TEXT + + +@pytest.mark.parametrize( + "case", + [ + "continue_final_message", + "template_closed_thinking", + "reasoning_open", + "earlier_turn_closed_reasoning", + ], +) +def test_prompt_reasoning_streaming_matches_non_streaming(tokenizer, case): + prompt_ids, request, output = prompt_case(tokenizer, case) + output_ids = encode(tokenizer, output) + + parser = reasoning_parser(tokenizer) + parser.set_prompt_token_ids(prompt_ids) + reasoning, content, _ = parser.parse( + output, request, model_output_token_ids=output_ids + ) + non_streaming = (reasoning or "", content or "") + non_streaming_count = parser.count_reasoning_tokens(output_ids) + + parser = reasoning_parser(tokenizer) + streamed_reasoning = streamed_content = "" + for i, token_id in enumerate(output_ids): + delta = parser.parse_delta( + tokenizer.decode([token_id]), + [token_id], + request, + prompt_token_ids=prompt_ids, + finished=i == len(output_ids) - 1, + ) + if delta is not None: + streamed_reasoning += delta.reasoning or "" + streamed_content += delta.content or "" + + assert (streamed_reasoning, streamed_content) == non_streaming + assert parser.count_reasoning_tokens(output_ids) == non_streaming_count diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index fbddc45783..287cad5c24 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -769,6 +769,10 @@ class DelegatingParser(Parser): return False return self._reasoning_parser.is_reasoning_end(input_ids) + def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None: + if self._reasoning_parser is not None: + self._reasoning_parser.set_prompt_token_ids(prompt_token_ids) + def _is_reasoning_end_streaming( self, input_ids: list[int], delta_ids: list[int] ) -> bool: @@ -858,6 +862,7 @@ class DelegatingParser(Parser): if not state.prompt_reasoning_checked and prompt_token_ids is not None: state.prompt_reasoning_checked = True + self.set_prompt_token_ids(prompt_token_ids) if self._reasoning_parser is None or self.is_reasoning_end( prompt_token_ids ): diff --git a/vllm/parser/engine/adapters.py b/vllm/parser/engine/adapters.py index a4a307a43a..52764e9fa8 100644 --- a/vllm/parser/engine/adapters.py +++ b/vllm/parser/engine/adapters.py @@ -68,6 +68,11 @@ class ParserEngineReasoningAdapter(ReasoningParser): def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None: return self._parser_engine.find_reasoning_end(token_ids) + def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None: + self._parser_engine.set_prompt_token_ids(prompt_token_ids) + if self._counting_parser_engine is not None: + self._counting_parser_engine.set_prompt_token_ids(prompt_token_ids) + def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None: self._parser_engine.adjust_initial_state_from_prompt(prompt_token_ids) @@ -148,6 +153,9 @@ class ParserEngineReasoningAdapter(ReasoningParser): self._counting_parser_engine = self._parser_engine_cls( self.model_tokenizer, **self._parser_engine_kwargs ) # type: ignore[call-arg] + self._counting_parser_engine._prompt_closes_reasoning = ( + self._parser_engine._prompt_closes_reasoning + ) self._counting_parser_engine._single_pass_parse( self.model_tokenizer.decode(token_ids), token_ids ) diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index bbb3605ed7..de695f4bdf 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -147,6 +147,7 @@ class ParserEngine(Parser): or parser_engine_config.initial_state == ParserState.REASONING ) self._reasoning_ended: bool = not self._has_reasoning + self._prompt_closes_reasoning: bool = False self._streaming_initialized: bool = False self._prompt_streaming_prepared: bool = False @@ -231,6 +232,18 @@ class ParserEngine(Parser): self._streaming_initialized = True self._reset(initial_state=initial_state) + def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None: + """Start parsing in content when the prompt already closes reasoning, + as with continue_final_message or a template that switches thinking + off. Only configs with turn boundaries can tell a closed block in + this turn from one in an earlier turn, so others are left as is. + """ + self._prompt_closes_reasoning = ( + self.parser_engine_config.initial_state is ParserState.REASONING + and bool(self._turn_boundary_token_ids) + and self.is_reasoning_end(list(prompt_token_ids)) + ) + def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None: """See :meth:`ReasoningParser.adjust_initial_state_from_prompt`.""" return @@ -242,8 +255,11 @@ class ParserEngine(Parser): return None def _reset(self, initial_state: ParserState | None = None) -> None: - self._engine.reset(initial_state=initial_state) - self._reasoning_ended = not self._has_reasoning + prompt_closed = initial_state is None and self._prompt_closes_reasoning + self._engine.reset( + initial_state=ParserState.CONTENT if prompt_closed else initial_state + ) + self._reasoning_ended = not self._has_reasoning or prompt_closed self._tool_slots.clear() self._deferred_content = "" self._deferred_reasoning = "" diff --git a/vllm/reasoning/abs_reasoning_parsers.py b/vllm/reasoning/abs_reasoning_parsers.py index accc8c9c27..f55cbfd686 100644 --- a/vllm/reasoning/abs_reasoning_parsers.py +++ b/vllm/reasoning/abs_reasoning_parsers.py @@ -192,6 +192,15 @@ class ReasoningParser: """Adjust request parameters; override in subclasses as needed.""" return request + def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None: + """Provide the rendered prompt before the output is parsed. + + Parsers that can tell whether the prompt already closes reasoning + (e.g. continue_final_message) use it to classify the output. + Default is a no-op; override in subclasses as needed. + """ + return + def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None: """Hook called once at the start of streaming with the prompt tokens.