From 24bfbd5d4d530a2657a8925e5d12e86b1ccb9cca Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Tue, 15 Sep 2026 06:07:58 -0400 Subject: [PATCH] [Refactor] Derive is_reasoning_end from the engine grammar (#56200) Signed-off-by: sfeng33 <4florafeng@gmail.com> --- .../engine/test_gemma4_streaming_reasoning.py | 33 +++++++++++++++ tests/parser/engine/test_qwen3_reasoning.py | 42 +++++++++++++++++-- vllm/parser/deepseek_v32.py | 1 + vllm/parser/deepseek_v4.py | 1 + vllm/parser/engine/parser_engine.py | 36 +++++++++------- vllm/parser/engine/parser_engine_config.py | 2 + vllm/parser/gemma4.py | 34 ++++----------- vllm/parser/glm47_moe.py | 6 +-- vllm/parser/inkling.py | 1 + vllm/parser/kimi_k2.py | 19 +-------- vllm/parser/minimax_m2.py | 1 + vllm/parser/mistral.py | 19 +-------- vllm/parser/qwen3.py | 33 ++------------- 13 files changed, 111 insertions(+), 117 deletions(-) diff --git a/tests/parser/engine/test_gemma4_streaming_reasoning.py b/tests/parser/engine/test_gemma4_streaming_reasoning.py index 74224c15b7..ae21d69fc3 100644 --- a/tests/parser/engine/test_gemma4_streaming_reasoning.py +++ b/tests/parser/engine/test_gemma4_streaming_reasoning.py @@ -1616,3 +1616,36 @@ class TestCommaInStringValueRegression: assert result.tools_called is True args = json.loads(result.tool_calls[0].function.arguments) assert args["destination"] == "456 Oakwood Avenue, Rivermist, 83214" + + +class TestGemma4IsReasoningEnd: + @staticmethod + def _parser(thinking: bool): + return Gemma4Parser( + _make_tokenizer(_PLAIN_ANSWER_TOKENS), + chat_template_kwargs={"enable_thinking": thinking}, + ) + + @pytest.mark.parametrize( + ("ids", "thinking", "ended"), + [ + ([CHANNEL_START_ID, 3000, CHANNEL_END_ID], True, True), + ([CHANNEL_START_ID, 3000, TOOL_CALL_START_ID], True, True), + ([CHANNEL_START_ID, 3000], True, False), + ([NEW_TURN_ID, 9100], True, False), + ([CHANNEL_END_ID, NEW_TURN_ID, 9100], True, False), + ([NEW_TURN_ID, 9100], False, True), + # The bundled template's thinking-off prompt: closed channel. + ([NEW_TURN_ID, 9100, CHANNEL_START_ID, 3000, CHANNEL_END_ID], False, True), + # A marker still wins with thinking off: the grammar honours it. + ([CHANNEL_START_ID, 3000], False, False), + # A tool-call opener after the channel closed: still ended. + ( + [CHANNEL_START_ID, 3000, CHANNEL_END_ID, TOOL_CALL_START_ID, 9200], + True, + True, + ), + ], + ) + def test_is_reasoning_end(self, ids, thinking, ended): + assert self._parser(thinking).is_reasoning_end(ids) is ended diff --git a/tests/parser/engine/test_qwen3_reasoning.py b/tests/parser/engine/test_qwen3_reasoning.py index 37e6b5d130..3f285e4d03 100644 --- a/tests/parser/engine/test_qwen3_reasoning.py +++ b/tests/parser/engine/test_qwen3_reasoning.py @@ -180,9 +180,8 @@ class TestIsReasoningEnd: """Tool examples before the generation must not end reasoning.""" assert not parser.is_reasoning_end([_TOOL_CALL_ID, _TEXT_ID, _THINK_START_ID]) - def test_paired_tool_call_not_end(self, parser): - """Paired ... (from template) is NOT end.""" - assert not parser.is_reasoning_end( + def test_completed_tool_call_after_open_think_is_end(self, parser): + assert parser.is_reasoning_end( [_THINK_START_ID, 1, _TOOL_CALL_ID, 2, _TOOL_CALL_END_ID] ) @@ -196,6 +195,43 @@ class TestIsReasoningEnd: assert not parser.is_reasoning_end([]) +class TestIsReasoningEndDerivedFromGrammar: + @pytest.mark.parametrize( + ("ids", "thinking", "ended"), + [ + # Explicit close, and the implicit close on . + ([_THINK_START_ID, 1, _THINK_END_ID], True, True), + ([_THINK_START_ID, 1, _TOOL_CALL_ID], True, True), + # Still inside the block. + ([_THINK_START_ID, 1], True, False), + ([_THINK_END_ID, _THINK_START_ID, 1], True, False), + # Uncommitted: a bare turn, an empty sequence. Thinking on starts + # in REASONING, so reasoning is expected. + ([_IM_END_ID, _IM_START_ID, 1], True, False), + ([], True, False), + # Thinking off starts in CONTENT with no way into REASONING, so + # a the template left behind is inert, and a silent tail + # is ended because no block is expected. + ([_THINK_START_ID, 1], False, True), + ([_THINK_END_ID, _THINK_START_ID, 1], False, True), + ([_IM_END_ID, _IM_START_ID, 1], False, True), + ([], False, True), + ], + ) + def test_is_reasoning_end(self, mock_tokenizer, ids, thinking, ended): + parser = Qwen3Parser( + mock_tokenizer, chat_template_kwargs={"enable_thinking": thinking} + ) + assert parser.is_reasoning_end(ids) is ended + + @pytest.mark.parametrize("thinking", [True, False]) + def test_wait_for_reasoning_follows_thinking(self, mock_tokenizer, thinking): + parser = Qwen3Parser( + mock_tokenizer, chat_template_kwargs={"enable_thinking": thinking} + ) + assert parser.parser_engine_config.wait_for_reasoning is thinking + + class TestIsReasoningEndTurnBoundaries: """Reasoning markers from earlier turns must not mark the new turn's reasoning as finished (think-marker injection / interleaved-thinking diff --git a/vllm/parser/deepseek_v32.py b/vllm/parser/deepseek_v32.py index f76c954a93..de6634cf73 100644 --- a/vllm/parser/deepseek_v32.py +++ b/vllm/parser/deepseek_v32.py @@ -53,6 +53,7 @@ def deepseek_v32_config() -> ParserEngineConfig: return ParserEngineConfig( name="deepseek_v32", initial_state=ParserState.CONTENT, + wait_for_reasoning=False, terminals={ "TOOL_START": DSML_FUNC_START, "TOOL_END": DSML_FUNC_END, diff --git a/vllm/parser/deepseek_v4.py b/vllm/parser/deepseek_v4.py index 13eb83e6b8..79ddd93f6b 100644 --- a/vllm/parser/deepseek_v4.py +++ b/vllm/parser/deepseek_v4.py @@ -140,6 +140,7 @@ def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig: return ParserEngineConfig( name="deepseek_v4", initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, + wait_for_reasoning=thinking, terminals={ "THINK_START": DSML_THINK_START, "THINK_END": DSML_THINK_END, diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 4817dedb46..f873e36342 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -648,24 +648,28 @@ class ParserEngine(Parser): return None def is_reasoning_end(self, input_ids: list[int]) -> bool: - end_id = self._reasoning_end_token_id + config = self.parser_engine_config + wait_for_reasoning = config.wait_for_reasoning + if wait_for_reasoning is None: + wait_for_reasoning = config.initial_state is ParserState.REASONING + start_transition = config.transitions.get((config.initial_state, "THINK_START")) + start_opens_reasoning = ( + start_transition is not None + and start_transition.next_state is ParserState.REASONING + ) + end_ids = self._reasoning_end_token_ids start_id = self._reasoning_start_token_id - if end_id is not None: - if not input_ids: - return self.parser_engine_config.initial_state != ParserState.REASONING - boundary_ids = self._turn_boundary_token_ids - for i in range(len(input_ids) - 1, -1, -1): - token_id = input_ids[i] - if token_id == end_id: - return True - if start_id is not None and token_id == start_id: + boundary_ids = self._turn_boundary_token_ids + for token_id in reversed(input_ids): + if token_id in end_ids: + return True + if token_id == start_id: + if start_opens_reasoning: return False - if token_id in boundary_ids: - return ( - self.parser_engine_config.initial_state != ParserState.REASONING - ) - return False - return self._reasoning_ended + break + if token_id in boundary_ids: + break + return not wait_for_reasoning def extract_content_ids(self, input_ids: list[int]) -> list[int]: end_id = self._reasoning_end_token_id diff --git a/vllm/parser/engine/parser_engine_config.py b/vllm/parser/engine/parser_engine_config.py index 5c5cd2b467..b8d7dc6e46 100644 --- a/vllm/parser/engine/parser_engine_config.py +++ b/vllm/parser/engine/parser_engine_config.py @@ -74,6 +74,8 @@ class ParserEngineConfig: initial_state: ParserState = ParserState.CONTENT + wait_for_reasoning: bool | None = None + arg_converter: Callable[[str, bool], str] | None = None stream_arg_deltas: bool = True diff --git a/vllm/parser/gemma4.py b/vllm/parser/gemma4.py index 51c4696636..e3baaf488a 100644 --- a/vllm/parser/gemma4.py +++ b/vllm/parser/gemma4.py @@ -293,10 +293,14 @@ def _gemma4_arg_converter(raw_args: str, partial: bool) -> str: @functools.cache -def gemma4_config() -> ParserEngineConfig: +def gemma4_config(thinking: bool = False) -> ParserEngineConfig: return ParserEngineConfig( name="gemma4", initial_state=ParserState.CONTENT, + # Decides a silent tail: the E-series HF templates write no marker + # with thinking off, and no template does after a tool response. + wait_for_reasoning=thinking, + turn_boundary_tokens=frozenset({"<|turn>", "<|tool_response>"}), terminals={ "THINK_START": CHANNEL_START, "THINK_END": CHANNEL_END, @@ -417,11 +421,11 @@ class Gemma4Parser(ParserEngine): **kwargs, ) -> None: chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - self._thinking_enabled = chat_kwargs.get("enable_thinking", False) + thinking = chat_kwargs.get("enable_thinking", False) super().__init__( tokenizer, tools, - parser_engine_config=gemma4_config(), + parser_engine_config=gemma4_config(thinking=thinking), **kwargs, ) vocab = self.vocab @@ -472,30 +476,6 @@ class Gemma4Parser(ParserEngine): return delta_text, delta_token_ids - def is_reasoning_end(self, input_ids: list[int]) -> bool: - end_id = self._reasoning_end_token_id - start_id = self._reasoning_start_token_id - tool_call_id = self._tool_call_token_id - new_turn_id = self._new_turn_token_id - tool_response_id = self._tool_response_token_id - - if end_id is not None and not input_ids: - return self.parser_engine_config.initial_state != ParserState.REASONING - - for i in range(len(input_ids) - 1, -1, -1): - tid = input_ids[i] - if start_id is not None and tid == start_id: - return False - if tool_call_id is not None and tid == tool_call_id: - return True - if new_turn_id is not None and tid == new_turn_id: - return not self._thinking_enabled - if tool_response_id is not None and tid == tool_response_id: - return not self._thinking_enabled - if end_id is not None and tid == end_id: - return True - return True - def _prompt_ends_in_open_reasoning(self, prompt_token_ids: Sequence[int]) -> bool: """Whether the prompt tail is inside an open ``<|channel>`` block. diff --git a/vllm/parser/glm47_moe.py b/vllm/parser/glm47_moe.py index 8aa4feef25..4a804a4c9c 100644 --- a/vllm/parser/glm47_moe.py +++ b/vllm/parser/glm47_moe.py @@ -123,6 +123,7 @@ def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig: return ParserEngineConfig( name="glm47_moe", initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, + wait_for_reasoning=thinking, terminals={ **reasoning_terminals, "TOOL_START": TOOL_CALL_START, @@ -206,11 +207,6 @@ class Glm47MoeParser(ParserEngine): self._tool_slots[idx].name = self._tool_slots[idx].name.strip() super()._handle_tool_end(event, deltas) - def is_reasoning_end(self, input_ids: list[int]) -> bool: - if not self.thinking_enabled: - return True - return super().is_reasoning_end(input_ids) - def extract_content_ids(self, input_ids: list[int]) -> list[int]: if not self.thinking_enabled: return input_ids diff --git a/vllm/parser/inkling.py b/vllm/parser/inkling.py index 16883b58aa..2081fb2784 100644 --- a/vllm/parser/inkling.py +++ b/vllm/parser/inkling.py @@ -282,6 +282,7 @@ def inkling_config() -> ParserEngineConfig: # `<|message_model|>`. Non-streaming parsing receives only the generated # suffix, so begin in the corresponding message-header state as well. initial_state=ParserState.MESSAGE_HEADER, + wait_for_reasoning=True, terminals=terminals, # Inkling content-kind markers are the grammar. When the engine is # used through DelegatingParser, the reasoning pass can hand the tool diff --git a/vllm/parser/kimi_k2.py b/vllm/parser/kimi_k2.py index 1e1067a8c2..7c5f30aafe 100644 --- a/vllm/parser/kimi_k2.py +++ b/vllm/parser/kimi_k2.py @@ -81,6 +81,7 @@ def kimi_k2_config(thinking: bool = True) -> ParserEngineConfig: return ParserEngineConfig( name="kimi_k2", initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, + wait_for_reasoning=thinking, terminals={ **reasoning_terminals, "TOOL_SECTION_START": TOOL_SECTION_START, @@ -236,24 +237,6 @@ class KimiK2Parser(ParserEngine): def _extract_args_json(self, raw_args: str, func_name: str) -> str: return raw_args.strip() or "{}" - def is_reasoning_end(self, input_ids: list[int]) -> bool: - if not self.thinking_enabled: - return True - - start_id = self._start_token_id - end_id = self._end_token_id - tool_section_id = self._tool_section_start_token_id - - for i in range(len(input_ids) - 1, -1, -1): - token_id = input_ids[i] - if start_id is not None and token_id == start_id: - return False - if end_id is not None and token_id == end_id: - return True - if tool_section_id is not None and token_id == tool_section_id: - return True - return False - def extract_content_ids(self, input_ids: list[int]) -> list[int]: if not self.thinking_enabled: return input_ids diff --git a/vllm/parser/minimax_m2.py b/vllm/parser/minimax_m2.py index ab1b7eb114..c96f852b9d 100644 --- a/vllm/parser/minimax_m2.py +++ b/vllm/parser/minimax_m2.py @@ -95,6 +95,7 @@ def minimax_m2_config() -> ParserEngineConfig: return ParserEngineConfig( name="minimax_m2", initial_state=ParserState.REASONING, + wait_for_reasoning=True, terminals={ "THINK_START": THINK_START, "THINK_END": THINK_END, diff --git a/vllm/parser/mistral.py b/vllm/parser/mistral.py index fc9b8fbefe..ce516eea14 100644 --- a/vllm/parser/mistral.py +++ b/vllm/parser/mistral.py @@ -193,6 +193,7 @@ def mistral_config( return ParserEngineConfig( name=name, initial_state=ParserState.CONTENT, + wait_for_reasoning=reasoning_encoding != "none", terminals={ **reasoning_terminals, "TOOL_CALLS": _TOOL_CALLS, @@ -891,24 +892,6 @@ class MistralParser(ParserEngine): return (delta_to_be_parsed, "") - def is_reasoning_end(self, input_ids: list[int]) -> bool: - if self._reasoning_encoding == "none": - return True - if super().is_reasoning_end(input_ids): - return True - # [TOOL_CALLS] acts as an implicit reasoning-end marker - if self.bot_token_id is not None: - reasoning_start_id = self._reasoning_start_token_id - for i in range(len(input_ids) - 1, -1, -1): - if ( - reasoning_start_id is not None - and input_ids[i] == reasoning_start_id - ): - return False - if input_ids[i] == self.bot_token_id: - return True - return False - def extract_reasoning( self, model_output: str, diff --git a/vllm/parser/qwen3.py b/vllm/parser/qwen3.py index 8edf42e263..a11551a209 100644 --- a/vllm/parser/qwen3.py +++ b/vllm/parser/qwen3.py @@ -100,6 +100,7 @@ def qwen3_config( return ParserEngineConfig( name=name, initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, + wait_for_reasoning=thinking, turn_boundary_tokens=turn_boundary_tokens, terminals={ # Reasoning terminals @@ -205,8 +206,8 @@ class Qwen3Parser(ParserEngine): """Qwen3 parser: ````/```` reasoning + ```` XML tool calls in a single engine. - - ```` as implicit reasoning end - - Unpaired ```` token ID detection for ``is_reasoning_end`` + - ```` as implicit reasoning end (a grammar transition, so it + also feeds ``is_reasoning_end`` and the structured-output gate) Subclasses that share the grammar but differ only in the four wrapper token strings (reasoning + tool-call) override the class attributes @@ -245,9 +246,6 @@ class Qwen3Parser(ParserEngine): tools, **kwargs, ) - vocab = self.vocab - self._tool_call_token_id: int | None = vocab.get(self.TOOL_START) - self._tool_call_end_token_id: int | None = vocab.get(self.TOOL_END) def extract_reasoning( self, @@ -257,28 +255,3 @@ class Qwen3Parser(ParserEngine): if not self.thinking_enabled: return None, model_output return super().extract_reasoning(model_output, request) - - def is_reasoning_end(self, input_ids: list[int]) -> bool: - if super().is_reasoning_end(input_ids): - return True - tool_call_id = self._tool_call_token_id - tool_call_end_id = self._tool_call_end_token_id - reasoning_start_id = self._reasoning_start_token_id - boundary_ids = self._turn_boundary_token_ids - if tool_call_id is not None: - for i in range(len(input_ids) - 1, -1, -1): - if ( - reasoning_start_id is not None - and input_ids[i] == reasoning_start_id - ): - return False - if input_ids[i] in boundary_ids: - return False - if input_ids[i] == tool_call_id: - if tool_call_end_id is not None and any( - input_ids[j] == tool_call_end_id - for j in range(i + 1, len(input_ids)) - ): - continue - return True - return False