diff --git a/tests/parser/engine/test_qwen3_reasoning.py b/tests/parser/engine/test_qwen3_reasoning.py index cac3e3b2a6..a9bec494bc 100644 --- a/tests/parser/engine/test_qwen3_reasoning.py +++ b/tests/parser/engine/test_qwen3_reasoning.py @@ -27,6 +27,8 @@ _THINK_START_ID = 50 _THINK_END_ID = 51 _TOOL_CALL_ID = 60 _TOOL_CALL_END_ID = 61 +_IM_START_ID = 70 +_IM_END_ID = 71 _TEXT_ID = 100 _QWEN3_VOCAB = { @@ -34,6 +36,8 @@ _QWEN3_VOCAB = { "": _THINK_END_ID, "": _TOOL_CALL_ID, "": _TOOL_CALL_END_ID, + "<|im_start|>": _IM_START_ID, + "<|im_end|>": _IM_END_ID, } @@ -192,6 +196,72 @@ class TestIsReasoningEnd: assert not parser.is_reasoning_end([]) +class TestIsReasoningEndTurnBoundaries: + """Reasoning markers from earlier turns must not mark the new turn's + reasoning as finished (think-marker injection / interleaved-thinking + replay). + """ + + def test_injected_think_end_in_history_not_end(self, parser): + assert not parser.is_reasoning_end( + [_IM_START_ID, _TEXT_ID, _THINK_END_ID, _IM_END_ID, _IM_START_ID] + ) + + def test_replayed_reasoning_block_in_history_not_end(self, parser): + assert not parser.is_reasoning_end( + [ + _IM_START_ID, + _THINK_START_ID, + _TEXT_ID, + _THINK_END_ID, + _TEXT_ID, + _IM_END_ID, + _IM_START_ID, + ] + ) + + def test_think_end_in_current_turn_is_end(self, parser): + assert parser.is_reasoning_end( + [_IM_START_ID, _TEXT_ID, _IM_END_ID, _IM_START_ID, _THINK_END_ID] + ) + + def test_unpaired_tool_call_in_history_not_end(self, parser): + assert not parser.is_reasoning_end( + [_TOOL_CALL_ID, _TEXT_ID, _IM_END_ID, _IM_START_ID] + ) + + def test_continued_turn_with_closed_thinking_is_end(self, parser): + assert parser.is_reasoning_end( + [_IM_START_ID, _THINK_START_ID, _TEXT_ID, _THINK_END_ID, _TEXT_ID] + ) + + def test_continued_turn_with_open_thinking_not_end(self, parser): + assert not parser.is_reasoning_end([_IM_START_ID, _THINK_START_ID, _TEXT_ID]) + + def test_empty_boundary_config_keeps_global_walk(self): + """Configs without turn_boundary_tokens keep the pre-existing + behavior: any in the sequence ends reasoning.""" + cfg = qwen3_config(turn_boundary_tokens=frozenset()) + parser = Qwen3Parser( + make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg + ) + assert parser.is_reasoning_end( + [_IM_START_ID, _TEXT_ID, _THINK_END_ID, _IM_END_ID, _IM_START_ID] + ) + + def test_boundary_tokens_missing_from_vocab_keep_global_walk(self): + """Boundary tokens absent from the vocabulary resolve to nothing, + so subclasses reusing the default on a non-ChatML vocab keep the + pre-existing behavior.""" + vocab = { + token: token_id + for token, token_id in _QWEN3_VOCAB.items() + if not token.startswith("<|im_") + } + parser = Qwen3Parser(make_mock_tokenizer(vocab)) + assert parser.is_reasoning_end([_TEXT_ID, _THINK_END_ID, _TEXT_ID]) + + class TestDelegatingPromptDetection: def test_prompt_tool_example_does_not_skip_streaming_reasoning( self, mock_tokenizer, mock_request @@ -211,6 +281,56 @@ class TestDelegatingPromptDetection: assert delta.reasoning == "thinking" assert delta.content is None + def test_injected_think_end_in_history_keeps_streaming_reasoning( + self, mock_tokenizer, mock_request + ): + parser = _Qwen3DelegatingParser(mock_tokenizer) + prompt_ids = [ + _IM_START_ID, + _TEXT_ID, + _THINK_END_ID, + _TEXT_ID, + _IM_END_ID, + _IM_START_ID, + ] + + delta = parser.parse_delta( + "thinking", + [_TEXT_ID], + mock_request, + prompt_token_ids=prompt_ids, + finished=False, + ) + + assert delta is not None + assert delta.reasoning == "thinking" + assert delta.content is None + + def test_think_end_in_generation_prompt_still_seeds_reasoning_ended( + self, mock_tokenizer, mock_request + ): + parser = _Qwen3DelegatingParser(mock_tokenizer) + prompt_ids = [ + _IM_START_ID, + _TEXT_ID, + _IM_END_ID, + _IM_START_ID, + _THINK_START_ID, + _THINK_END_ID, + ] + + delta = parser.parse_delta( + "answer", + [_TEXT_ID], + mock_request, + prompt_token_ids=prompt_ids, + finished=False, + ) + + assert delta is not None + assert delta.reasoning is None + assert delta.content == "answer" + class TestStreaming: def test_basic_streaming(self, parser): diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 5ed837d22d..545bec6500 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -147,6 +147,11 @@ class ParserEngine(Parser): self._reasoning_start_token_id = vocab.get(start_text) if end_text: self._reasoning_end_token_id = vocab.get(end_text) + self._turn_boundary_token_ids: frozenset[int] = frozenset( + token_id + for token in parser_engine_config.turn_boundary_tokens + if (token_id := vocab.get(token)) is not None + ) @property def reasoning_start_str(self) -> str | None: @@ -601,11 +606,17 @@ class ParserEngine(Parser): if end_id is not None: if not input_ids: return self.parser_engine_config.initial_state != ParserState.REASONING + boundary_ids = self._turn_boundary_token_ids for i in range(len(input_ids) - 1, -1, -1): - if input_ids[i] == end_id: + token_id = input_ids[i] + if token_id == end_id: return True - if start_id is not None and input_ids[i] == start_id: + if start_id is not None and token_id == start_id: return False + if token_id in boundary_ids: + return ( + self.parser_engine_config.initial_state != ParserState.REASONING + ) return False return self._reasoning_ended diff --git a/vllm/parser/engine/parser_engine_config.py b/vllm/parser/engine/parser_engine_config.py index ad83e33149..4799604dab 100644 --- a/vllm/parser/engine/parser_engine_config.py +++ b/vllm/parser/engine/parser_engine_config.py @@ -84,6 +84,10 @@ class ParserEngineConfig: # Special tokens exempt from auto-drop but not state-machine terminals. preserve_tokens: frozenset[str] = field(default_factory=frozenset) + # Special tokens delimiting conversation turns in the prompt only + # considers reasoning markers after the last boundary token. + turn_boundary_tokens: frozenset[str] = field(default_factory=frozenset) + # Prevents trailing-whitespace accumulation across multi-turn conversations. strip_trailing_reasoning_whitespace: bool = True diff --git a/vllm/parser/nemotron_v3.py b/vllm/parser/nemotron_v3.py index 7884480fee..3cdd8b70d7 100644 --- a/vllm/parser/nemotron_v3.py +++ b/vllm/parser/nemotron_v3.py @@ -16,7 +16,7 @@ import dataclasses import functools from typing import TYPE_CHECKING -from vllm.parser.qwen3 import Qwen3Parser, qwen3_config +from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config if TYPE_CHECKING: from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -33,7 +33,7 @@ if TYPE_CHECKING: @functools.cache def nemotron_v3_config(thinking: bool = True) -> ParserEngineConfig: return dataclasses.replace( - qwen3_config(thinking=thinking), + qwen3_config(thinking=thinking, turn_boundary_tokens=CHATML_TURN_BOUNDARIES), name="nemotron_v3", strip_trailing_reasoning_whitespace=True, ) diff --git a/vllm/parser/qwen3.py b/vllm/parser/qwen3.py index 0041a46eaa..8edf42e263 100644 --- a/vllm/parser/qwen3.py +++ b/vllm/parser/qwen3.py @@ -42,6 +42,7 @@ THINK_START = "" THINK_END = "" TOOL_CALL_START = "" TOOL_CALL_END = "" +CHATML_TURN_BOUNDARIES = frozenset(("<|im_start|>", "<|im_end|>")) FUNC_PREFIX = " ParserEngineConfig: return ParserEngineConfig( name=name, initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, + turn_boundary_tokens=turn_boundary_tokens, terminals={ # Reasoning terminals "THINK_START": think_start, @@ -215,6 +218,7 @@ class Qwen3Parser(ParserEngine): THINK_END = THINK_END TOOL_START = TOOL_CALL_START TOOL_END = TOOL_CALL_END + TURN_BOUNDARIES: frozenset[str] = CHATML_TURN_BOUNDARIES def __init__( self, @@ -233,6 +237,7 @@ class Qwen3Parser(ParserEngine): think_end=self.THINK_END, tool_start=self.TOOL_START, tool_end=self.TOOL_END, + turn_boundary_tokens=self.TURN_BOUNDARIES, ), ) super().__init__( @@ -259,6 +264,7 @@ class Qwen3Parser(ParserEngine): tool_call_id = self._tool_call_token_id tool_call_end_id = self._tool_call_end_token_id reasoning_start_id = self._reasoning_start_token_id + boundary_ids = self._turn_boundary_token_ids if tool_call_id is not None: for i in range(len(input_ids) - 1, -1, -1): if ( @@ -266,6 +272,8 @@ class Qwen3Parser(ParserEngine): and input_ids[i] == reasoning_start_id ): return False + if input_ids[i] in boundary_ids: + return False if input_ids[i] == tool_call_id: if tool_call_end_id is not None and any( input_ids[j] == tool_call_end_id