[Bugfix][Parser] Scope reasoning-end detection to the current turn via turn-boundary tokens (#54089)
Signed-off-by: Mikhail Podvitskii <podvitskiymichael@gmail.com>
This commit is contained in:
@@ -27,6 +27,8 @@ _THINK_START_ID = 50
|
|||||||
_THINK_END_ID = 51
|
_THINK_END_ID = 51
|
||||||
_TOOL_CALL_ID = 60
|
_TOOL_CALL_ID = 60
|
||||||
_TOOL_CALL_END_ID = 61
|
_TOOL_CALL_END_ID = 61
|
||||||
|
_IM_START_ID = 70
|
||||||
|
_IM_END_ID = 71
|
||||||
_TEXT_ID = 100
|
_TEXT_ID = 100
|
||||||
|
|
||||||
_QWEN3_VOCAB = {
|
_QWEN3_VOCAB = {
|
||||||
@@ -34,6 +36,8 @@ _QWEN3_VOCAB = {
|
|||||||
"</think>": _THINK_END_ID,
|
"</think>": _THINK_END_ID,
|
||||||
"<tool_call>": _TOOL_CALL_ID,
|
"<tool_call>": _TOOL_CALL_ID,
|
||||||
"</tool_call>": _TOOL_CALL_END_ID,
|
"</tool_call>": _TOOL_CALL_END_ID,
|
||||||
|
"<|im_start|>": _IM_START_ID,
|
||||||
|
"<|im_end|>": _IM_END_ID,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -192,6 +196,72 @@ class TestIsReasoningEnd:
|
|||||||
assert not parser.is_reasoning_end([])
|
assert not parser.is_reasoning_end([])
|
||||||
|
|
||||||
|
|
||||||
|
class TestIsReasoningEndTurnBoundaries:
|
||||||
|
"""Reasoning markers from earlier turns must not mark the new turn's
|
||||||
|
reasoning as finished (think-marker injection / interleaved-thinking
|
||||||
|
replay).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_injected_think_end_in_history_not_end(self, parser):
|
||||||
|
assert not parser.is_reasoning_end(
|
||||||
|
[_IM_START_ID, _TEXT_ID, _THINK_END_ID, _IM_END_ID, _IM_START_ID]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_replayed_reasoning_block_in_history_not_end(self, parser):
|
||||||
|
assert not parser.is_reasoning_end(
|
||||||
|
[
|
||||||
|
_IM_START_ID,
|
||||||
|
_THINK_START_ID,
|
||||||
|
_TEXT_ID,
|
||||||
|
_THINK_END_ID,
|
||||||
|
_TEXT_ID,
|
||||||
|
_IM_END_ID,
|
||||||
|
_IM_START_ID,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_think_end_in_current_turn_is_end(self, parser):
|
||||||
|
assert parser.is_reasoning_end(
|
||||||
|
[_IM_START_ID, _TEXT_ID, _IM_END_ID, _IM_START_ID, _THINK_END_ID]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_unpaired_tool_call_in_history_not_end(self, parser):
|
||||||
|
assert not parser.is_reasoning_end(
|
||||||
|
[_TOOL_CALL_ID, _TEXT_ID, _IM_END_ID, _IM_START_ID]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_continued_turn_with_closed_thinking_is_end(self, parser):
|
||||||
|
assert parser.is_reasoning_end(
|
||||||
|
[_IM_START_ID, _THINK_START_ID, _TEXT_ID, _THINK_END_ID, _TEXT_ID]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_continued_turn_with_open_thinking_not_end(self, parser):
|
||||||
|
assert not parser.is_reasoning_end([_IM_START_ID, _THINK_START_ID, _TEXT_ID])
|
||||||
|
|
||||||
|
def test_empty_boundary_config_keeps_global_walk(self):
|
||||||
|
"""Configs without turn_boundary_tokens keep the pre-existing
|
||||||
|
behavior: any </think> in the sequence ends reasoning."""
|
||||||
|
cfg = qwen3_config(turn_boundary_tokens=frozenset())
|
||||||
|
parser = Qwen3Parser(
|
||||||
|
make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg
|
||||||
|
)
|
||||||
|
assert parser.is_reasoning_end(
|
||||||
|
[_IM_START_ID, _TEXT_ID, _THINK_END_ID, _IM_END_ID, _IM_START_ID]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_boundary_tokens_missing_from_vocab_keep_global_walk(self):
|
||||||
|
"""Boundary tokens absent from the vocabulary resolve to nothing,
|
||||||
|
so subclasses reusing the default on a non-ChatML vocab keep the
|
||||||
|
pre-existing behavior."""
|
||||||
|
vocab = {
|
||||||
|
token: token_id
|
||||||
|
for token, token_id in _QWEN3_VOCAB.items()
|
||||||
|
if not token.startswith("<|im_")
|
||||||
|
}
|
||||||
|
parser = Qwen3Parser(make_mock_tokenizer(vocab))
|
||||||
|
assert parser.is_reasoning_end([_TEXT_ID, _THINK_END_ID, _TEXT_ID])
|
||||||
|
|
||||||
|
|
||||||
class TestDelegatingPromptDetection:
|
class TestDelegatingPromptDetection:
|
||||||
def test_prompt_tool_example_does_not_skip_streaming_reasoning(
|
def test_prompt_tool_example_does_not_skip_streaming_reasoning(
|
||||||
self, mock_tokenizer, mock_request
|
self, mock_tokenizer, mock_request
|
||||||
@@ -211,6 +281,56 @@ class TestDelegatingPromptDetection:
|
|||||||
assert delta.reasoning == "thinking"
|
assert delta.reasoning == "thinking"
|
||||||
assert delta.content is None
|
assert delta.content is None
|
||||||
|
|
||||||
|
def test_injected_think_end_in_history_keeps_streaming_reasoning(
|
||||||
|
self, mock_tokenizer, mock_request
|
||||||
|
):
|
||||||
|
parser = _Qwen3DelegatingParser(mock_tokenizer)
|
||||||
|
prompt_ids = [
|
||||||
|
_IM_START_ID,
|
||||||
|
_TEXT_ID,
|
||||||
|
_THINK_END_ID,
|
||||||
|
_TEXT_ID,
|
||||||
|
_IM_END_ID,
|
||||||
|
_IM_START_ID,
|
||||||
|
]
|
||||||
|
|
||||||
|
delta = parser.parse_delta(
|
||||||
|
"thinking",
|
||||||
|
[_TEXT_ID],
|
||||||
|
mock_request,
|
||||||
|
prompt_token_ids=prompt_ids,
|
||||||
|
finished=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert delta is not None
|
||||||
|
assert delta.reasoning == "thinking"
|
||||||
|
assert delta.content is None
|
||||||
|
|
||||||
|
def test_think_end_in_generation_prompt_still_seeds_reasoning_ended(
|
||||||
|
self, mock_tokenizer, mock_request
|
||||||
|
):
|
||||||
|
parser = _Qwen3DelegatingParser(mock_tokenizer)
|
||||||
|
prompt_ids = [
|
||||||
|
_IM_START_ID,
|
||||||
|
_TEXT_ID,
|
||||||
|
_IM_END_ID,
|
||||||
|
_IM_START_ID,
|
||||||
|
_THINK_START_ID,
|
||||||
|
_THINK_END_ID,
|
||||||
|
]
|
||||||
|
|
||||||
|
delta = parser.parse_delta(
|
||||||
|
"answer",
|
||||||
|
[_TEXT_ID],
|
||||||
|
mock_request,
|
||||||
|
prompt_token_ids=prompt_ids,
|
||||||
|
finished=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert delta is not None
|
||||||
|
assert delta.reasoning is None
|
||||||
|
assert delta.content == "answer"
|
||||||
|
|
||||||
|
|
||||||
class TestStreaming:
|
class TestStreaming:
|
||||||
def test_basic_streaming(self, parser):
|
def test_basic_streaming(self, parser):
|
||||||
|
|||||||
@@ -147,6 +147,11 @@ class ParserEngine(Parser):
|
|||||||
self._reasoning_start_token_id = vocab.get(start_text)
|
self._reasoning_start_token_id = vocab.get(start_text)
|
||||||
if end_text:
|
if end_text:
|
||||||
self._reasoning_end_token_id = vocab.get(end_text)
|
self._reasoning_end_token_id = vocab.get(end_text)
|
||||||
|
self._turn_boundary_token_ids: frozenset[int] = frozenset(
|
||||||
|
token_id
|
||||||
|
for token in parser_engine_config.turn_boundary_tokens
|
||||||
|
if (token_id := vocab.get(token)) is not None
|
||||||
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def reasoning_start_str(self) -> str | None:
|
def reasoning_start_str(self) -> str | None:
|
||||||
@@ -601,11 +606,17 @@ class ParserEngine(Parser):
|
|||||||
if end_id is not None:
|
if end_id is not None:
|
||||||
if not input_ids:
|
if not input_ids:
|
||||||
return self.parser_engine_config.initial_state != ParserState.REASONING
|
return self.parser_engine_config.initial_state != ParserState.REASONING
|
||||||
|
boundary_ids = self._turn_boundary_token_ids
|
||||||
for i in range(len(input_ids) - 1, -1, -1):
|
for i in range(len(input_ids) - 1, -1, -1):
|
||||||
if input_ids[i] == end_id:
|
token_id = input_ids[i]
|
||||||
|
if token_id == end_id:
|
||||||
return True
|
return True
|
||||||
if start_id is not None and input_ids[i] == start_id:
|
if start_id is not None and token_id == start_id:
|
||||||
return False
|
return False
|
||||||
|
if token_id in boundary_ids:
|
||||||
|
return (
|
||||||
|
self.parser_engine_config.initial_state != ParserState.REASONING
|
||||||
|
)
|
||||||
return False
|
return False
|
||||||
return self._reasoning_ended
|
return self._reasoning_ended
|
||||||
|
|
||||||
|
|||||||
@@ -84,6 +84,10 @@ class ParserEngineConfig:
|
|||||||
# Special tokens exempt from auto-drop but not state-machine terminals.
|
# Special tokens exempt from auto-drop but not state-machine terminals.
|
||||||
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
|
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
|
||||||
|
|
||||||
|
# Special tokens delimiting conversation turns in the prompt only
|
||||||
|
# considers reasoning markers after the last boundary token.
|
||||||
|
turn_boundary_tokens: frozenset[str] = field(default_factory=frozenset)
|
||||||
|
|
||||||
# Prevents trailing-whitespace accumulation across multi-turn conversations.
|
# Prevents trailing-whitespace accumulation across multi-turn conversations.
|
||||||
strip_trailing_reasoning_whitespace: bool = True
|
strip_trailing_reasoning_whitespace: bool = True
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import dataclasses
|
|||||||
import functools
|
import functools
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
|
from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
@@ -33,7 +33,7 @@ if TYPE_CHECKING:
|
|||||||
@functools.cache
|
@functools.cache
|
||||||
def nemotron_v3_config(thinking: bool = True) -> ParserEngineConfig:
|
def nemotron_v3_config(thinking: bool = True) -> ParserEngineConfig:
|
||||||
return dataclasses.replace(
|
return dataclasses.replace(
|
||||||
qwen3_config(thinking=thinking),
|
qwen3_config(thinking=thinking, turn_boundary_tokens=CHATML_TURN_BOUNDARIES),
|
||||||
name="nemotron_v3",
|
name="nemotron_v3",
|
||||||
strip_trailing_reasoning_whitespace=True,
|
strip_trailing_reasoning_whitespace=True,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ THINK_START = "<think>"
|
|||||||
THINK_END = "</think>"
|
THINK_END = "</think>"
|
||||||
TOOL_CALL_START = "<tool_call>"
|
TOOL_CALL_START = "<tool_call>"
|
||||||
TOOL_CALL_END = "</tool_call>"
|
TOOL_CALL_END = "</tool_call>"
|
||||||
|
CHATML_TURN_BOUNDARIES = frozenset(("<|im_start|>", "<|im_end|>"))
|
||||||
FUNC_PREFIX = "<function="
|
FUNC_PREFIX = "<function="
|
||||||
FUNC_END = "</function>"
|
FUNC_END = "</function>"
|
||||||
PARAM_START = "<parameter="
|
PARAM_START = "<parameter="
|
||||||
@@ -94,10 +95,12 @@ def qwen3_config(
|
|||||||
think_end: str = THINK_END,
|
think_end: str = THINK_END,
|
||||||
tool_start: str = TOOL_CALL_START,
|
tool_start: str = TOOL_CALL_START,
|
||||||
tool_end: str = TOOL_CALL_END,
|
tool_end: str = TOOL_CALL_END,
|
||||||
|
turn_boundary_tokens: frozenset[str] = frozenset(),
|
||||||
) -> ParserEngineConfig:
|
) -> ParserEngineConfig:
|
||||||
return ParserEngineConfig(
|
return ParserEngineConfig(
|
||||||
name=name,
|
name=name,
|
||||||
initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
|
initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
|
||||||
|
turn_boundary_tokens=turn_boundary_tokens,
|
||||||
terminals={
|
terminals={
|
||||||
# Reasoning terminals
|
# Reasoning terminals
|
||||||
"THINK_START": think_start,
|
"THINK_START": think_start,
|
||||||
@@ -215,6 +218,7 @@ class Qwen3Parser(ParserEngine):
|
|||||||
THINK_END = THINK_END
|
THINK_END = THINK_END
|
||||||
TOOL_START = TOOL_CALL_START
|
TOOL_START = TOOL_CALL_START
|
||||||
TOOL_END = TOOL_CALL_END
|
TOOL_END = TOOL_CALL_END
|
||||||
|
TURN_BOUNDARIES: frozenset[str] = CHATML_TURN_BOUNDARIES
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -233,6 +237,7 @@ class Qwen3Parser(ParserEngine):
|
|||||||
think_end=self.THINK_END,
|
think_end=self.THINK_END,
|
||||||
tool_start=self.TOOL_START,
|
tool_start=self.TOOL_START,
|
||||||
tool_end=self.TOOL_END,
|
tool_end=self.TOOL_END,
|
||||||
|
turn_boundary_tokens=self.TURN_BOUNDARIES,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
super().__init__(
|
super().__init__(
|
||||||
@@ -259,6 +264,7 @@ class Qwen3Parser(ParserEngine):
|
|||||||
tool_call_id = self._tool_call_token_id
|
tool_call_id = self._tool_call_token_id
|
||||||
tool_call_end_id = self._tool_call_end_token_id
|
tool_call_end_id = self._tool_call_end_token_id
|
||||||
reasoning_start_id = self._reasoning_start_token_id
|
reasoning_start_id = self._reasoning_start_token_id
|
||||||
|
boundary_ids = self._turn_boundary_token_ids
|
||||||
if tool_call_id is not None:
|
if tool_call_id is not None:
|
||||||
for i in range(len(input_ids) - 1, -1, -1):
|
for i in range(len(input_ids) - 1, -1, -1):
|
||||||
if (
|
if (
|
||||||
@@ -266,6 +272,8 @@ class Qwen3Parser(ParserEngine):
|
|||||||
and input_ids[i] == reasoning_start_id
|
and input_ids[i] == reasoning_start_id
|
||||||
):
|
):
|
||||||
return False
|
return False
|
||||||
|
if input_ids[i] in boundary_ids:
|
||||||
|
return False
|
||||||
if input_ids[i] == tool_call_id:
|
if input_ids[i] == tool_call_id:
|
||||||
if tool_call_end_id is not None and any(
|
if tool_call_end_id is not None and any(
|
||||||
input_ids[j] == tool_call_end_id
|
input_ids[j] == tool_call_end_id
|
||||||
|
|||||||
Reference in New Issue
Block a user