[Bugfix][Parser] Honor a prompt-closed reasoning block in non-streaming parse and reasoning token counts (#61857)
Signed-off-by: Alberto Perdomo <aperdomo@redhat.com>
This commit is contained in:
@@ -380,6 +380,40 @@ class TestDelegatingPromptDetection:
|
||||
assert delta.reasoning is None
|
||||
assert delta.content == "answer"
|
||||
|
||||
def test_prior_turn_think_end_keeps_non_streaming_reasoning(
|
||||
self, mock_tokenizer, mock_request
|
||||
):
|
||||
parser = _Qwen3DelegatingParser(mock_tokenizer)
|
||||
parser.set_prompt_token_ids([_THINK_END_ID, _IM_END_ID, _IM_START_ID, _TEXT_ID])
|
||||
output_ids = [ord("a"), _THINK_END_ID, ord("b")]
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
mock_tokenizer.decode(output_ids),
|
||||
mock_request,
|
||||
model_output_token_ids=output_ids,
|
||||
)
|
||||
|
||||
assert reasoning == "a"
|
||||
assert content == "b"
|
||||
assert parser.count_reasoning_tokens(output_ids) > 0
|
||||
|
||||
def test_think_end_in_current_turn_parses_output_as_content(
|
||||
self, mock_tokenizer, mock_request
|
||||
):
|
||||
parser = _Qwen3DelegatingParser(mock_tokenizer)
|
||||
parser.set_prompt_token_ids([_IM_START_ID, _THINK_START_ID, _THINK_END_ID])
|
||||
output_ids = [ord("a"), ord("b")]
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
mock_tokenizer.decode(output_ids),
|
||||
mock_request,
|
||||
model_output_token_ids=output_ids,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "ab"
|
||||
assert parser.count_reasoning_tokens(output_ids) == 0
|
||||
|
||||
|
||||
class TestStreaming:
|
||||
def test_basic_streaming(self, parser):
|
||||
|
||||
@@ -9,6 +9,9 @@ import pytest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.parser.qwen3 import qwen3_config
|
||||
from vllm.parser.utils import count_history_tool_calls
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
||||
@@ -417,3 +420,179 @@ def test_parse_auto_tools_no_calls_returns_none(tokenizer):
|
||||
assert reasoning is None
|
||||
assert content == PLAIN_TEXT
|
||||
assert tool_calls is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Prompt that already closes the reasoning block
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
CONTINUE_MESSAGES = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "Sure,"},
|
||||
]
|
||||
REASONED_OUTPUT = "<think>\nlet me think\n</think>\n\n" + PLAIN_TEXT
|
||||
TOOL_TURN_MESSAGES = [
|
||||
{"role": "user", "content": "Weather in Paris?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "<think>\nneed a tool\n</think>\n\n",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
|
||||
]
|
||||
|
||||
|
||||
def reasoning_parser(tokenizer, name="qwen3"):
|
||||
parser_cls = ParserManager.get_parser(
|
||||
reasoning_parser_name=name, tokenizer=tokenizer
|
||||
)
|
||||
return parser_cls(tokenizer, None)
|
||||
|
||||
|
||||
def encode(tokenizer, text):
|
||||
return tokenizer.encode(text, add_special_tokens=False)
|
||||
|
||||
|
||||
def prompt_case(tokenizer, case):
|
||||
"""(rendered prompt ids, request, model output) for each prompt shape."""
|
||||
if case == "continue_final_message":
|
||||
# The template renders an empty, closed think block before "Sure,".
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
CONTINUE_MESSAGES,
|
||||
tokenize=False,
|
||||
add_generation_prompt=False,
|
||||
continue_final_message=True,
|
||||
)
|
||||
request = make_request(
|
||||
messages=CONTINUE_MESSAGES,
|
||||
add_generation_prompt=False,
|
||||
continue_final_message=True,
|
||||
)
|
||||
return encode(tokenizer, prompt), request, " here is the answer"
|
||||
if case == "template_closed_thinking":
|
||||
# The template switches thinking off, but the parser is not told
|
||||
# through chat_template_kwargs (#53284).
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
)
|
||||
return encode(tokenizer, prompt), make_request(), PLAIN_TEXT
|
||||
if case == "earlier_turn_closed_reasoning":
|
||||
# Only a previous assistant turn closes reasoning; this one is open.
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
TOOL_TURN_MESSAGES,
|
||||
tools=[{"type": "function", "function": {"name": "get_weather"}}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
|
||||
assert case == "reasoning_open"
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["continue_final_message", "template_closed_thinking"])
|
||||
def test_parse_prompt_closed_reasoning_is_content(tokenizer, case):
|
||||
prompt_ids, request, output = prompt_case(tokenizer, case)
|
||||
parser = reasoning_parser(tokenizer)
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=encode(tokenizer, output)
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == output
|
||||
assert parser.count_reasoning_tokens(encode(tokenizer, output)) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("parser_name", "case"),
|
||||
[
|
||||
("qwen3", "reasoning_open"),
|
||||
("qwen3", "earlier_turn_closed_reasoning"),
|
||||
# BaseThinkingReasoningParser scans the prompt without turn
|
||||
# boundaries, so it must not decide non-streaming classification.
|
||||
("deepseek_r1", "earlier_turn_closed_reasoning"),
|
||||
],
|
||||
)
|
||||
def test_parse_prompt_open_reasoning_still_extracted(tokenizer, parser_name, case):
|
||||
prompt_ids, request, output = prompt_case(tokenizer, case)
|
||||
parser = reasoning_parser(tokenizer, parser_name)
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=encode(tokenizer, output)
|
||||
)
|
||||
|
||||
assert reasoning.strip() == "let me think"
|
||||
assert content.strip() == PLAIN_TEXT
|
||||
assert parser.count_reasoning_tokens(encode(tokenizer, output)) > 0
|
||||
|
||||
|
||||
def test_parse_prompt_ignored_without_turn_boundaries(tokenizer):
|
||||
"""Without turn boundaries, an earlier turn's </think> looks like this
|
||||
turn's, so the engine must not start the output in content."""
|
||||
prompt_ids, request, output = prompt_case(
|
||||
tokenizer, "earlier_turn_closed_reasoning"
|
||||
)
|
||||
parser = ParserEngine(tokenizer, parser_engine_config=qwen3_config(thinking=True))
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=encode(tokenizer, output)
|
||||
)
|
||||
|
||||
assert reasoning.strip() == "let me think"
|
||||
assert content.strip() == PLAIN_TEXT
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
"continue_final_message",
|
||||
"template_closed_thinking",
|
||||
"reasoning_open",
|
||||
"earlier_turn_closed_reasoning",
|
||||
],
|
||||
)
|
||||
def test_prompt_reasoning_streaming_matches_non_streaming(tokenizer, case):
|
||||
prompt_ids, request, output = prompt_case(tokenizer, case)
|
||||
output_ids = encode(tokenizer, output)
|
||||
|
||||
parser = reasoning_parser(tokenizer)
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=output_ids
|
||||
)
|
||||
non_streaming = (reasoning or "", content or "")
|
||||
non_streaming_count = parser.count_reasoning_tokens(output_ids)
|
||||
|
||||
parser = reasoning_parser(tokenizer)
|
||||
streamed_reasoning = streamed_content = ""
|
||||
for i, token_id in enumerate(output_ids):
|
||||
delta = parser.parse_delta(
|
||||
tokenizer.decode([token_id]),
|
||||
[token_id],
|
||||
request,
|
||||
prompt_token_ids=prompt_ids,
|
||||
finished=i == len(output_ids) - 1,
|
||||
)
|
||||
if delta is not None:
|
||||
streamed_reasoning += delta.reasoning or ""
|
||||
streamed_content += delta.content or ""
|
||||
|
||||
assert (streamed_reasoning, streamed_content) == non_streaming
|
||||
assert parser.count_reasoning_tokens(output_ids) == non_streaming_count
|
||||
|
||||
@@ -769,6 +769,10 @@ class DelegatingParser(Parser):
|
||||
return False
|
||||
return self._reasoning_parser.is_reasoning_end(input_ids)
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
if self._reasoning_parser is not None:
|
||||
self._reasoning_parser.set_prompt_token_ids(prompt_token_ids)
|
||||
|
||||
def _is_reasoning_end_streaming(
|
||||
self, input_ids: list[int], delta_ids: list[int]
|
||||
) -> bool:
|
||||
@@ -858,6 +862,7 @@ class DelegatingParser(Parser):
|
||||
|
||||
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
|
||||
state.prompt_reasoning_checked = True
|
||||
self.set_prompt_token_ids(prompt_token_ids)
|
||||
if self._reasoning_parser is None or self.is_reasoning_end(
|
||||
prompt_token_ids
|
||||
):
|
||||
|
||||
@@ -68,6 +68,11 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None:
|
||||
return self._parser_engine.find_reasoning_end(token_ids)
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
self._parser_engine.set_prompt_token_ids(prompt_token_ids)
|
||||
if self._counting_parser_engine is not None:
|
||||
self._counting_parser_engine.set_prompt_token_ids(prompt_token_ids)
|
||||
|
||||
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
self._parser_engine.adjust_initial_state_from_prompt(prompt_token_ids)
|
||||
|
||||
@@ -148,6 +153,9 @@ class ParserEngineReasoningAdapter(ReasoningParser):
|
||||
self._counting_parser_engine = self._parser_engine_cls(
|
||||
self.model_tokenizer, **self._parser_engine_kwargs
|
||||
) # type: ignore[call-arg]
|
||||
self._counting_parser_engine._prompt_closes_reasoning = (
|
||||
self._parser_engine._prompt_closes_reasoning
|
||||
)
|
||||
self._counting_parser_engine._single_pass_parse(
|
||||
self.model_tokenizer.decode(token_ids), token_ids
|
||||
)
|
||||
|
||||
@@ -147,6 +147,7 @@ class ParserEngine(Parser):
|
||||
or parser_engine_config.initial_state == ParserState.REASONING
|
||||
)
|
||||
self._reasoning_ended: bool = not self._has_reasoning
|
||||
self._prompt_closes_reasoning: bool = False
|
||||
self._streaming_initialized: bool = False
|
||||
self._prompt_streaming_prepared: bool = False
|
||||
|
||||
@@ -231,6 +232,18 @@ class ParserEngine(Parser):
|
||||
self._streaming_initialized = True
|
||||
self._reset(initial_state=initial_state)
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
"""Start parsing in content when the prompt already closes reasoning,
|
||||
as with continue_final_message or a template that switches thinking
|
||||
off. Only configs with turn boundaries can tell a closed block in
|
||||
this turn from one in an earlier turn, so others are left as is.
|
||||
"""
|
||||
self._prompt_closes_reasoning = (
|
||||
self.parser_engine_config.initial_state is ParserState.REASONING
|
||||
and bool(self._turn_boundary_token_ids)
|
||||
and self.is_reasoning_end(list(prompt_token_ids))
|
||||
)
|
||||
|
||||
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
"""See :meth:`ReasoningParser.adjust_initial_state_from_prompt`."""
|
||||
return
|
||||
@@ -242,8 +255,11 @@ class ParserEngine(Parser):
|
||||
return None
|
||||
|
||||
def _reset(self, initial_state: ParserState | None = None) -> None:
|
||||
self._engine.reset(initial_state=initial_state)
|
||||
self._reasoning_ended = not self._has_reasoning
|
||||
prompt_closed = initial_state is None and self._prompt_closes_reasoning
|
||||
self._engine.reset(
|
||||
initial_state=ParserState.CONTENT if prompt_closed else initial_state
|
||||
)
|
||||
self._reasoning_ended = not self._has_reasoning or prompt_closed
|
||||
self._tool_slots.clear()
|
||||
self._deferred_content = ""
|
||||
self._deferred_reasoning = ""
|
||||
|
||||
@@ -192,6 +192,15 @@ class ReasoningParser:
|
||||
"""Adjust request parameters; override in subclasses as needed."""
|
||||
return request
|
||||
|
||||
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
"""Provide the rendered prompt before the output is parsed.
|
||||
|
||||
Parsers that can tell whether the prompt already closes reasoning
|
||||
(e.g. continue_final_message) use it to classify the output.
|
||||
Default is a no-op; override in subclasses as needed.
|
||||
"""
|
||||
return
|
||||
|
||||
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
|
||||
"""Hook called once at the start of streaming with the prompt tokens.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user