[Bugfix][Parser] Honor a prompt-closed reasoning block in non-streaming parse and reasoning token counts (#61857)

Signed-off-by: Alberto Perdomo <aperdomo@redhat.com>
This commit is contained in:
alberto
2026-10-16 12:54:42 -04:00
committed by GitHub
parent cffc0cfa31
commit ae8f41238c
6 changed files with 253 additions and 2 deletions
@@ -380,6 +380,40 @@ class TestDelegatingPromptDetection:
assert delta.reasoning is None
assert delta.content == "answer"
def test_prior_turn_think_end_keeps_non_streaming_reasoning(
self, mock_tokenizer, mock_request
):
parser = _Qwen3DelegatingParser(mock_tokenizer)
parser.set_prompt_token_ids([_THINK_END_ID, _IM_END_ID, _IM_START_ID, _TEXT_ID])
output_ids = [ord("a"), _THINK_END_ID, ord("b")]
reasoning, content, _ = parser.parse(
mock_tokenizer.decode(output_ids),
mock_request,
model_output_token_ids=output_ids,
)
assert reasoning == "a"
assert content == "b"
assert parser.count_reasoning_tokens(output_ids) > 0
def test_think_end_in_current_turn_parses_output_as_content(
self, mock_tokenizer, mock_request
):
parser = _Qwen3DelegatingParser(mock_tokenizer)
parser.set_prompt_token_ids([_IM_START_ID, _THINK_START_ID, _THINK_END_ID])
output_ids = [ord("a"), ord("b")]
reasoning, content, _ = parser.parse(
mock_tokenizer.decode(output_ids),
mock_request,
model_output_token_ids=output_ids,
)
assert reasoning is None
assert content == "ab"
assert parser.count_reasoning_tokens(output_ids) == 0
class TestStreaming:
def test_basic_streaming(self, parser):
+179
View File
@@ -9,6 +9,9 @@ import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.abstract_parser import DelegatingParser
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.parser_manager import ParserManager
from vllm.parser.qwen3 import qwen3_config
from vllm.parser.utils import count_history_tool_calls
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
@@ -417,3 +420,179 @@ def test_parse_auto_tools_no_calls_returns_none(tokenizer):
assert reasoning is None
assert content == PLAIN_TEXT
assert tool_calls is None
# --------------------------------------------------------------------------
# Prompt that already closes the reasoning block
# --------------------------------------------------------------------------
CONTINUE_MESSAGES = [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "Sure,"},
]
REASONED_OUTPUT = "<think>\nlet me think\n</think>\n\n" + PLAIN_TEXT
TOOL_TURN_MESSAGES = [
{"role": "user", "content": "Weather in Paris?"},
{
"role": "assistant",
"content": "<think>\nneed a tool\n</think>\n\n",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
]
def reasoning_parser(tokenizer, name="qwen3"):
parser_cls = ParserManager.get_parser(
reasoning_parser_name=name, tokenizer=tokenizer
)
return parser_cls(tokenizer, None)
def encode(tokenizer, text):
return tokenizer.encode(text, add_special_tokens=False)
def prompt_case(tokenizer, case):
"""(rendered prompt ids, request, model output) for each prompt shape."""
if case == "continue_final_message":
# The template renders an empty, closed think block before "Sure,".
prompt = tokenizer.apply_chat_template(
CONTINUE_MESSAGES,
tokenize=False,
add_generation_prompt=False,
continue_final_message=True,
)
request = make_request(
messages=CONTINUE_MESSAGES,
add_generation_prompt=False,
continue_final_message=True,
)
return encode(tokenizer, prompt), request, " here is the answer"
if case == "template_closed_thinking":
# The template switches thinking off, but the parser is not told
# through chat_template_kwargs (#53284).
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": "hi"}],
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
)
return encode(tokenizer, prompt), make_request(), PLAIN_TEXT
if case == "earlier_turn_closed_reasoning":
# Only a previous assistant turn closes reasoning; this one is open.
prompt = tokenizer.apply_chat_template(
TOOL_TURN_MESSAGES,
tools=[{"type": "function", "function": {"name": "get_weather"}}],
tokenize=False,
add_generation_prompt=True,
)
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
assert case == "reasoning_open"
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": "hi"}],
tokenize=False,
add_generation_prompt=True,
)
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
@pytest.mark.parametrize("case", ["continue_final_message", "template_closed_thinking"])
def test_parse_prompt_closed_reasoning_is_content(tokenizer, case):
prompt_ids, request, output = prompt_case(tokenizer, case)
parser = reasoning_parser(tokenizer)
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=encode(tokenizer, output)
)
assert reasoning is None
assert content == output
assert parser.count_reasoning_tokens(encode(tokenizer, output)) == 0
@pytest.mark.parametrize(
("parser_name", "case"),
[
("qwen3", "reasoning_open"),
("qwen3", "earlier_turn_closed_reasoning"),
# BaseThinkingReasoningParser scans the prompt without turn
# boundaries, so it must not decide non-streaming classification.
("deepseek_r1", "earlier_turn_closed_reasoning"),
],
)
def test_parse_prompt_open_reasoning_still_extracted(tokenizer, parser_name, case):
prompt_ids, request, output = prompt_case(tokenizer, case)
parser = reasoning_parser(tokenizer, parser_name)
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=encode(tokenizer, output)
)
assert reasoning.strip() == "let me think"
assert content.strip() == PLAIN_TEXT
assert parser.count_reasoning_tokens(encode(tokenizer, output)) > 0
def test_parse_prompt_ignored_without_turn_boundaries(tokenizer):
"""Without turn boundaries, an earlier turn's </think> looks like this
turn's, so the engine must not start the output in content."""
prompt_ids, request, output = prompt_case(
tokenizer, "earlier_turn_closed_reasoning"
)
parser = ParserEngine(tokenizer, parser_engine_config=qwen3_config(thinking=True))
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=encode(tokenizer, output)
)
assert reasoning.strip() == "let me think"
assert content.strip() == PLAIN_TEXT
@pytest.mark.parametrize(
"case",
[
"continue_final_message",
"template_closed_thinking",
"reasoning_open",
"earlier_turn_closed_reasoning",
],
)
def test_prompt_reasoning_streaming_matches_non_streaming(tokenizer, case):
prompt_ids, request, output = prompt_case(tokenizer, case)
output_ids = encode(tokenizer, output)
parser = reasoning_parser(tokenizer)
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=output_ids
)
non_streaming = (reasoning or "", content or "")
non_streaming_count = parser.count_reasoning_tokens(output_ids)
parser = reasoning_parser(tokenizer)
streamed_reasoning = streamed_content = ""
for i, token_id in enumerate(output_ids):
delta = parser.parse_delta(
tokenizer.decode([token_id]),
[token_id],
request,
prompt_token_ids=prompt_ids,
finished=i == len(output_ids) - 1,
)
if delta is not None:
streamed_reasoning += delta.reasoning or ""
streamed_content += delta.content or ""
assert (streamed_reasoning, streamed_content) == non_streaming
assert parser.count_reasoning_tokens(output_ids) == non_streaming_count
+5
View File
@@ -769,6 +769,10 @@ class DelegatingParser(Parser):
return False
return self._reasoning_parser.is_reasoning_end(input_ids)
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
if self._reasoning_parser is not None:
self._reasoning_parser.set_prompt_token_ids(prompt_token_ids)
def _is_reasoning_end_streaming(
self, input_ids: list[int], delta_ids: list[int]
) -> bool:
@@ -858,6 +862,7 @@ class DelegatingParser(Parser):
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
state.prompt_reasoning_checked = True
self.set_prompt_token_ids(prompt_token_ids)
if self._reasoning_parser is None or self.is_reasoning_end(
prompt_token_ids
):
+8
View File
@@ -68,6 +68,11 @@ class ParserEngineReasoningAdapter(ReasoningParser):
def find_reasoning_end(self, token_ids: Sequence[int]) -> ReasoningEnd | None:
return self._parser_engine.find_reasoning_end(token_ids)
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
self._parser_engine.set_prompt_token_ids(prompt_token_ids)
if self._counting_parser_engine is not None:
self._counting_parser_engine.set_prompt_token_ids(prompt_token_ids)
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
self._parser_engine.adjust_initial_state_from_prompt(prompt_token_ids)
@@ -148,6 +153,9 @@ class ParserEngineReasoningAdapter(ReasoningParser):
self._counting_parser_engine = self._parser_engine_cls(
self.model_tokenizer, **self._parser_engine_kwargs
) # type: ignore[call-arg]
self._counting_parser_engine._prompt_closes_reasoning = (
self._parser_engine._prompt_closes_reasoning
)
self._counting_parser_engine._single_pass_parse(
self.model_tokenizer.decode(token_ids), token_ids
)
+18 -2
View File
@@ -147,6 +147,7 @@ class ParserEngine(Parser):
or parser_engine_config.initial_state == ParserState.REASONING
)
self._reasoning_ended: bool = not self._has_reasoning
self._prompt_closes_reasoning: bool = False
self._streaming_initialized: bool = False
self._prompt_streaming_prepared: bool = False
@@ -231,6 +232,18 @@ class ParserEngine(Parser):
self._streaming_initialized = True
self._reset(initial_state=initial_state)
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
"""Start parsing in content when the prompt already closes reasoning,
as with continue_final_message or a template that switches thinking
off. Only configs with turn boundaries can tell a closed block in
this turn from one in an earlier turn, so others are left as is.
"""
self._prompt_closes_reasoning = (
self.parser_engine_config.initial_state is ParserState.REASONING
and bool(self._turn_boundary_token_ids)
and self.is_reasoning_end(list(prompt_token_ids))
)
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
"""See :meth:`ReasoningParser.adjust_initial_state_from_prompt`."""
return
@@ -242,8 +255,11 @@ class ParserEngine(Parser):
return None
def _reset(self, initial_state: ParserState | None = None) -> None:
self._engine.reset(initial_state=initial_state)
self._reasoning_ended = not self._has_reasoning
prompt_closed = initial_state is None and self._prompt_closes_reasoning
self._engine.reset(
initial_state=ParserState.CONTENT if prompt_closed else initial_state
)
self._reasoning_ended = not self._has_reasoning or prompt_closed
self._tool_slots.clear()
self._deferred_content = ""
self._deferred_reasoning = ""
+9
View File
@@ -192,6 +192,15 @@ class ReasoningParser:
"""Adjust request parameters; override in subclasses as needed."""
return request
def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None:
"""Provide the rendered prompt before the output is parsed.
Parsers that can tell whether the prompt already closes reasoning
(e.g. continue_final_message) use it to classify the output.
Default is a no-op; override in subclasses as needed.
"""
return
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
"""Hook called once at the start of streaming with the prompt tokens.