Fix Qwen3 streaming content routing (#40820)

Signed-off-by: xy3 <120182408@qq.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
xy3
2026-05-06 17:22:01 -04:00
committed by GitHub
co-authored by sfeng33
parent deb737e323
commit 50acdc5b5c
4 changed files with 220 additions and 7 deletions
@@ -287,3 +287,47 @@ async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_
f"index {i}: reasoning decode token ids ({n_reason}) != "
f"thinking_token_budget ({expected_budget})"
)
@pytest.mark.asyncio
@pytest.mark.parametrize("client", ["default", "auto_config"], indirect=True)
async def test_streaming_with_thinking_disabled_stays_in_content(
client: openai.AsyncOpenAI,
):
request_kwargs = {
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": "Which is larger, 4 or 12?"
" Output exactly one token: 4 or 12.",
}
],
"max_tokens": 16,
"temperature": 0.0,
"extra_body": {"chat_template_kwargs": {"enable_thinking": False}},
}
response = await client.chat.completions.create(**request_kwargs)
message = response.choices[0].message
assert message.content is not None and message.content.strip() != ""
assert getattr(message, "reasoning", None) in (None, "")
stream = await client.chat.completions.create(
**request_kwargs,
stream=True,
)
content_chunks = []
reasoning_chunks = []
async for chunk in stream:
if not chunk.choices:
continue
delta = chunk.choices[0].delta
if getattr(delta, "content", None):
content_chunks.append(delta.content)
if getattr(delta, "reasoning", None):
reasoning_chunks.append(delta.reasoning)
assert "".join(content_chunks).strip() != ""
assert reasoning_chunks == []
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+165
View File
@@ -0,0 +1,165 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
from vllm.parser.abstract_parser import _WrappedParser
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
class ThinkReasoningParser(BaseThinkingReasoningParser):
@property
def start_token(self) -> str:
return "<think>"
@property
def end_token(self) -> str:
return "</think>"
MODEL_OUTPUT = (
"<think>let me think about this</think>"
'<tool_call>\n{"name": "get_weather", '
'"arguments": {"city": "Dallas"}}\n</tool_call>'
)
@pytest.fixture(scope="module")
def tokenizer():
from vllm.tokenizers import get_tokenizer
return get_tokenizer("Qwen/Qwen3-32B")
@pytest.fixture
def request_obj():
return ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
)
def make_parser(tokenizer, reasoning=False, tool=False):
_WrappedParser.reasoning_parser_cls = ThinkReasoningParser if reasoning else None
_WrappedParser.tool_parser_cls = Hermes2ProToolParser if tool else None
return _WrappedParser(tokenizer)
def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
token_ids = tokenizer.encode(text, add_special_tokens=False)
results: list[DeltaMessage | None] = []
for tid in token_ids:
delta_text = tokenizer.decode([tid])
result = parser.parse_delta(
delta_text, [tid], request, prompt_token_ids=prompt_token_ids
)
prompt_token_ids = None
results.append(result)
return results
def collect_fields(results):
all_reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
all_content = "".join(r.content for r in results if r and r.content)
all_tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls]
return all_reasoning, all_content, all_tool_calls
def test_parse_delta_neither_parser(tokenizer, request_obj):
parser = make_parser(tokenizer, reasoning=False, tool=False)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert reasoning == ""
assert len(tool_calls) == 0
assert "<think>" in content
assert "let me think about this" in content
assert "<tool_call>" in content
assert "get_weather" in content
def test_parse_delta_tool_parser_only(tokenizer, request_obj):
parser = make_parser(tokenizer, reasoning=False, tool=True)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert reasoning == ""
assert "<think>" in content
assert "let me think about this" in content
assert "</think>" in content
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
def test_parse_delta_reasoning_parser_only(tokenizer, request_obj):
parser = make_parser(tokenizer, reasoning=True, tool=False)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert len(tool_calls) == 0
assert "<tool_call>" in content
assert "get_weather" in content
assert "</tool_call>" in content
def test_parse_delta_both_parsers(tokenizer, request_obj):
parser = make_parser(tokenizer, reasoning=True, tool=True)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert content == ""
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
"""Regression test for vllm-project/vllm#40466.
When enable_thinking=False, the chat template places <think>\\n\\n</think>
in the prompt. The model then generates pure content (no think tokens).
All streaming output must go to delta.content, not delta.reasoning.
"""
parser = make_parser(tokenizer, reasoning=True, tool=False)
end_token_id = parser._reasoning_parser.end_token_id
prompt_token_ids = [1, 2, end_token_id, 3]
content_text = "Hello! How can I assist you today?"
results = stream_text(
parser,
tokenizer,
content_text,
request_obj,
prompt_token_ids=prompt_token_ids,
)
reasoning, content, tool_calls = collect_fields(results)
assert reasoning == "", f"Expected no reasoning, got: {reasoning!r}"
assert "Hello" in content
assert "assist" in content
assert len(tool_calls) == 0
+9 -7
View File
@@ -636,15 +636,11 @@ class DelegatingParser(Parser):
def _in_reasoning_phase(self, state: StreamState) -> bool:
if self._reasoning_parser is None:
return False
if self._tool_parser is None:
return True
return not state.reasoning_ended
def _in_tool_call_phase(self, state: StreamState) -> bool:
if self._tool_parser is None:
return False
if self._reasoning_parser is None:
return True
return state.reasoning_ended
def parse_delta(
@@ -658,7 +654,9 @@ class DelegatingParser(Parser):
if not state.prompt_reasoning_checked and prompt_token_ids is not None:
state.prompt_reasoning_checked = True
if self.is_reasoning_end(prompt_token_ids):
if self._reasoning_parser is None or self.is_reasoning_end(
prompt_token_ids
):
state.reasoning_ended = True
current_text = state.previous_text + delta_text
@@ -709,8 +707,12 @@ class DelegatingParser(Parser):
)
)
# No parsers: pass through as content
if self._reasoning_parser is None and self._tool_parser is None:
# No phase active: pass through as content
if (
delta_message is None
and not self._in_reasoning_phase(state)
and not self._in_tool_call_phase(state)
):
delta_message = DeltaMessage(content=delta_text)
state.previous_text = current_text