[Feature][Parser] Support include_reasoning param for non-Harmony models (#44301)
Signed-off-by: Alberto Perdomo <aperdomo@redhat.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -351,6 +351,69 @@ print(response.choices[0].message.reasoning)
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
## Suppressing Reasoning Output
|
||||
|
||||
You can suppress reasoning content from API responses using the `include_reasoning` parameter. When set to `false`, reasoning tokens are still generated (so model quality is unaffected) but excluded from the response. This reduces network traffic without changing inference behavior.
|
||||
|
||||
The parameter is supported in both the Chat Completions API and the Responses API, for streaming and non-streaming requests.
|
||||
|
||||
When `include_reasoning=false`, vLLM also suppresses per-token metadata (logprobs and token IDs) to prevent leaking reasoning content through decoded token text in logprob entries or raw token IDs.
|
||||
|
||||
### Chat Completions API
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY")
|
||||
model = client.models.list().data[0].id
|
||||
|
||||
# Reasoning is included by default (include_reasoning=True)
|
||||
response = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is 15 * 37?"}],
|
||||
extra_body={"include_reasoning": False},
|
||||
)
|
||||
|
||||
msg = response.choices[0].message
|
||||
assert msg.content # Content is still present
|
||||
assert not getattr(msg, "reasoning", None) # Reasoning is suppressed
|
||||
```
|
||||
|
||||
Streaming works the same way, reasoning deltas are omitted from chunks:
|
||||
|
||||
```python
|
||||
stream = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "What is 15 * 37?"}],
|
||||
stream=True,
|
||||
extra_body={"include_reasoning": False},
|
||||
)
|
||||
|
||||
for chunk in stream:
|
||||
delta = chunk.choices[0].delta
|
||||
# delta.reasoning will always be None
|
||||
if delta.content:
|
||||
print(delta.content, end="", flush=True)
|
||||
```
|
||||
|
||||
### Responses API
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY")
|
||||
|
||||
response = client.responses.create(
|
||||
model=client.models.list().data[0].id,
|
||||
input="What is 15 * 37?",
|
||||
include_reasoning=False,
|
||||
)
|
||||
|
||||
# No "reasoning" items in output
|
||||
types = [item.type for item in response.output]
|
||||
assert "reasoning" not in types
|
||||
```
|
||||
|
||||
## Limitations
|
||||
|
||||
- The reasoning content is only available for online serving's chat completion endpoint (`/v1/chat/completions`), Anthropic Messages API (`/v1/messages`) and the Responses API (`/v1/responses`).
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for ``include_reasoning`` with non-Harmony reasoning models.
|
||||
|
||||
Verifies that reasoning content is included by default and suppressed
|
||||
when ``include_reasoning=False``, for both streaming and non-streaming
|
||||
Chat Completions.
|
||||
"""
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
MESSAGES = [{"role": "user", "content": "What is 1+1? Be concise."}]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--reasoning-parser",
|
||||
"qwen3",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--enforce-eager",
|
||||
"--gpu-memory-utilization",
|
||||
"0.4",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def client(server):
|
||||
async with server.get_async_client() as async_client:
|
||||
yield async_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_include_reasoning_true_non_streaming(client: openai.AsyncOpenAI):
|
||||
"""Default: reasoning content appears in non-streaming response."""
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
extra_body={"include_reasoning": True},
|
||||
)
|
||||
|
||||
msg = response.choices[0].message
|
||||
reasoning = getattr(msg, "reasoning", None) or getattr(
|
||||
msg, "reasoning_content", None
|
||||
)
|
||||
assert reasoning, "Expected reasoning content when include_reasoning=True"
|
||||
assert msg.content, "Expected content in response"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_include_reasoning_false_non_streaming(client: openai.AsyncOpenAI):
|
||||
"""Reasoning content is suppressed when include_reasoning=False."""
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
extra_body={"include_reasoning": False},
|
||||
)
|
||||
|
||||
msg = response.choices[0].message
|
||||
reasoning = getattr(msg, "reasoning", None) or getattr(
|
||||
msg, "reasoning_content", None
|
||||
)
|
||||
assert not reasoning, (
|
||||
f"Expected no reasoning when include_reasoning=False, got: {reasoning}"
|
||||
)
|
||||
assert msg.content, "Expected content in response even without reasoning"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_include_reasoning_true_streaming(client: openai.AsyncOpenAI):
|
||||
"""Default: reasoning deltas appear in streaming response."""
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
stream=True,
|
||||
extra_body={"include_reasoning": True},
|
||||
)
|
||||
|
||||
reasoning_parts = []
|
||||
content_parts = []
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta if chunk.choices else None
|
||||
if delta:
|
||||
r = getattr(delta, "reasoning", None) or getattr(
|
||||
delta, "reasoning_content", None
|
||||
)
|
||||
if r:
|
||||
reasoning_parts.append(r)
|
||||
if delta.content:
|
||||
content_parts.append(delta.content)
|
||||
|
||||
reasoning_text = "".join(reasoning_parts)
|
||||
content_text = "".join(content_parts)
|
||||
|
||||
assert reasoning_text, "Expected reasoning deltas when include_reasoning=True"
|
||||
assert content_text, "Expected content deltas in streaming response"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_include_reasoning_false_streaming(client: openai.AsyncOpenAI):
|
||||
"""Reasoning deltas are suppressed in streaming when include_reasoning=False."""
|
||||
stream = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
stream=True,
|
||||
extra_body={"include_reasoning": False},
|
||||
)
|
||||
|
||||
reasoning_parts = []
|
||||
content_parts = []
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta if chunk.choices else None
|
||||
if delta:
|
||||
r = getattr(delta, "reasoning", None) or getattr(
|
||||
delta, "reasoning_content", None
|
||||
)
|
||||
if r:
|
||||
reasoning_parts.append(r)
|
||||
if delta.content:
|
||||
content_parts.append(delta.content)
|
||||
|
||||
reasoning_text = "".join(reasoning_parts)
|
||||
content_text = "".join(content_parts)
|
||||
|
||||
assert not reasoning_text, (
|
||||
f"Expected no reasoning deltas when include_reasoning=False, "
|
||||
f"got: {reasoning_text[:100]}"
|
||||
)
|
||||
assert content_text, "Expected content deltas even without reasoning"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_includes_reasoning(client: openai.AsyncOpenAI):
|
||||
"""Without specifying include_reasoning, reasoning appears (default=True)."""
|
||||
response = await client.chat.completions.create(
|
||||
model=MODEL_NAME,
|
||||
messages=MESSAGES,
|
||||
max_tokens=200,
|
||||
)
|
||||
|
||||
msg = response.choices[0].message
|
||||
reasoning = getattr(msg, "reasoning", None) or getattr(
|
||||
msg, "reasoning_content", None
|
||||
)
|
||||
assert reasoning, "Expected reasoning content by default"
|
||||
@@ -49,4 +49,5 @@ def mock_request():
|
||||
req = MagicMock(spec=ChatCompletionRequest)
|
||||
req.tools = []
|
||||
req.tool_choice = "auto"
|
||||
req.include_reasoning = True
|
||||
return req
|
||||
|
||||
@@ -43,6 +43,7 @@ def _make_request(**chat_template_kwargs):
|
||||
request = MagicMock(spec=ChatCompletionRequest)
|
||||
request.tools = []
|
||||
request.tool_choice = "auto"
|
||||
request.include_reasoning = True
|
||||
request.chat_template_kwargs = chat_template_kwargs or None
|
||||
return request
|
||||
|
||||
|
||||
@@ -896,6 +896,7 @@ def _make_delegating_request():
|
||||
req = MagicMock(spec=ChatCompletionRequest)
|
||||
req.tools = []
|
||||
req.tool_choice = "auto"
|
||||
req.include_reasoning = True
|
||||
return req
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,456 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for include_reasoning suppression in the unified Parser interface.
|
||||
|
||||
Covers non-streaming (parser.parse() + build_response_output_items),
|
||||
streaming (parse_delta), and ParsableContext.append_output() paths.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
_STRICT_TOOL_CALLING_ENV = "VLLM_ENFORCE_STRICT_TOOL_CALLING"
|
||||
_STRICT_TOOL_CALLING_ENV_VALUE = os.environ.get(_STRICT_TOOL_CALLING_ENV)
|
||||
os.environ[_STRICT_TOOL_CALLING_ENV] = "0"
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ( # noqa: E402
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage # noqa: E402
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest # noqa: E402
|
||||
from vllm.parser.abstract_parser import DelegatingParser # noqa: E402
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser # noqa: E402
|
||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def restore_strict_tool_calling_env():
|
||||
yield
|
||||
if _STRICT_TOOL_CALLING_ENV_VALUE is None:
|
||||
os.environ.pop(_STRICT_TOOL_CALLING_ENV, None)
|
||||
else:
|
||||
os.environ[_STRICT_TOOL_CALLING_ENV] = _STRICT_TOOL_CALLING_ENV_VALUE
|
||||
|
||||
|
||||
class ThinkReasoningParser(BaseThinkingReasoningParser):
|
||||
@property
|
||||
def start_token(self) -> str:
|
||||
return "<think>"
|
||||
|
||||
@property
|
||||
def end_token(self) -> str:
|
||||
return "</think>"
|
||||
|
||||
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT = (
|
||||
"<think>let me think about this</think>The answer is 42."
|
||||
)
|
||||
|
||||
MODEL_OUTPUT_REASONING_AND_TOOL = (
|
||||
"<think>I need to call a tool</think>"
|
||||
'<tool_call>\n{"name": "get_weather", '
|
||||
'"arguments": {"city": "Dallas"}}\n</tool_call>'
|
||||
)
|
||||
|
||||
MODEL_OUTPUT_CONTENT_ONLY = "The answer is 42."
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
return get_tokenizer("Qwen/Qwen3-32B")
|
||||
|
||||
|
||||
def make_responses_request(**kwargs) -> ResponsesRequest:
|
||||
defaults = dict(model="test-model", input="test input")
|
||||
defaults.update(kwargs)
|
||||
return ResponsesRequest(**defaults)
|
||||
|
||||
|
||||
def make_chat_request(**kwargs) -> ChatCompletionRequest:
|
||||
defaults = dict(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return ChatCompletionRequest(**defaults)
|
||||
|
||||
|
||||
def make_parser(tokenizer, reasoning=False, tool=False):
|
||||
class TestParser(DelegatingParser):
|
||||
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
|
||||
tool_parser_cls = Hermes2ProToolParser if tool else None
|
||||
|
||||
return TestParser(tokenizer)
|
||||
|
||||
|
||||
# ── Non-streaming: parser.parse() + build_response_output_items ──────
|
||||
|
||||
|
||||
def parse_and_build(parser, request, model_output, enable_auto_tools=False):
|
||||
"""Mirrors the non-streaming path in _make_response_output_items /
|
||||
ParsableContext.append_output(): parse → suppress reasoning → build items.
|
||||
"""
|
||||
from vllm.entrypoints.openai.responses.utils import (
|
||||
build_response_output_items,
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
model_output, request, enable_auto_tools=enable_auto_tools
|
||||
)
|
||||
if not request.include_reasoning:
|
||||
reasoning = None
|
||||
return build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
)
|
||||
|
||||
|
||||
class TestNonStreamingIncludeReasoning:
|
||||
def test_include_reasoning_true_has_reasoning_item(self, tokenizer):
|
||||
"""Default: reasoning items appear in output."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=True)
|
||||
|
||||
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
|
||||
|
||||
types = [o.type for o in outputs]
|
||||
assert "reasoning" in types
|
||||
assert "message" in types
|
||||
|
||||
def test_include_reasoning_false_no_reasoning_item(self, tokenizer):
|
||||
"""Reasoning item is suppressed when include_reasoning=False."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
|
||||
|
||||
types = [o.type for o in outputs]
|
||||
assert "reasoning" not in types
|
||||
assert "message" in types
|
||||
assert outputs[0].content[0].text == "The answer is 42."
|
||||
|
||||
def test_include_reasoning_false_content_preserved(self, tokenizer):
|
||||
"""Content is extracted correctly even when reasoning is suppressed."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
|
||||
|
||||
message = next(o for o in outputs if o.type == "message")
|
||||
assert message.content[0].text == "The answer is 42."
|
||||
|
||||
def test_include_reasoning_false_tool_calls_preserved(self, tokenizer):
|
||||
"""Tool calls still work when reasoning is suppressed."""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
request = make_responses_request(
|
||||
include_reasoning=False,
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
outputs = parse_and_build(
|
||||
parser,
|
||||
request,
|
||||
MODEL_OUTPUT_REASONING_AND_TOOL,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
types = [o.type for o in outputs]
|
||||
assert "reasoning" not in types
|
||||
assert "function_call" in types
|
||||
fc = next(o for o in outputs if o.type == "function_call")
|
||||
assert fc.name == "get_weather"
|
||||
assert json.loads(fc.arguments) == {"city": "Dallas"}
|
||||
|
||||
def test_no_reasoning_parser_include_false_is_noop(self, tokenizer):
|
||||
"""include_reasoning=False is harmless when no reasoning parser."""
|
||||
parser = make_parser(tokenizer, reasoning=False)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
outputs = parse_and_build(parser, request, MODEL_OUTPUT_CONTENT_ONLY)
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].type == "message"
|
||||
assert outputs[0].content[0].text == MODEL_OUTPUT_CONTENT_ONLY
|
||||
|
||||
def test_default_include_reasoning_is_true(self, tokenizer):
|
||||
"""ResponsesRequest defaults to include_reasoning=True."""
|
||||
request = make_responses_request()
|
||||
assert request.include_reasoning is True
|
||||
|
||||
def test_include_reasoning_false_suppresses_all_reasoning(self, tokenizer):
|
||||
"""Reasoning is suppressed regardless of request type."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
|
||||
|
||||
assert all(o.type != "reasoning" for o in outputs)
|
||||
|
||||
|
||||
# ── Streaming: parse_delta ───────────────────────────────────────────
|
||||
|
||||
|
||||
def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
|
||||
token_ids = tokenizer.encode(text, add_special_tokens=False)
|
||||
results: list[DeltaMessage | None] = []
|
||||
for i, tid in enumerate(token_ids):
|
||||
delta_text = tokenizer.decode([tid])
|
||||
is_last = i == len(token_ids) - 1
|
||||
result = parser.parse_delta(
|
||||
delta_text,
|
||||
[tid],
|
||||
request,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=is_last,
|
||||
)
|
||||
prompt_token_ids = None
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def collect_fields(results):
|
||||
all_reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
|
||||
all_content = "".join(r.content for r in results if r and r.content)
|
||||
all_tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls]
|
||||
return all_reasoning, all_content, all_tool_calls
|
||||
|
||||
|
||||
class TestParseDeltaIncludeReasoning:
|
||||
def test_streaming_include_true_emits_reasoning(self, tokenizer):
|
||||
"""With include_reasoning=True, reasoning deltas are emitted."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=True)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
reasoning, content, _ = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert "42" in content
|
||||
|
||||
def test_streaming_include_false_suppresses_reasoning(self, tokenizer):
|
||||
"""With include_reasoning=False, no reasoning deltas are emitted."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
reasoning, content, _ = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert "42" in content
|
||||
|
||||
def test_streaming_include_false_content_still_works(self, tokenizer):
|
||||
"""Content is correctly extracted in streaming even with suppression."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
_, content, _ = collect_fields(results)
|
||||
|
||||
assert "The answer is 42" in content
|
||||
|
||||
def test_streaming_include_false_tool_calls_preserved(self, tokenizer):
|
||||
"""Tool calls stream correctly when reasoning is suppressed."""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
request = make_responses_request(
|
||||
include_reasoning=False,
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_TOOL,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert len(tool_calls) > 0
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
|
||||
def test_streaming_no_reasoning_parser_include_false(self, tokenizer):
|
||||
"""No crash when reasoning parser absent and include_reasoning=False."""
|
||||
parser = make_parser(tokenizer, reasoning=False)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_CONTENT_ONLY,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
reasoning, content, _ = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert "42" in content
|
||||
|
||||
def test_streaming_chat_completion_include_false(self, tokenizer):
|
||||
"""parse_delta also respects ChatCompletionRequest.include_reasoning."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_chat_request(include_reasoning=False)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
reasoning, content, _ = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert "42" in content
|
||||
|
||||
def test_streaming_reasoning_only_deltas_become_none(self, tokenizer):
|
||||
"""Deltas that carry only reasoning become None (not empty)."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
|
||||
for r in results:
|
||||
if r is not None:
|
||||
assert r.reasoning is None
|
||||
|
||||
|
||||
# ── ParsableContext.append_output() ───────────────────────────────────
|
||||
|
||||
|
||||
class TestParsableContextIncludeReasoning:
|
||||
def _make_context(self, tokenizer, request):
|
||||
from vllm.entrypoints.openai.responses.context import ParsableContext
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
reasoning_parser_cls = ThinkReasoningParser
|
||||
tool_parser_cls = None
|
||||
|
||||
return ParsableContext(
|
||||
tokenizer=tokenizer,
|
||||
parser_cls=TestParser,
|
||||
response_messages=[],
|
||||
request=request,
|
||||
available_tools=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
def test_process_include_false_suppresses_reasoning(self, tokenizer):
|
||||
"""ParsableContext.process() suppresses reasoning items."""
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
ctx = self._make_context(tokenizer, request)
|
||||
|
||||
output = RequestOutput(
|
||||
request_id="test",
|
||||
prompt=None,
|
||||
prompt_token_ids=[],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
token_ids=tokenizer.encode(
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
add_special_tokens=False,
|
||||
),
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
ctx.append_output(output)
|
||||
|
||||
types = [getattr(m, "type", None) for m in ctx.response_messages]
|
||||
assert "reasoning" not in types
|
||||
assert "message" in types
|
||||
|
||||
def test_process_include_true_has_reasoning(self, tokenizer):
|
||||
"""ParsableContext.process() includes reasoning by default."""
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
|
||||
request = make_responses_request(include_reasoning=True)
|
||||
ctx = self._make_context(tokenizer, request)
|
||||
|
||||
output = RequestOutput(
|
||||
request_id="test",
|
||||
prompt=None,
|
||||
prompt_token_ids=[],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
token_ids=tokenizer.encode(
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
add_special_tokens=False,
|
||||
),
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
ctx.append_output(output)
|
||||
|
||||
types = [getattr(m, "type", None) for m in ctx.response_messages]
|
||||
assert "reasoning" in types
|
||||
assert "message" in types
|
||||
@@ -600,19 +600,8 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
prompt_token_ids=res.prompt_token_ids,
|
||||
finished=output.finish_reason is not None,
|
||||
)
|
||||
if delta_message is not None:
|
||||
if delta_message.tool_calls:
|
||||
tools_streamed[i] = True
|
||||
|
||||
if (
|
||||
delta_message.reasoning
|
||||
and not request.include_reasoning
|
||||
):
|
||||
delta_message.reasoning = None
|
||||
if not (
|
||||
delta_message.content or delta_message.tool_calls
|
||||
):
|
||||
delta_message = None
|
||||
if delta_message is not None and delta_message.tool_calls:
|
||||
tools_streamed[i] = True
|
||||
|
||||
# handle streaming just a content delta (no parsers)
|
||||
else:
|
||||
@@ -627,13 +616,22 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# "control token" for tool calls or the parser otherwise
|
||||
# wasn't ready to send a token, then
|
||||
# get the next token without streaming a chunk
|
||||
# When reasoning is hidden, suppress per-token
|
||||
# metadata (logprobs, token_ids) on every chunk to
|
||||
# prevent leaking reasoning tokens through decoded
|
||||
# token text in logprob entries or raw token IDs.
|
||||
hide_stream_metadata = (
|
||||
not request.include_reasoning and parser is not None
|
||||
)
|
||||
if hide_stream_metadata:
|
||||
logprobs = None
|
||||
|
||||
if delta_message is None:
|
||||
# NOTE: If return_token_ids is enabled, we still need to
|
||||
# send a chunk with token_ids even if delta_message is None
|
||||
# to ensure all tokens are included in the response
|
||||
if (
|
||||
output.finish_reason is None
|
||||
and not request.return_token_ids
|
||||
if output.finish_reason is None and (
|
||||
not request.return_token_ids or hide_stream_metadata
|
||||
):
|
||||
continue
|
||||
delta_message = DeltaMessage()
|
||||
@@ -666,6 +664,10 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
delta=True,
|
||||
)
|
||||
|
||||
include_token_ids = (
|
||||
request.return_token_ids and not hide_stream_metadata
|
||||
)
|
||||
|
||||
if output.finish_reason is None:
|
||||
# Send token-by-token response for each request.n
|
||||
choice_data = ChatCompletionResponseStreamChoice(
|
||||
@@ -674,9 +676,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
logprobs=logprobs,
|
||||
finish_reason=None,
|
||||
token_ids=(
|
||||
as_list(output.token_ids)
|
||||
if request.return_token_ids
|
||||
else None
|
||||
as_list(output.token_ids) if include_token_ids else None
|
||||
),
|
||||
)
|
||||
|
||||
@@ -704,9 +704,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
finish_reason=finish_reason_,
|
||||
stop_reason=output.stop_reason,
|
||||
token_ids=(
|
||||
as_list(output.token_ids)
|
||||
if request.return_token_ids
|
||||
else None
|
||||
as_list(output.token_ids) if include_token_ids else None
|
||||
),
|
||||
)
|
||||
|
||||
@@ -883,12 +881,16 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
model_output_token_ids=token_ids,
|
||||
)
|
||||
suppress_metadata = not request.include_reasoning and parser is not None
|
||||
if not request.include_reasoning:
|
||||
reasoning = None
|
||||
if suppress_metadata:
|
||||
logprobs = None
|
||||
else:
|
||||
reasoning = None
|
||||
content = output.text
|
||||
tool_calls = []
|
||||
suppress_metadata = False
|
||||
|
||||
auto_tools_called = False
|
||||
is_named_tool_choice = (
|
||||
@@ -982,7 +984,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
else "stop",
|
||||
stop_reason=output.stop_reason,
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
as_list(output.token_ids)
|
||||
if request.return_token_ids and not suppress_metadata
|
||||
else None
|
||||
),
|
||||
routed_experts=routed_experts_b64,
|
||||
)
|
||||
|
||||
@@ -310,7 +310,9 @@ class ParsableContext(ConversationContext):
|
||||
self.finish_reason: str | None = None
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
|
||||
self.response_parser = response_parser
|
||||
self.response_parser = response_parser or (
|
||||
parser_cls(tokenizer, request.tools) if parser_cls is not None else None
|
||||
)
|
||||
self.parser_cls = parser_cls
|
||||
self.request = request
|
||||
|
||||
@@ -344,6 +346,8 @@ class ParsableContext(ConversationContext):
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
model_output_token_ids=completion.token_ids,
|
||||
)
|
||||
if not self.request.include_reasoning:
|
||||
reasoning = None
|
||||
self.response_messages.extend(
|
||||
build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
|
||||
@@ -161,6 +161,15 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
previous_response_id: str | None = None
|
||||
prompt: ResponsePrompt | None = None
|
||||
reasoning: Reasoning | None = None
|
||||
include_reasoning: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"Whether to include reasoning content in the response. "
|
||||
"When false, reasoning tokens are still generated but "
|
||||
"excluded from the output. This reduces network traffic "
|
||||
"without affecting model inference."
|
||||
),
|
||||
)
|
||||
service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto"
|
||||
store: bool | None = True
|
||||
stream: bool | None = False
|
||||
|
||||
@@ -1068,6 +1068,9 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
model_output_token_ids=final_output.token_ids,
|
||||
)
|
||||
if not request.include_reasoning:
|
||||
reasoning = None
|
||||
logprobs = None
|
||||
return build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
@@ -1342,11 +1345,15 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
) -> AsyncGenerator[StreamingResponsesResponse, None]:
|
||||
processor = SimpleStreamingEventProcessor(tools=request.tools)
|
||||
|
||||
hide_stream_metadata = not request.include_reasoning and self.parser is not None
|
||||
|
||||
def _get_logprobs(
|
||||
output: CompletionOutput,
|
||||
) -> list[response_text_delta_event.Logprob]:
|
||||
if not request.is_include_output_logprobs():
|
||||
return []
|
||||
if hide_stream_metadata:
|
||||
return []
|
||||
return self._create_stream_response_logprobs(
|
||||
token_ids=output.token_ids,
|
||||
logprobs=output.logprobs,
|
||||
|
||||
@@ -447,7 +447,7 @@ class DelegatingParser(Parser):
|
||||
|
||||
tool_calls = list[FunctionCall]()
|
||||
if is_named_tool_choice and supports_required_and_named:
|
||||
if content is None:
|
||||
if content is None or (isinstance(content, str) and not content.strip()):
|
||||
return [], None
|
||||
function_name = self._get_function_name(request)
|
||||
tool_calls.append(
|
||||
@@ -496,6 +496,14 @@ class DelegatingParser(Parser):
|
||||
content = None
|
||||
else:
|
||||
# No tool calls.
|
||||
# For required/named tool choice (when falling back to auto
|
||||
# parsing), if content is empty or whitespace-only, return
|
||||
# empty list with None content.
|
||||
if (is_required_tool_choice or is_named_tool_choice) and (
|
||||
content is None
|
||||
or (isinstance(content, str) and not content.strip())
|
||||
):
|
||||
return [], None
|
||||
return None, content
|
||||
|
||||
return tool_calls, content
|
||||
@@ -917,6 +925,15 @@ class DelegatingParser(Parser):
|
||||
delta_message = self.finalize_generation(delta_message, request, state)
|
||||
delta_message = self._flush_engine_parsers(delta_message)
|
||||
|
||||
# Suppress reasoning deltas if not requested
|
||||
if delta_message and not request.include_reasoning:
|
||||
delta_message.reasoning = None
|
||||
|
||||
# If only reasoning was in the message (no content, no tool_calls)
|
||||
# skip emitting entirely
|
||||
if not delta_message.content and not delta_message.tool_calls:
|
||||
delta_message = None
|
||||
|
||||
return delta_message
|
||||
|
||||
def _flush_engine_parsers(
|
||||
|
||||
@@ -443,7 +443,15 @@ class ParserEngine(Parser):
|
||||
if finished:
|
||||
events.extend(self._engine.finish())
|
||||
result = self._events_to_delta(events, finished=finished)
|
||||
return self._strip_trailing_reasoning(result)
|
||||
result = self._strip_trailing_reasoning(result)
|
||||
|
||||
# Suppress reasoning deltas if not requested
|
||||
if result and not request.include_reasoning:
|
||||
result.reasoning = None
|
||||
if not result.content and not result.tool_calls:
|
||||
result = None
|
||||
|
||||
return result
|
||||
|
||||
def _strip_trailing_reasoning(
|
||||
self,
|
||||
|
||||
@@ -293,6 +293,16 @@ class HarmonyParser(DelegatingParser):
|
||||
delta_message.reasoning = combined_reasoning
|
||||
if tool_messages:
|
||||
delta_message.tool_calls = tool_messages
|
||||
|
||||
# Suppress reasoning deltas if not requested
|
||||
if delta_message and not request.include_reasoning:
|
||||
delta_message.reasoning = None
|
||||
|
||||
# If only reasoning was in the message (no content, no tool_calls)
|
||||
# skip emitting entirely
|
||||
if not delta_message.content and not delta_message.tool_calls:
|
||||
return None
|
||||
|
||||
return delta_message
|
||||
|
||||
def process_chunk(self, token_ids: Sequence[int]) -> ChunkResult:
|
||||
|
||||
Reference in New Issue
Block a user