[Frontend] Handle Disable Thinking in /v1/messages (#58613)
Signed-off-by: jryberg <johan.ryberg@security.ntt> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: jryberg <johan.ryberg@security.ntt> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Robert Shaw
co-authored by
jryberg
Robert Shaw
Claude Opus 5.5
jryberg
Robert Shaw
Claude Opus 5.5
parent
35f92d2abc
commit
5a2ed2e7ee
@@ -15,8 +15,9 @@ Also covers cache usage computation in ``_build_anthropic_usage``.
|
||||
import json
|
||||
from argparse import Namespace
|
||||
from http import HTTPStatus
|
||||
from types import SimpleNamespace
|
||||
from typing import Annotated
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
@@ -1727,3 +1728,175 @@ class TestClientErrorResponses:
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# thinking configuration pass-through
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestThinkingConfig:
|
||||
def test_absent_thinking_leaves_reasoning_untouched(self):
|
||||
"""Requests without `thinking` must convert exactly as before."""
|
||||
request = _make_request([{"role": "user", "content": "Hello"}])
|
||||
|
||||
result = _convert(request)
|
||||
assert result.reasoning_effort is None
|
||||
assert result.thinking_token_budget is None
|
||||
assert result.include_reasoning is True
|
||||
|
||||
def test_disabled_clears_reasoning_effort(self):
|
||||
"""`disabled` maps to reasoning_effort="none", which is what clears
|
||||
enable_thinking for templates that honor it."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
thinking={"type": "disabled"},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.reasoning_effort == "none"
|
||||
|
||||
def test_disabled_overrides_output_config_effort(self):
|
||||
"""`thinking` is applied after output_config so an explicit opt-out wins
|
||||
over an inherited effort ceiling."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
output_config={"effort": "high"},
|
||||
thinking={"type": "disabled"},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.reasoning_effort == "none"
|
||||
|
||||
def test_enabled_sets_thinking_token_budget(self):
|
||||
request = AnthropicMessagesRequest(
|
||||
model="test-model",
|
||||
max_tokens=4096,
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
thinking={"type": "enabled", "budget_tokens": 2048},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.thinking_token_budget == 2048
|
||||
assert result.reasoning_effort is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"thinking",
|
||||
[
|
||||
pytest.param({"budget_tokens": 2048}, id="missing-type"),
|
||||
pytest.param({"type": "enabled"}, id="enabled-missing-budget"),
|
||||
pytest.param(
|
||||
{"type": "enabled", "budget_tokens": 1023}, id="budget-below-1024"
|
||||
),
|
||||
pytest.param(
|
||||
{"type": "enabled", "budget_tokens": 4096}, id="budget-not-below-max"
|
||||
),
|
||||
pytest.param({"type": "adaptive", "display": "full"}, id="bad-display"),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_thinking(self, thinking):
|
||||
"""Mirror the Anthropic API's BetaThinkingConfigParam constraints."""
|
||||
with pytest.raises(ValidationError):
|
||||
AnthropicMessagesRequest(
|
||||
model="test-model",
|
||||
max_tokens=4096,
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
thinking=thinking,
|
||||
)
|
||||
|
||||
def test_adaptive_pins_nothing_and_keeps_effort_ceiling(self):
|
||||
"""`adaptive` lets the model choose depth, so only the ceiling from
|
||||
output_config.effort should survive."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
output_config={"effort": "low"},
|
||||
thinking={"type": "adaptive"},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.reasoning_effort == "low"
|
||||
assert result.thinking_token_budget is None
|
||||
|
||||
def test_disabled_uses_configured_effort(self):
|
||||
"""Models that always think (GLM-5.3) or reject "none" (Harmony) are
|
||||
served with a low effort instead."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
thinking={"type": "disabled"},
|
||||
)
|
||||
|
||||
result = _convert(request, disabled_thinking_effort="low")
|
||||
assert result.reasoning_effort == "low"
|
||||
|
||||
@pytest.mark.parametrize("display", ["omitted", "summarized", "updates"])
|
||||
def test_display_keeps_reasoning_included(self, display):
|
||||
"""Suppressing reasoning would mark it ended for structured outputs and
|
||||
drop it from multi-turn history, so `display` is ignored."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
thinking={"type": "adaptive", "display": display},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.include_reasoning is True
|
||||
|
||||
def test_claude_code_payload(self):
|
||||
"""The combination Claude Code sends on every request: an effort ceiling
|
||||
plus adaptive thinking with reasoning display omitted."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
output_config={"effort": "high"},
|
||||
thinking={"type": "adaptive", "display": "omitted"},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.reasoning_effort == "high"
|
||||
assert result.include_reasoning is True
|
||||
assert result.thinking_token_budget is None
|
||||
|
||||
|
||||
class TestProbeDisabledThinkingEffort:
|
||||
"""``auto`` falls back to ``low`` when ``none`` cannot turn thinking off."""
|
||||
|
||||
@staticmethod
|
||||
async def _probe(render):
|
||||
obj = MagicMock(spec=AnthropicServingMessages)
|
||||
obj.online_renderer = MagicMock()
|
||||
obj.online_renderer.render_chat = AsyncMock(
|
||||
side_effect=lambda req: ([], [render(req.reasoning_effort)])
|
||||
)
|
||||
obj._extract_prompt_components = lambda engine_input: SimpleNamespace(
|
||||
token_ids=engine_input, text=None
|
||||
)
|
||||
obj._render_probe_prompt = (
|
||||
AnthropicServingMessages._render_probe_prompt.__get__(obj)
|
||||
)
|
||||
return await AnthropicServingMessages._probe_disabled_thinking_effort(obj)
|
||||
|
||||
@staticmethod
|
||||
def _reject_none(effort):
|
||||
"""Harmony (gpt-oss) raises on ``none``."""
|
||||
if effort == "none":
|
||||
raise ValueError(f"unsupported {effort=}")
|
||||
return [1]
|
||||
|
||||
@staticmethod
|
||||
def _none_as_max(effort):
|
||||
"""GLM-5.3 treats efforts other than low/high as max."""
|
||||
return [1, {"low": 0, "high": 1}.get(effort, 2)]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_template_honors_none(self):
|
||||
assert await self._probe(lambda effort: [1, int(effort == "none")]) == "none"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_template_ignores_effort(self):
|
||||
assert await self._probe(lambda effort: [1]) == "low"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_template_renders_none_as_thinking_effort(self):
|
||||
assert await self._probe(self._none_as_max) == "low"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_renderer_rejects_none(self):
|
||||
assert await self._probe(self._reject_none) == "low"
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicMessagesResponse,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicStreamEvent,
|
||||
AnthropicThinkingConfig,
|
||||
AnthropicUsage,
|
||||
)
|
||||
|
||||
@@ -35,6 +36,7 @@ SERVING_PROTOCOL_EXPORTS = (
|
||||
AnthropicMessagesResponse,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicStreamEvent,
|
||||
AnthropicThinkingConfig,
|
||||
AnthropicUsage,
|
||||
)
|
||||
|
||||
|
||||
@@ -111,13 +111,53 @@ class AnthropicJsonOutputFormat(BaseModel):
|
||||
type: Literal["json_schema"] = "json_schema"
|
||||
|
||||
|
||||
AnthropicEffort = Literal["low", "medium", "high", "xhigh", "max"]
|
||||
# reasoning_effort sent to the chat template for thinking.type=disabled.
|
||||
AnthropicDisabledThinkingEffort = Literal["none", AnthropicEffort]
|
||||
AnthropicDisabledThinkingEffortOption = Literal["auto", AnthropicDisabledThinkingEffort]
|
||||
AnthropicThinkingDisplay = Literal["summarized", "omitted", "updates"]
|
||||
|
||||
|
||||
class AnthropicOutputConfig(BaseModel):
|
||||
"""Configuration options for the model's output, such as the output format."""
|
||||
|
||||
effort: Literal["low", "medium", "high", "xhigh", "max"] | None = None
|
||||
effort: AnthropicEffort | None = None
|
||||
format: AnthropicJsonOutputFormat | None = None
|
||||
|
||||
|
||||
class AnthropicThinkingConfigEnabled(BaseModel):
|
||||
"""Extended thinking with a fixed token budget.
|
||||
|
||||
``display`` is accepted but ignored: reasoning is always returned.
|
||||
"""
|
||||
|
||||
type: Literal["enabled"]
|
||||
budget_tokens: int = Field(ge=1024)
|
||||
display: AnthropicThinkingDisplay | None = None
|
||||
|
||||
|
||||
class AnthropicThinkingConfigDisabled(BaseModel):
|
||||
type: Literal["disabled"]
|
||||
|
||||
|
||||
class AnthropicThinkingConfigAdaptive(BaseModel):
|
||||
"""Extended thinking whose depth the model chooses.
|
||||
|
||||
``display`` is accepted but ignored: reasoning is always returned.
|
||||
"""
|
||||
|
||||
type: Literal["adaptive"]
|
||||
display: AnthropicThinkingDisplay | None = None
|
||||
|
||||
|
||||
AnthropicThinkingConfig = Annotated[
|
||||
AnthropicThinkingConfigEnabled
|
||||
| AnthropicThinkingConfigDisabled
|
||||
| AnthropicThinkingConfigAdaptive,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class AnthropicMessagesRequest(BaseModel):
|
||||
"""Anthropic Messages API request."""
|
||||
|
||||
@@ -126,6 +166,7 @@ class AnthropicMessagesRequest(BaseModel):
|
||||
max_tokens: int
|
||||
metadata: dict[str, Any] | None = None
|
||||
output_config: AnthropicOutputConfig | None = None
|
||||
thinking: AnthropicThinkingConfig | None = None
|
||||
stop_sequences: (
|
||||
Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
|
||||
) = None
|
||||
@@ -190,6 +231,15 @@ class AnthropicMessagesRequest(BaseModel):
|
||||
raise ValueError("max_tokens must be positive")
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_thinking_budget(self) -> "AnthropicMessagesRequest":
|
||||
if (
|
||||
isinstance(self.thinking, AnthropicThinkingConfigEnabled)
|
||||
and self.thinking.budget_tokens >= self.max_tokens
|
||||
):
|
||||
raise ValueError("thinking.budget_tokens must be less than max_tokens")
|
||||
return self
|
||||
|
||||
|
||||
class AnthropicDelta(BaseModel):
|
||||
"""Delta for streaming responses."""
|
||||
|
||||
@@ -10,7 +10,7 @@ import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any
|
||||
from typing import Any, get_args
|
||||
|
||||
import jinja2
|
||||
from fastapi import Request
|
||||
@@ -22,11 +22,15 @@ from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicCountTokensRequest,
|
||||
AnthropicCountTokensResponse,
|
||||
AnthropicDelta,
|
||||
AnthropicDisabledThinkingEffort,
|
||||
AnthropicDisabledThinkingEffortOption,
|
||||
AnthropicEffort,
|
||||
AnthropicError,
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicMessagesResponse,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicStreamEvent,
|
||||
AnthropicThinkingConfig,
|
||||
AnthropicUsage,
|
||||
)
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
@@ -116,6 +120,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
enable_prompt_tokens_details: bool = False,
|
||||
enable_force_include_usage: bool = False,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto",
|
||||
):
|
||||
super().__init__(
|
||||
engine_client=engine_client,
|
||||
@@ -140,6 +145,53 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
"tool_calls": "tool_use",
|
||||
}
|
||||
self._merge_inline_system = self._detect_merge_inline_system(chat_template)
|
||||
# Resolved lazily from the renderer when "auto".
|
||||
self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = (
|
||||
None if disabled_thinking_effort == "auto" else disabled_thinking_effort
|
||||
)
|
||||
|
||||
async def _get_disabled_thinking_effort(self) -> AnthropicDisabledThinkingEffort:
|
||||
if self._disabled_thinking_effort is None:
|
||||
self._disabled_thinking_effort = (
|
||||
await self._probe_disabled_thinking_effort()
|
||||
)
|
||||
logger.info(
|
||||
"Anthropic thinking.type=disabled maps to reasoning_effort=%r",
|
||||
self._disabled_thinking_effort,
|
||||
)
|
||||
return self._disabled_thinking_effort
|
||||
|
||||
async def _probe_disabled_thinking_effort(self) -> AnthropicDisabledThinkingEffort:
|
||||
"""Use ``low`` if the renderer rejects or ignores ``none``.
|
||||
|
||||
``none`` is ignored when it renders the same prompt as a thinking
|
||||
effort, e.g. GLM-5.3 treats unknown efforts as ``max``.
|
||||
"""
|
||||
none_prompt = await self._render_probe_prompt("none")
|
||||
if none_prompt is None:
|
||||
return "low"
|
||||
for effort in get_args(AnthropicEffort):
|
||||
if await self._render_probe_prompt(effort) == none_prompt:
|
||||
return "low"
|
||||
return "none"
|
||||
|
||||
async def _render_probe_prompt(
|
||||
self, effort: AnthropicDisabledThinkingEffort
|
||||
) -> tuple[list[int] | None, str | None] | None:
|
||||
request = ChatCompletionRequest(
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
reasoning_effort=effort,
|
||||
)
|
||||
try:
|
||||
result = await self.online_renderer.render_chat(request)
|
||||
except Exception:
|
||||
logger.debug("Rendering reasoning_effort=%r failed", effort, exc_info=True)
|
||||
return None
|
||||
if isinstance(result, ErrorResponse):
|
||||
return None
|
||||
_, (engine_input,) = result
|
||||
components = self._extract_prompt_components(engine_input)
|
||||
return components.token_ids, components.text
|
||||
|
||||
@staticmethod
|
||||
def _detect_merge_inline_system(chat_template: str | None) -> bool:
|
||||
@@ -197,6 +249,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||
*,
|
||||
merge_inline_system: bool = False,
|
||||
disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none",
|
||||
) -> ChatCompletionRequest:
|
||||
"""Convert Anthropic message format to OpenAI format."""
|
||||
openai_messages: list[dict[str, Any]] = []
|
||||
@@ -214,6 +267,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
req = cls._build_base_request(anthropic_request, openai_messages)
|
||||
cls._handle_streaming_options(req, anthropic_request)
|
||||
cls._handle_output_config(req, anthropic_request)
|
||||
cls._handle_thinking(req, anthropic_request, disabled_thinking_effort)
|
||||
cls._convert_tool_choice(anthropic_request, req)
|
||||
cls._convert_tools(anthropic_request, req)
|
||||
return req
|
||||
@@ -494,6 +548,33 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
chat_template_kwargs=anthropic_request.chat_template_kwargs,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _handle_thinking(
|
||||
cls,
|
||||
req: ChatCompletionRequest,
|
||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||
disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none",
|
||||
) -> None:
|
||||
"""Handle extended-thinking configuration.
|
||||
|
||||
``display`` is intentionally ignored: suppressing reasoning would mark
|
||||
it ended for structured outputs and drop it from multi-turn history.
|
||||
"""
|
||||
if isinstance(anthropic_request, AnthropicCountTokensRequest):
|
||||
return
|
||||
thinking: AnthropicThinkingConfig | None = anthropic_request.thinking
|
||||
if thinking is None:
|
||||
return
|
||||
|
||||
if thinking.type == "disabled":
|
||||
# "none" clears enable_thinking for templates that honor it; models
|
||||
# that cannot disable thinking are configured with a low effort.
|
||||
req.reasoning_effort = disabled_thinking_effort
|
||||
elif thinking.type == "enabled":
|
||||
req.thinking_token_budget = thinking.budget_tokens
|
||||
# "adaptive" pins nothing: the model chooses depth beneath the ceiling
|
||||
# already set from output_config.effort.
|
||||
|
||||
@classmethod
|
||||
def _handle_output_config(
|
||||
cls,
|
||||
@@ -602,9 +683,13 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
"""
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug("Received messages request %s", request.model_dump_json())
|
||||
disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none"
|
||||
if request.thinking is not None and request.thinking.type == "disabled":
|
||||
disabled_thinking_effort = await self._get_disabled_thinking_effort()
|
||||
chat_req = self.to_chat_completion_request(
|
||||
request,
|
||||
merge_inline_system=self._merge_inline_system,
|
||||
disabled_thinking_effort=disabled_thinking_effort,
|
||||
)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug("Convert to OpenAI request %s", chat_req.model_dump_json())
|
||||
|
||||
@@ -185,6 +185,7 @@ async def init_generate_state(
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
disabled_thinking_effort=args.anthropic_disabled_thinking_effort,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
|
||||
@@ -14,6 +14,9 @@ from typing import Any, Literal
|
||||
import vllm.envs as envs
|
||||
from vllm.config import config
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
|
||||
from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicDisabledThinkingEffortOption,
|
||||
)
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatTemplateContentFormatOption,
|
||||
validate_chat_template,
|
||||
@@ -181,6 +184,13 @@ class BaseFrontendArgs:
|
||||
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
|
||||
explicit request-level ``chat_template_kwargs.cohere_format`` takes
|
||||
priority."""
|
||||
anthropic_disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto"
|
||||
"""Anthropic ``/v1/messages`` only. The ``reasoning_effort`` used for
|
||||
requests with ``thinking: {"type": "disabled"}``. ``none`` turns thinking
|
||||
off for models that support it; ``low`` suits models that always think
|
||||
(e.g. GLM-5.3) or reject ``none`` (e.g. gpt-oss). ``auto`` (default) uses
|
||||
``low`` when the renderer rejects ``none`` or renders it the same as a
|
||||
thinking effort, and ``none`` otherwise."""
|
||||
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
|
||||
"""If set to True, log the stack trace of error responses"""
|
||||
tokens_only: bool = False
|
||||
|
||||
Reference in New Issue
Block a user