diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index 226908d673..5790b9b082 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -15,8 +15,9 @@ Also covers cache usage computation in ``_build_anthropic_usage``. import json from argparse import Namespace from http import HTTPStatus +from types import SimpleNamespace from typing import Annotated -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import FastAPI @@ -1727,3 +1728,175 @@ class TestClientErrorResponses: assert response.status_code == HTTPStatus.BAD_REQUEST assert response.json()["error"]["type"] == "BadRequestError" + + +# ====================================================================== +# thinking configuration pass-through +# ====================================================================== + + +class TestThinkingConfig: + def test_absent_thinking_leaves_reasoning_untouched(self): + """Requests without `thinking` must convert exactly as before.""" + request = _make_request([{"role": "user", "content": "Hello"}]) + + result = _convert(request) + assert result.reasoning_effort is None + assert result.thinking_token_budget is None + assert result.include_reasoning is True + + def test_disabled_clears_reasoning_effort(self): + """`disabled` maps to reasoning_effort="none", which is what clears + enable_thinking for templates that honor it.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + thinking={"type": "disabled"}, + ) + + result = _convert(request) + assert result.reasoning_effort == "none" + + def test_disabled_overrides_output_config_effort(self): + """`thinking` is applied after output_config so an explicit opt-out wins + over an inherited effort ceiling.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + output_config={"effort": "high"}, + thinking={"type": "disabled"}, + ) + + result = _convert(request) + assert result.reasoning_effort == "none" + + def test_enabled_sets_thinking_token_budget(self): + request = AnthropicMessagesRequest( + model="test-model", + max_tokens=4096, + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "enabled", "budget_tokens": 2048}, + ) + + result = _convert(request) + assert result.thinking_token_budget == 2048 + assert result.reasoning_effort is None + + @pytest.mark.parametrize( + "thinking", + [ + pytest.param({"budget_tokens": 2048}, id="missing-type"), + pytest.param({"type": "enabled"}, id="enabled-missing-budget"), + pytest.param( + {"type": "enabled", "budget_tokens": 1023}, id="budget-below-1024" + ), + pytest.param( + {"type": "enabled", "budget_tokens": 4096}, id="budget-not-below-max" + ), + pytest.param({"type": "adaptive", "display": "full"}, id="bad-display"), + ], + ) + def test_rejects_invalid_thinking(self, thinking): + """Mirror the Anthropic API's BetaThinkingConfigParam constraints.""" + with pytest.raises(ValidationError): + AnthropicMessagesRequest( + model="test-model", + max_tokens=4096, + messages=[{"role": "user", "content": "Hello"}], + thinking=thinking, + ) + + def test_adaptive_pins_nothing_and_keeps_effort_ceiling(self): + """`adaptive` lets the model choose depth, so only the ceiling from + output_config.effort should survive.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + output_config={"effort": "low"}, + thinking={"type": "adaptive"}, + ) + + result = _convert(request) + assert result.reasoning_effort == "low" + assert result.thinking_token_budget is None + + def test_disabled_uses_configured_effort(self): + """Models that always think (GLM-5.3) or reject "none" (Harmony) are + served with a low effort instead.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + thinking={"type": "disabled"}, + ) + + result = _convert(request, disabled_thinking_effort="low") + assert result.reasoning_effort == "low" + + @pytest.mark.parametrize("display", ["omitted", "summarized", "updates"]) + def test_display_keeps_reasoning_included(self, display): + """Suppressing reasoning would mark it ended for structured outputs and + drop it from multi-turn history, so `display` is ignored.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + thinking={"type": "adaptive", "display": display}, + ) + + result = _convert(request) + assert result.include_reasoning is True + + def test_claude_code_payload(self): + """The combination Claude Code sends on every request: an effort ceiling + plus adaptive thinking with reasoning display omitted.""" + request = _make_request( + [{"role": "user", "content": "Hello"}], + output_config={"effort": "high"}, + thinking={"type": "adaptive", "display": "omitted"}, + ) + + result = _convert(request) + assert result.reasoning_effort == "high" + assert result.include_reasoning is True + assert result.thinking_token_budget is None + + +class TestProbeDisabledThinkingEffort: + """``auto`` falls back to ``low`` when ``none`` cannot turn thinking off.""" + + @staticmethod + async def _probe(render): + obj = MagicMock(spec=AnthropicServingMessages) + obj.online_renderer = MagicMock() + obj.online_renderer.render_chat = AsyncMock( + side_effect=lambda req: ([], [render(req.reasoning_effort)]) + ) + obj._extract_prompt_components = lambda engine_input: SimpleNamespace( + token_ids=engine_input, text=None + ) + obj._render_probe_prompt = ( + AnthropicServingMessages._render_probe_prompt.__get__(obj) + ) + return await AnthropicServingMessages._probe_disabled_thinking_effort(obj) + + @staticmethod + def _reject_none(effort): + """Harmony (gpt-oss) raises on ``none``.""" + if effort == "none": + raise ValueError(f"unsupported {effort=}") + return [1] + + @staticmethod + def _none_as_max(effort): + """GLM-5.3 treats efforts other than low/high as max.""" + return [1, {"low": 0, "high": 1}.get(effort, 2)] + + @pytest.mark.asyncio + async def test_template_honors_none(self): + assert await self._probe(lambda effort: [1, int(effort == "none")]) == "none" + + @pytest.mark.asyncio + async def test_template_ignores_effort(self): + assert await self._probe(lambda effort: [1]) == "low" + + @pytest.mark.asyncio + async def test_template_renders_none_as_thinking_effort(self): + assert await self._probe(self._none_as_max) == "low" + + @pytest.mark.asyncio + async def test_renderer_rejects_none(self): + assert await self._probe(self._reject_none) == "low" diff --git a/tests/entrypoints/anthropic/test_protocol_exports.py b/tests/entrypoints/anthropic/test_protocol_exports.py index 466f40e3cc..1123175793 100644 --- a/tests/entrypoints/anthropic/test_protocol_exports.py +++ b/tests/entrypoints/anthropic/test_protocol_exports.py @@ -19,6 +19,7 @@ from vllm.entrypoints.anthropic.protocol import ( AnthropicMessagesResponse, AnthropicOutputConfig, AnthropicStreamEvent, + AnthropicThinkingConfig, AnthropicUsage, ) @@ -35,6 +36,7 @@ SERVING_PROTOCOL_EXPORTS = ( AnthropicMessagesResponse, AnthropicOutputConfig, AnthropicStreamEvent, + AnthropicThinkingConfig, AnthropicUsage, ) diff --git a/vllm/entrypoints/anthropic/protocol.py b/vllm/entrypoints/anthropic/protocol.py index c879866dfb..48202831fe 100644 --- a/vllm/entrypoints/anthropic/protocol.py +++ b/vllm/entrypoints/anthropic/protocol.py @@ -111,13 +111,53 @@ class AnthropicJsonOutputFormat(BaseModel): type: Literal["json_schema"] = "json_schema" +AnthropicEffort = Literal["low", "medium", "high", "xhigh", "max"] +# reasoning_effort sent to the chat template for thinking.type=disabled. +AnthropicDisabledThinkingEffort = Literal["none", AnthropicEffort] +AnthropicDisabledThinkingEffortOption = Literal["auto", AnthropicDisabledThinkingEffort] +AnthropicThinkingDisplay = Literal["summarized", "omitted", "updates"] + + class AnthropicOutputConfig(BaseModel): """Configuration options for the model's output, such as the output format.""" - effort: Literal["low", "medium", "high", "xhigh", "max"] | None = None + effort: AnthropicEffort | None = None format: AnthropicJsonOutputFormat | None = None +class AnthropicThinkingConfigEnabled(BaseModel): + """Extended thinking with a fixed token budget. + + ``display`` is accepted but ignored: reasoning is always returned. + """ + + type: Literal["enabled"] + budget_tokens: int = Field(ge=1024) + display: AnthropicThinkingDisplay | None = None + + +class AnthropicThinkingConfigDisabled(BaseModel): + type: Literal["disabled"] + + +class AnthropicThinkingConfigAdaptive(BaseModel): + """Extended thinking whose depth the model chooses. + + ``display`` is accepted but ignored: reasoning is always returned. + """ + + type: Literal["adaptive"] + display: AnthropicThinkingDisplay | None = None + + +AnthropicThinkingConfig = Annotated[ + AnthropicThinkingConfigEnabled + | AnthropicThinkingConfigDisabled + | AnthropicThinkingConfigAdaptive, + Field(discriminator="type"), +] + + class AnthropicMessagesRequest(BaseModel): """Anthropic Messages API request.""" @@ -126,6 +166,7 @@ class AnthropicMessagesRequest(BaseModel): max_tokens: int metadata: dict[str, Any] | None = None output_config: AnthropicOutputConfig | None = None + thinking: AnthropicThinkingConfig | None = None stop_sequences: ( Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None ) = None @@ -190,6 +231,15 @@ class AnthropicMessagesRequest(BaseModel): raise ValueError("max_tokens must be positive") return v + @model_validator(mode="after") + def validate_thinking_budget(self) -> "AnthropicMessagesRequest": + if ( + isinstance(self.thinking, AnthropicThinkingConfigEnabled) + and self.thinking.budget_tokens >= self.max_tokens + ): + raise ValueError("thinking.budget_tokens must be less than max_tokens") + return self + class AnthropicDelta(BaseModel): """Delta for streaming responses.""" diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 6cc1271add..08f208f824 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -10,7 +10,7 @@ import logging import time import uuid from collections.abc import AsyncGenerator -from typing import Any +from typing import Any, get_args import jinja2 from fastapi import Request @@ -22,11 +22,15 @@ from vllm.entrypoints.anthropic.protocol import ( AnthropicCountTokensRequest, AnthropicCountTokensResponse, AnthropicDelta, + AnthropicDisabledThinkingEffort, + AnthropicDisabledThinkingEffortOption, + AnthropicEffort, AnthropicError, AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicOutputConfig, AnthropicStreamEvent, + AnthropicThinkingConfig, AnthropicUsage, ) from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption @@ -116,6 +120,7 @@ class AnthropicServingMessages(OpenAIServingChat): enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, + disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto", ): super().__init__( engine_client=engine_client, @@ -140,6 +145,53 @@ class AnthropicServingMessages(OpenAIServingChat): "tool_calls": "tool_use", } self._merge_inline_system = self._detect_merge_inline_system(chat_template) + # Resolved lazily from the renderer when "auto". + self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = ( + None if disabled_thinking_effort == "auto" else disabled_thinking_effort + ) + + async def _get_disabled_thinking_effort(self) -> AnthropicDisabledThinkingEffort: + if self._disabled_thinking_effort is None: + self._disabled_thinking_effort = ( + await self._probe_disabled_thinking_effort() + ) + logger.info( + "Anthropic thinking.type=disabled maps to reasoning_effort=%r", + self._disabled_thinking_effort, + ) + return self._disabled_thinking_effort + + async def _probe_disabled_thinking_effort(self) -> AnthropicDisabledThinkingEffort: + """Use ``low`` if the renderer rejects or ignores ``none``. + + ``none`` is ignored when it renders the same prompt as a thinking + effort, e.g. GLM-5.3 treats unknown efforts as ``max``. + """ + none_prompt = await self._render_probe_prompt("none") + if none_prompt is None: + return "low" + for effort in get_args(AnthropicEffort): + if await self._render_probe_prompt(effort) == none_prompt: + return "low" + return "none" + + async def _render_probe_prompt( + self, effort: AnthropicDisabledThinkingEffort + ) -> tuple[list[int] | None, str | None] | None: + request = ChatCompletionRequest( + messages=[{"role": "user", "content": "Hi"}], + reasoning_effort=effort, + ) + try: + result = await self.online_renderer.render_chat(request) + except Exception: + logger.debug("Rendering reasoning_effort=%r failed", effort, exc_info=True) + return None + if isinstance(result, ErrorResponse): + return None + _, (engine_input,) = result + components = self._extract_prompt_components(engine_input) + return components.token_ids, components.text @staticmethod def _detect_merge_inline_system(chat_template: str | None) -> bool: @@ -197,6 +249,7 @@ class AnthropicServingMessages(OpenAIServingChat): anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, *, merge_inline_system: bool = False, + disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none", ) -> ChatCompletionRequest: """Convert Anthropic message format to OpenAI format.""" openai_messages: list[dict[str, Any]] = [] @@ -214,6 +267,7 @@ class AnthropicServingMessages(OpenAIServingChat): req = cls._build_base_request(anthropic_request, openai_messages) cls._handle_streaming_options(req, anthropic_request) cls._handle_output_config(req, anthropic_request) + cls._handle_thinking(req, anthropic_request, disabled_thinking_effort) cls._convert_tool_choice(anthropic_request, req) cls._convert_tools(anthropic_request, req) return req @@ -494,6 +548,33 @@ class AnthropicServingMessages(OpenAIServingChat): chat_template_kwargs=anthropic_request.chat_template_kwargs, ) + @classmethod + def _handle_thinking( + cls, + req: ChatCompletionRequest, + anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, + disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none", + ) -> None: + """Handle extended-thinking configuration. + + ``display`` is intentionally ignored: suppressing reasoning would mark + it ended for structured outputs and drop it from multi-turn history. + """ + if isinstance(anthropic_request, AnthropicCountTokensRequest): + return + thinking: AnthropicThinkingConfig | None = anthropic_request.thinking + if thinking is None: + return + + if thinking.type == "disabled": + # "none" clears enable_thinking for templates that honor it; models + # that cannot disable thinking are configured with a low effort. + req.reasoning_effort = disabled_thinking_effort + elif thinking.type == "enabled": + req.thinking_token_budget = thinking.budget_tokens + # "adaptive" pins nothing: the model chooses depth beneath the ceiling + # already set from output_config.effort. + @classmethod def _handle_output_config( cls, @@ -602,9 +683,13 @@ class AnthropicServingMessages(OpenAIServingChat): """ if logger.isEnabledFor(logging.DEBUG): logger.debug("Received messages request %s", request.model_dump_json()) + disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none" + if request.thinking is not None and request.thinking.type == "disabled": + disabled_thinking_effort = await self._get_disabled_thinking_effort() chat_req = self.to_chat_completion_request( request, merge_inline_system=self._merge_inline_system, + disabled_thinking_effort=disabled_thinking_effort, ) if logger.isEnabledFor(logging.DEBUG): logger.debug("Convert to OpenAI request %s", chat_req.model_dump_json()) diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 189c4e6779..c5703515d8 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -185,6 +185,7 @@ async def init_generate_state( enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, default_chat_template_kwargs=default_chat_template_kwargs, + disabled_thinking_effort=args.anthropic_disabled_thinking_effort, ) if "generate" in supported_tasks else None diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 488a7f2c43..9b397bd282 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -14,6 +14,9 @@ from typing import Any, Literal import vllm.envs as envs from vllm.config import config from vllm.engine.arg_utils import AsyncEngineArgs, optional_type +from vllm.entrypoints.anthropic.protocol import ( + AnthropicDisabledThinkingEffortOption, +) from vllm.entrypoints.chat_utils import ( ChatTemplateContentFormatOption, validate_chat_template, @@ -181,6 +184,13 @@ class BaseFrontendArgs: ``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any explicit request-level ``chat_template_kwargs.cohere_format`` takes priority.""" + anthropic_disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto" + """Anthropic ``/v1/messages`` only. The ``reasoning_effort`` used for + requests with ``thinking: {"type": "disabled"}``. ``none`` turns thinking + off for models that support it; ``low`` suits models that always think + (e.g. GLM-5.3) or reject ``none`` (e.g. gpt-oss). ``auto`` (default) uses + ``low`` when the renderer rejects ``none`` or renders it the same as a + thinking effort, and ``none`` otherwise.""" log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE """If set to True, log the stack trace of error responses""" tokens_only: bool = False