[Frontend] Handle Disable Thinking in /v1/messages (#58613)
Signed-off-by: jryberg <johan.ryberg@security.ntt> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: jryberg <johan.ryberg@security.ntt> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
jryberg
Robert Shaw
Claude Opus 5.5
parent
35f92d2abc
commit
5a2ed2e7ee
@@ -15,8 +15,9 @@ Also covers cache usage computation in ``_build_anthropic_usage``.
|
||||
import json
|
||||
from argparse import Namespace
|
||||
from http import HTTPStatus
|
||||
from types import SimpleNamespace
|
||||
from typing import Annotated
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
@@ -1727,3 +1728,175 @@ class TestClientErrorResponses:
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json()["error"]["type"] == "BadRequestError"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# thinking configuration pass-through
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestThinkingConfig:
|
||||
def test_absent_thinking_leaves_reasoning_untouched(self):
|
||||
"""Requests without `thinking` must convert exactly as before."""
|
||||
request = _make_request([{"role": "user", "content": "Hello"}])
|
||||
|
||||
result = _convert(request)
|
||||
assert result.reasoning_effort is None
|
||||
assert result.thinking_token_budget is None
|
||||
assert result.include_reasoning is True
|
||||
|
||||
def test_disabled_clears_reasoning_effort(self):
|
||||
"""`disabled` maps to reasoning_effort="none", which is what clears
|
||||
enable_thinking for templates that honor it."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
thinking={"type": "disabled"},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.reasoning_effort == "none"
|
||||
|
||||
def test_disabled_overrides_output_config_effort(self):
|
||||
"""`thinking` is applied after output_config so an explicit opt-out wins
|
||||
over an inherited effort ceiling."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
output_config={"effort": "high"},
|
||||
thinking={"type": "disabled"},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.reasoning_effort == "none"
|
||||
|
||||
def test_enabled_sets_thinking_token_budget(self):
|
||||
request = AnthropicMessagesRequest(
|
||||
model="test-model",
|
||||
max_tokens=4096,
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
thinking={"type": "enabled", "budget_tokens": 2048},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.thinking_token_budget == 2048
|
||||
assert result.reasoning_effort is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"thinking",
|
||||
[
|
||||
pytest.param({"budget_tokens": 2048}, id="missing-type"),
|
||||
pytest.param({"type": "enabled"}, id="enabled-missing-budget"),
|
||||
pytest.param(
|
||||
{"type": "enabled", "budget_tokens": 1023}, id="budget-below-1024"
|
||||
),
|
||||
pytest.param(
|
||||
{"type": "enabled", "budget_tokens": 4096}, id="budget-not-below-max"
|
||||
),
|
||||
pytest.param({"type": "adaptive", "display": "full"}, id="bad-display"),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_thinking(self, thinking):
|
||||
"""Mirror the Anthropic API's BetaThinkingConfigParam constraints."""
|
||||
with pytest.raises(ValidationError):
|
||||
AnthropicMessagesRequest(
|
||||
model="test-model",
|
||||
max_tokens=4096,
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
thinking=thinking,
|
||||
)
|
||||
|
||||
def test_adaptive_pins_nothing_and_keeps_effort_ceiling(self):
|
||||
"""`adaptive` lets the model choose depth, so only the ceiling from
|
||||
output_config.effort should survive."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
output_config={"effort": "low"},
|
||||
thinking={"type": "adaptive"},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.reasoning_effort == "low"
|
||||
assert result.thinking_token_budget is None
|
||||
|
||||
def test_disabled_uses_configured_effort(self):
|
||||
"""Models that always think (GLM-5.3) or reject "none" (Harmony) are
|
||||
served with a low effort instead."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
thinking={"type": "disabled"},
|
||||
)
|
||||
|
||||
result = _convert(request, disabled_thinking_effort="low")
|
||||
assert result.reasoning_effort == "low"
|
||||
|
||||
@pytest.mark.parametrize("display", ["omitted", "summarized", "updates"])
|
||||
def test_display_keeps_reasoning_included(self, display):
|
||||
"""Suppressing reasoning would mark it ended for structured outputs and
|
||||
drop it from multi-turn history, so `display` is ignored."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
thinking={"type": "adaptive", "display": display},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.include_reasoning is True
|
||||
|
||||
def test_claude_code_payload(self):
|
||||
"""The combination Claude Code sends on every request: an effort ceiling
|
||||
plus adaptive thinking with reasoning display omitted."""
|
||||
request = _make_request(
|
||||
[{"role": "user", "content": "Hello"}],
|
||||
output_config={"effort": "high"},
|
||||
thinking={"type": "adaptive", "display": "omitted"},
|
||||
)
|
||||
|
||||
result = _convert(request)
|
||||
assert result.reasoning_effort == "high"
|
||||
assert result.include_reasoning is True
|
||||
assert result.thinking_token_budget is None
|
||||
|
||||
|
||||
class TestProbeDisabledThinkingEffort:
|
||||
"""``auto`` falls back to ``low`` when ``none`` cannot turn thinking off."""
|
||||
|
||||
@staticmethod
|
||||
async def _probe(render):
|
||||
obj = MagicMock(spec=AnthropicServingMessages)
|
||||
obj.online_renderer = MagicMock()
|
||||
obj.online_renderer.render_chat = AsyncMock(
|
||||
side_effect=lambda req: ([], [render(req.reasoning_effort)])
|
||||
)
|
||||
obj._extract_prompt_components = lambda engine_input: SimpleNamespace(
|
||||
token_ids=engine_input, text=None
|
||||
)
|
||||
obj._render_probe_prompt = (
|
||||
AnthropicServingMessages._render_probe_prompt.__get__(obj)
|
||||
)
|
||||
return await AnthropicServingMessages._probe_disabled_thinking_effort(obj)
|
||||
|
||||
@staticmethod
|
||||
def _reject_none(effort):
|
||||
"""Harmony (gpt-oss) raises on ``none``."""
|
||||
if effort == "none":
|
||||
raise ValueError(f"unsupported {effort=}")
|
||||
return [1]
|
||||
|
||||
@staticmethod
|
||||
def _none_as_max(effort):
|
||||
"""GLM-5.3 treats efforts other than low/high as max."""
|
||||
return [1, {"low": 0, "high": 1}.get(effort, 2)]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_template_honors_none(self):
|
||||
assert await self._probe(lambda effort: [1, int(effort == "none")]) == "none"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_template_ignores_effort(self):
|
||||
assert await self._probe(lambda effort: [1]) == "low"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_template_renders_none_as_thinking_effort(self):
|
||||
assert await self._probe(self._none_as_max) == "low"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_renderer_rejects_none(self):
|
||||
assert await self._probe(self._reject_none) == "low"
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicMessagesResponse,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicStreamEvent,
|
||||
AnthropicThinkingConfig,
|
||||
AnthropicUsage,
|
||||
)
|
||||
|
||||
@@ -35,6 +36,7 @@ SERVING_PROTOCOL_EXPORTS = (
|
||||
AnthropicMessagesResponse,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicStreamEvent,
|
||||
AnthropicThinkingConfig,
|
||||
AnthropicUsage,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user