[Bugfix][Frontend] Apply output_config and thinking in Anthropic count_tokens (#59180)
Signed-off-by: Kang Nahoon <ejrtks102020@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
1283362e76
commit
1995c0fcd0
@@ -1987,6 +1987,32 @@ class TestThinkingConfig:
|
|||||||
assert result.include_reasoning is True
|
assert result.include_reasoning is True
|
||||||
assert result.thinking_token_budget is None
|
assert result.thinking_token_budget is None
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"config",
|
||||||
|
[
|
||||||
|
pytest.param({"output_config": {"effort": "low"}}, id="effort"),
|
||||||
|
pytest.param({"thinking": {"type": "disabled"}}, id="disabled"),
|
||||||
|
pytest.param(
|
||||||
|
{"thinking": {"type": "enabled", "budget_tokens": 2048}},
|
||||||
|
id="enabled",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_count_tokens_matches_messages(self, config):
|
||||||
|
"""Effort and thinking change the rendered prompt, so count_tokens must
|
||||||
|
apply them the same way /v1/messages does."""
|
||||||
|
messages = [{"role": "user", "content": "Hello"}]
|
||||||
|
expected = _convert(
|
||||||
|
AnthropicMessagesRequest(
|
||||||
|
model="test-model", max_tokens=4096, messages=messages, **config
|
||||||
|
)
|
||||||
|
)
|
||||||
|
result = _convert(
|
||||||
|
AnthropicCountTokensRequest(model="test-model", messages=messages, **config)
|
||||||
|
)
|
||||||
|
assert result.reasoning_effort == expected.reasoning_effort
|
||||||
|
assert result.thinking_token_budget == expected.thinking_token_budget
|
||||||
|
|
||||||
|
|
||||||
class TestProbeDisabledThinkingEffort:
|
class TestProbeDisabledThinkingEffort:
|
||||||
"""``auto`` falls back to ``low`` when ``none`` cannot turn thinking off."""
|
"""``auto`` falls back to ``low`` when ``none`` cannot turn thinking off."""
|
||||||
|
|||||||
@@ -385,7 +385,9 @@ class AnthropicCountTokensRequest(BaseModel):
|
|||||||
|
|
||||||
model: str
|
model: str
|
||||||
messages: list[AnthropicMessage]
|
messages: list[AnthropicMessage]
|
||||||
|
output_config: AnthropicOutputConfig | None = None
|
||||||
system: str | list[AnthropicContentBlock] | None = None
|
system: str | list[AnthropicContentBlock] | None = None
|
||||||
|
thinking: AnthropicThinkingConfig | None = None
|
||||||
tool_choice: AnthropicToolChoice | None = None
|
tool_choice: AnthropicToolChoice | None = None
|
||||||
tools: list[AnthropicTool] | None = None
|
tools: list[AnthropicTool] | None = None
|
||||||
|
|
||||||
|
|||||||
@@ -633,8 +633,6 @@ class AnthropicServingMessages(OpenAIServingChat):
|
|||||||
``display`` is intentionally ignored: suppressing reasoning would mark
|
``display`` is intentionally ignored: suppressing reasoning would mark
|
||||||
it ended for structured outputs and drop it from multi-turn history.
|
it ended for structured outputs and drop it from multi-turn history.
|
||||||
"""
|
"""
|
||||||
if isinstance(anthropic_request, AnthropicCountTokensRequest):
|
|
||||||
return
|
|
||||||
thinking: AnthropicThinkingConfig | None = anthropic_request.thinking
|
thinking: AnthropicThinkingConfig | None = anthropic_request.thinking
|
||||||
if thinking is None:
|
if thinking is None:
|
||||||
return
|
return
|
||||||
@@ -655,8 +653,6 @@ class AnthropicServingMessages(OpenAIServingChat):
|
|||||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Handle output configuration such as output format and effort."""
|
"""Handle output configuration such as output format and effort."""
|
||||||
if isinstance(anthropic_request, AnthropicCountTokensRequest):
|
|
||||||
return
|
|
||||||
output_config: AnthropicOutputConfig | None = anthropic_request.output_config
|
output_config: AnthropicOutputConfig | None = anthropic_request.output_config
|
||||||
if output_config and output_config.format and output_config.format.json_schema:
|
if output_config and output_config.format and output_config.format.json_schema:
|
||||||
req.response_format = ResponseFormat(
|
req.response_format = ResponseFormat(
|
||||||
@@ -1238,9 +1234,13 @@ class AnthropicServingMessages(OpenAIServingChat):
|
|||||||
raw_request: Request | None = None,
|
raw_request: Request | None = None,
|
||||||
) -> AnthropicCountTokensResponse | ErrorResponse:
|
) -> AnthropicCountTokensResponse | ErrorResponse:
|
||||||
"""Implements Anthropic's messages.count_tokens endpoint."""
|
"""Implements Anthropic's messages.count_tokens endpoint."""
|
||||||
|
disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none"
|
||||||
|
if request.thinking is not None and request.thinking.type == "disabled":
|
||||||
|
disabled_thinking_effort = await self._get_disabled_thinking_effort()
|
||||||
chat_req = self.to_chat_completion_request(
|
chat_req = self.to_chat_completion_request(
|
||||||
request,
|
request,
|
||||||
merge_inline_system=self._merge_inline_system,
|
merge_inline_system=self._merge_inline_system,
|
||||||
|
disabled_thinking_effort=disabled_thinking_effort,
|
||||||
)
|
)
|
||||||
result = await self.render_chat_request(chat_req)
|
result = await self.render_chat_request(chat_req)
|
||||||
if isinstance(result, ErrorResponse):
|
if isinstance(result, ErrorResponse):
|
||||||
|
|||||||
Reference in New Issue
Block a user