[Bugfix][Frontend] Apply output_config and thinking in Anthropic count_tokens (#59180)
Signed-off-by: Kang Nahoon <ejrtks102020@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
1283362e76
commit
1995c0fcd0
@@ -385,7 +385,9 @@ class AnthropicCountTokensRequest(BaseModel):
|
||||
|
||||
model: str
|
||||
messages: list[AnthropicMessage]
|
||||
output_config: AnthropicOutputConfig | None = None
|
||||
system: str | list[AnthropicContentBlock] | None = None
|
||||
thinking: AnthropicThinkingConfig | None = None
|
||||
tool_choice: AnthropicToolChoice | None = None
|
||||
tools: list[AnthropicTool] | None = None
|
||||
|
||||
|
||||
@@ -633,8 +633,6 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
``display`` is intentionally ignored: suppressing reasoning would mark
|
||||
it ended for structured outputs and drop it from multi-turn history.
|
||||
"""
|
||||
if isinstance(anthropic_request, AnthropicCountTokensRequest):
|
||||
return
|
||||
thinking: AnthropicThinkingConfig | None = anthropic_request.thinking
|
||||
if thinking is None:
|
||||
return
|
||||
@@ -655,8 +653,6 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||
) -> None:
|
||||
"""Handle output configuration such as output format and effort."""
|
||||
if isinstance(anthropic_request, AnthropicCountTokensRequest):
|
||||
return
|
||||
output_config: AnthropicOutputConfig | None = anthropic_request.output_config
|
||||
if output_config and output_config.format and output_config.format.json_schema:
|
||||
req.response_format = ResponseFormat(
|
||||
@@ -1238,9 +1234,13 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
raw_request: Request | None = None,
|
||||
) -> AnthropicCountTokensResponse | ErrorResponse:
|
||||
"""Implements Anthropic's messages.count_tokens endpoint."""
|
||||
disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none"
|
||||
if request.thinking is not None and request.thinking.type == "disabled":
|
||||
disabled_thinking_effort = await self._get_disabled_thinking_effort()
|
||||
chat_req = self.to_chat_completion_request(
|
||||
request,
|
||||
merge_inline_system=self._merge_inline_system,
|
||||
disabled_thinking_effort=disabled_thinking_effort,
|
||||
)
|
||||
result = await self.render_chat_request(chat_req)
|
||||
if isinstance(result, ErrorResponse):
|
||||
|
||||
Reference in New Issue
Block a user