[Feature] Add structured output and effort support to Anthropic Messages API (#42396)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import anthropic
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
@@ -153,3 +155,39 @@ async def test_anthropic_tool_call_streaming(client: anthropic.AsyncAnthropic):
|
||||
|
||||
async for chunk in resp:
|
||||
print(chunk.model_dump_json())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_structured_output(client: anthropic.AsyncAnthropic):
|
||||
response = await client.messages.create(
|
||||
model="claude-3-7-sonnet-latest",
|
||||
max_tokens=1024,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Extract the key information from this email:"
|
||||
"John Smith (john@example.com) is interested in our "
|
||||
"Enterprise plan and wants to schedule a demo for next Tuesday at 2pm.",
|
||||
}
|
||||
],
|
||||
output_config={
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"email": {"type": "string"},
|
||||
"plan_interest": {"type": "string"},
|
||||
"demo_requested": {"type": "boolean"},
|
||||
},
|
||||
"required": ["name", "email", "plan_interest", "demo_requested"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
print(response.content[0].text)
|
||||
json_obj = json.loads(response.content[0].text)
|
||||
for key in ["name", "email", "plan_interest", "demo_requested"]:
|
||||
assert key in json_obj, f"Missing key in output: {key}"
|
||||
|
||||
@@ -100,6 +100,20 @@ class AnthropicToolChoice(BaseModel):
|
||||
return self
|
||||
|
||||
|
||||
class AnthropicJsonOutputFormat(BaseModel):
|
||||
"""JSON output format configuration"""
|
||||
|
||||
json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
|
||||
type: Literal["json_schema"] = "json_schema"
|
||||
|
||||
|
||||
class AnthropicOutputConfig(BaseModel):
|
||||
"""Configuration options for the model's output, such as the output format."""
|
||||
|
||||
effort: Literal["low", "medium", "high", "xhigh", "max"] | None = None
|
||||
format: AnthropicJsonOutputFormat | None = None
|
||||
|
||||
|
||||
class AnthropicMessagesRequest(BaseModel):
|
||||
"""Anthropic Messages API request"""
|
||||
|
||||
@@ -107,6 +121,7 @@ class AnthropicMessagesRequest(BaseModel):
|
||||
messages: list[AnthropicMessage]
|
||||
max_tokens: int
|
||||
metadata: dict[str, Any] | None = None
|
||||
output_config: AnthropicOutputConfig | None = None
|
||||
stop_sequences: list[str] | None = None
|
||||
stream: bool | None = False
|
||||
system: str | list[AnthropicContentBlock] | None = None
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicError,
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicMessagesResponse,
|
||||
AnthropicOutputConfig,
|
||||
AnthropicStreamEvent,
|
||||
AnthropicUsage,
|
||||
)
|
||||
@@ -39,6 +40,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
JsonSchemaResponseFormat,
|
||||
ResponseFormat,
|
||||
StreamOptions,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
@@ -128,6 +131,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
cls._convert_messages(anthropic_request.messages, openai_messages)
|
||||
req = cls._build_base_request(anthropic_request, openai_messages)
|
||||
cls._handle_streaming_options(req, anthropic_request)
|
||||
cls._handle_output_config(req, anthropic_request)
|
||||
cls._convert_tool_choice(anthropic_request, req)
|
||||
cls._convert_tools(anthropic_request, req)
|
||||
return req
|
||||
@@ -359,6 +363,27 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
chat_template_kwargs=anthropic_request.chat_template_kwargs,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _handle_output_config(
|
||||
cls,
|
||||
req: ChatCompletionRequest,
|
||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||
) -> None:
|
||||
"""Handle output configuration such as output format and effort"""
|
||||
if isinstance(anthropic_request, AnthropicCountTokensRequest):
|
||||
return
|
||||
output_config: AnthropicOutputConfig | None = anthropic_request.output_config
|
||||
if output_config and output_config.format and output_config.format.json_schema:
|
||||
req.response_format = ResponseFormat(
|
||||
type=output_config.format.type,
|
||||
json_schema=JsonSchemaResponseFormat(
|
||||
schema=output_config.format.json_schema,
|
||||
name=output_config.format.type,
|
||||
),
|
||||
)
|
||||
if output_config and output_config.effort is not None:
|
||||
req.reasoning_effort = output_config.effort
|
||||
|
||||
@classmethod
|
||||
def _handle_streaming_options(
|
||||
cls,
|
||||
|
||||
Reference in New Issue
Block a user