diff --git a/tests/entrypoints/anthropic/test_messages.py b/tests/entrypoints/anthropic/test_messages.py index 8f47351d67..c1f6858e83 100644 --- a/tests/entrypoints/anthropic/test_messages.py +++ b/tests/entrypoints/anthropic/test_messages.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json + import anthropic import pytest import pytest_asyncio @@ -153,3 +155,39 @@ async def test_anthropic_tool_call_streaming(client: anthropic.AsyncAnthropic): async for chunk in resp: print(chunk.model_dump_json()) + + +@pytest.mark.asyncio +async def test_anthropic_structured_output(client: anthropic.AsyncAnthropic): + response = await client.messages.create( + model="claude-3-7-sonnet-latest", + max_tokens=1024, + messages=[ + { + "role": "user", + "content": "Extract the key information from this email:" + "John Smith (john@example.com) is interested in our " + "Enterprise plan and wants to schedule a demo for next Tuesday at 2pm.", + } + ], + output_config={ + "format": { + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"}, + "plan_interest": {"type": "string"}, + "demo_requested": {"type": "boolean"}, + }, + "required": ["name", "email", "plan_interest", "demo_requested"], + "additionalProperties": False, + }, + } + }, + ) + print(response.content[0].text) + json_obj = json.loads(response.content[0].text) + for key in ["name", "email", "plan_interest", "demo_requested"]: + assert key in json_obj, f"Missing key in output: {key}" diff --git a/vllm/entrypoints/anthropic/protocol.py b/vllm/entrypoints/anthropic/protocol.py index f3c4dd7f3e..3ebc171173 100644 --- a/vllm/entrypoints/anthropic/protocol.py +++ b/vllm/entrypoints/anthropic/protocol.py @@ -100,6 +100,20 @@ class AnthropicToolChoice(BaseModel): return self +class AnthropicJsonOutputFormat(BaseModel): + """JSON output format configuration""" + + json_schema: dict[str, Any] | None = Field(default=None, alias="schema") + type: Literal["json_schema"] = "json_schema" + + +class AnthropicOutputConfig(BaseModel): + """Configuration options for the model's output, such as the output format.""" + + effort: Literal["low", "medium", "high", "xhigh", "max"] | None = None + format: AnthropicJsonOutputFormat | None = None + + class AnthropicMessagesRequest(BaseModel): """Anthropic Messages API request""" @@ -107,6 +121,7 @@ class AnthropicMessagesRequest(BaseModel): messages: list[AnthropicMessage] max_tokens: int metadata: dict[str, Any] | None = None + output_config: AnthropicOutputConfig | None = None stop_sequences: list[str] | None = None stream: bool | None = False system: str | list[AnthropicContentBlock] | None = None diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 867ee73948..915cee59f9 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -24,6 +24,7 @@ from vllm.entrypoints.anthropic.protocol import ( AnthropicError, AnthropicMessagesRequest, AnthropicMessagesResponse, + AnthropicOutputConfig, AnthropicStreamEvent, AnthropicUsage, ) @@ -39,6 +40,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.engine.protocol import ( ErrorResponse, + JsonSchemaResponseFormat, + ResponseFormat, StreamOptions, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels @@ -128,6 +131,7 @@ class AnthropicServingMessages(OpenAIServingChat): cls._convert_messages(anthropic_request.messages, openai_messages) req = cls._build_base_request(anthropic_request, openai_messages) cls._handle_streaming_options(req, anthropic_request) + cls._handle_output_config(req, anthropic_request) cls._convert_tool_choice(anthropic_request, req) cls._convert_tools(anthropic_request, req) return req @@ -359,6 +363,27 @@ class AnthropicServingMessages(OpenAIServingChat): chat_template_kwargs=anthropic_request.chat_template_kwargs, ) + @classmethod + def _handle_output_config( + cls, + req: ChatCompletionRequest, + anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, + ) -> None: + """Handle output configuration such as output format and effort""" + if isinstance(anthropic_request, AnthropicCountTokensRequest): + return + output_config: AnthropicOutputConfig | None = anthropic_request.output_config + if output_config and output_config.format and output_config.format.json_schema: + req.response_format = ResponseFormat( + type=output_config.format.type, + json_schema=JsonSchemaResponseFormat( + schema=output_config.format.json_schema, + name=output_config.format.type, + ), + ) + if output_config and output_config.effort is not None: + req.reasoning_effort = output_config.effort + @classmethod def _handle_streaming_options( cls,