From f25c2692fe3081bdf89b58208d6bb995b43e92f0 Mon Sep 17 00:00:00 2001 From: Hyunkyun Moon Date: Wed, 26 Aug 2026 12:33:59 +0900 Subject: [PATCH] [Frontend] Add `/v1/messages/render` endpoint for the Anthropic Messages API (#45803) Signed-off-by: HyunKyun Moon --- .../test_anthropic_messages_conversion.py | 4 +- .../scale_out/render/test_render.py | 112 ++++++++++++++++++ vllm/entrypoints/anthropic/serving.py | 6 +- .../scale_out/render/api_router.py | 25 ++++ vllm/entrypoints/scale_out/render/serving.py | 23 ++++ 5 files changed, 165 insertions(+), 5 deletions(-) diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index d1f178511d..2b4be6f736 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -3,7 +3,7 @@ """Unit tests for Anthropic-to-OpenAI request conversion. Tests the image source handling and tool_result content parsing in -AnthropicServingMessages._convert_anthropic_to_openai_request(). +AnthropicServingMessages.to_chat_completion_request(). Also covers extended-thinking edge cases such as ``redacted_thinking`` blocks echoed back by Anthropic clients, and streaming conversion in @@ -51,7 +51,7 @@ from vllm.entrypoints.serve.exception_handling.handlers.validation import ( ) from vllm.exceptions import VLLMValidationError -_convert = AnthropicServingMessages._convert_anthropic_to_openai_request +_convert = AnthropicServingMessages.to_chat_completion_request _img_url = AnthropicServingMessages._convert_image_source_to_url diff --git a/tests/entrypoints/scale_out/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py index ffd7f9f30a..444e66f898 100644 --- a/tests/entrypoints/scale_out/render/test_render.py +++ b/tests/entrypoints/scale_out/render/test_render.py @@ -493,3 +493,115 @@ async def test_chat_completion_render_assistant_tokens_mask_with_generation_tags assert detok_rest.status_code == 200 assert "Hi!" not in detok_rest.json()["prompt"] assert "Bye" in detok_rest.json()["prompt"] + + +@pytest.mark.asyncio +async def test_messages_render_basic(client): + """Test basic Anthropic Messages render endpoint.""" + response = await client.post( + "/v1/messages/render", + json={ + "model": MODEL_NAME, + "max_tokens": 16, + "messages": [{"role": "user", "content": "Render this Anthropic message."}], + }, + ) + + assert response.status_code == 200 + data = response.json() + + # Single GenerateRequest, like chat render. + assert isinstance(data, dict) + assert "token_ids" in data + assert "sampling_params" in data + assert "model" in data + assert data["model"] == MODEL_NAME + + token_ids = data["token_ids"] + assert isinstance(token_ids, list) + assert len(token_ids) > 0 + assert all(isinstance(tid, int) for tid in token_ids) + assert token_ids[0] == 1 # BOS + + +@pytest.mark.asyncio +async def test_messages_render_system_and_multi_turn(client): + """System field + multi-turn messages render to a single prompt.""" + response = await client.post( + "/v1/messages/render", + json={ + "model": MODEL_NAME, + "max_tokens": 16, + "system": "You are a helpful assistant.", + "messages": [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi! How can I help?"}, + {"role": "user", "content": "What is 2 + 2?"}, + ], + }, + ) + + assert response.status_code == 200 + data = response.json() + assert isinstance(data, dict) + assert len(data["token_ids"]) > 0 + assert data["token_ids"][0] == 1 # BOS + + +@pytest.mark.asyncio +async def test_messages_render_merges_inline_system(client): + """Inline system messages merge into the leading system block. + + Without a --chat-template arg the /v1/messages server path detects + merge_inline_system=True, so render must produce the same tokens as + the manually pre-merged request. + """ + inline = await client.post( + "/v1/messages/render", + json={ + "model": MODEL_NAME, + "max_tokens": 16, + "system": "You are a helpful assistant.", + "messages": [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi! How can I help?"}, + {"role": "system", "content": "Be brief."}, + {"role": "user", "content": "What is 2 + 2?"}, + ], + }, + ) + assert inline.status_code == 200 + + merged = await client.post( + "/v1/messages/render", + json={ + "model": MODEL_NAME, + "max_tokens": 16, + "system": "You are a helpful assistant.Be brief.", + "messages": [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi! How can I help?"}, + {"role": "user", "content": "What is 2 + 2?"}, + ], + }, + ) + assert merged.status_code == 200 + + assert inline.json()["token_ids"] == merged.json()["token_ids"] + + +@pytest.mark.asyncio +async def test_messages_render_error_invalid_model(client): + """Messages render with an invalid model returns an error.""" + response = await client.post( + "/v1/messages/render", + json={ + "model": "invalid-model-name", + "max_tokens": 16, + "messages": [{"role": "user", "content": "Hello"}], + }, + ) + + assert response.status_code == 404 + data = response.json() + assert "error" in data diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 9d6ad63dda..74e25f087d 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -191,7 +191,7 @@ class AnthropicServingMessages(OpenAIServingChat): return f"data:{media_type};base64,{data}" @classmethod - def _convert_anthropic_to_openai_request( + def to_chat_completion_request( cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, *, @@ -602,7 +602,7 @@ class AnthropicServingMessages(OpenAIServingChat): """ if logger.isEnabledFor(logging.DEBUG): logger.debug("Received messages request %s", request.model_dump_json()) - chat_req = self._convert_anthropic_to_openai_request( + chat_req = self.to_chat_completion_request( request, merge_inline_system=self._merge_inline_system, ) @@ -1023,7 +1023,7 @@ class AnthropicServingMessages(OpenAIServingChat): raw_request: Request | None = None, ) -> AnthropicCountTokensResponse | ErrorResponse: """Implements Anthropic's messages.count_tokens endpoint.""" - chat_req = self._convert_anthropic_to_openai_request( + chat_req = self.to_chat_completion_request( request, merge_inline_system=self._merge_inline_system, ) diff --git a/vllm/entrypoints/scale_out/render/api_router.py b/vllm/entrypoints/scale_out/render/api_router.py index d245286644..b54d2c8e0a 100644 --- a/vllm/entrypoints/scale_out/render/api_router.py +++ b/vllm/entrypoints/scale_out/render/api_router.py @@ -5,6 +5,7 @@ from http import HTTPStatus from fastapi import APIRouter, Depends, Request from fastapi.responses import JSONResponse +from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.engine.protocol import ErrorResponse @@ -49,6 +50,30 @@ async def render_chat_completion(request: ChatCompletionRequest, raw_request: Re return JSONResponse(content=result.model_dump()) +@router.post( + "/v1/messages/render", + dependencies=[Depends(validate_json_request)], + response_model=GenerateRequest, + responses={ + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, + HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, + }, +) +async def render_messages(request: AnthropicMessagesRequest, raw_request: Request): + handler = render(raw_request) + if handler is None: + raise NotImplementedError("The model does not support Messages Render API") + + result = await handler.render_messages_request(request) + + if isinstance(result, ErrorResponse): + return JSONResponse(content=result.model_dump(), status_code=result.error.code) + + return JSONResponse(content=result.model_dump()) + + @router.post( "/v1/completions/render", dependencies=[Depends(validate_json_request)], diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index 105bd75185..8b573e7a15 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import cast +from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest +from vllm.entrypoints.anthropic.serving import AnthropicServingMessages from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.engine.protocol import ErrorResponse @@ -51,6 +53,12 @@ class ServingRender(BaseServing): self.online_renderer = online_renderer + self._merge_inline_system = ( + AnthropicServingMessages._detect_merge_inline_system( + online_renderer.chat_template + ) + ) + self.default_sampling_params = ( online_renderer.model_config.get_diff_sampling_param() ) @@ -149,6 +157,21 @@ class ServingRender(BaseServing): token_offsets=engine_input.get("prompt_token_offsets"), ) + async def render_messages_request( + self, + request: AnthropicMessagesRequest, + ) -> GenerateRequest | ErrorResponse: + """Validate the model and preprocess an Anthropic Messages request. + + Converts the request to the OpenAI chat format using the same + conversion as the /v1/messages server path, then delegates to + render_chat_request so the rendered tokens match the server exactly. + """ + chat_req = AnthropicServingMessages.to_chat_completion_request( + request, merge_inline_system=self._merge_inline_system + ) + return await self.render_chat_request(chat_req) + async def render_completion_request( self, request: CompletionRequest,