From 6a2a2bb02b563b83f946012959fd3927984d072a Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Mon, 7 Sep 2026 08:50:42 -0400 Subject: [PATCH] [Frontend] Add stateless /v1/responses/render endpoint (#50195) Signed-off-by: Francisco Javier Arceo Co-authored-by: OpenAI Codex --- docs/serving/online_serving/README.md | 2 + docs/serving/online_serving/renderer.md | 23 + docs/usage/security.md | 1 + tests/entrypoints/launchers/test_cli_args.py | 22 + .../openai/responses/test_mcp_tools.py | 24 + .../responses/test_serving_responses.py | 482 +++++++++++++++++- .../entrypoints/openai/test_render_parity.py | 3 +- .../scale_out/render/test_render.py | 188 +++++++ tests/entrypoints/scale_out/test_factories.py | 1 + vllm/entrypoints/generate/api_router.py | 39 +- .../launchers/api_server/app_state.py | 19 +- vllm/entrypoints/launchers/cli_args.py | 10 + .../entrypoints/launchers/render/app_state.py | 10 +- vllm/entrypoints/mcp/tool_server.py | 15 + vllm/entrypoints/openai/responses/serving.py | 328 +++--------- vllm/entrypoints/scale_out/factories.py | 1 + .../scale_out/render/api_router.py | 23 + vllm/entrypoints/scale_out/render/serving.py | 66 +++ vllm/renderers/online_renderer.py | 269 +++++++++- 19 files changed, 1228 insertions(+), 298 deletions(-) diff --git a/docs/serving/online_serving/README.md b/docs/serving/online_serving/README.md index 65c67e6a5b..5c93ca1787 100644 --- a/docs/serving/online_serving/README.md +++ b/docs/serving/online_serving/README.md @@ -141,6 +141,8 @@ For further details on renderer APIs, please refer to [this page](renderer.md). - Render completion requests - [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`) - Render chat completions +- [Responses Render API](renderer.md) (`/v1/responses/render`) + - Render self-contained Responses requests ### Derenderer APIs diff --git a/docs/serving/online_serving/renderer.md b/docs/serving/online_serving/renderer.md index 66a19775e0..1f5e893570 100644 --- a/docs/serving/online_serving/renderer.md +++ b/docs/serving/online_serving/renderer.md @@ -25,6 +25,29 @@ VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve - Render completion requests - [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`) - Render chat completions +- [Responses Render API](renderer.md) (`/v1/responses/render`) + - Render a self-contained Responses request + +The Responses render endpoint uses the same prompt construction as +`/v1/responses` and returns one token-in `GenerateRequest`. It is stateless: +inline history is supported, but `previous_response_id` is not. Callers must +resolve stored response state and include the resulting history in the request +before rendering. + +For multimodal requests, the `GenerateRequest` contains the model-processed +multimodal payload, which can be substantially larger than the source image or +video. The caller must forward that payload unchanged to the generation service +and provision transport limits and memory accordingly. + +```bash +curl http://localhost:8000/v1/responses/render \ + -H "Content-Type: application/json" \ + -d '{ + "model": "meta-llama/Llama-3.1-8B-Instruct", + "input": "Explain prefix caching in one sentence.", + "max_output_tokens": 32 + }' +``` For the post processing counterpart that turns generated token IDs back into OpenAI compatible responses, see the [Derenderer APIs](derenderer.md). diff --git a/docs/usage/security.md b/docs/usage/security.md index 4314d39ce8..ba730bcb20 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -170,6 +170,7 @@ When `--api-key` is configured, the following `/v1` endpoints require Bearer tok - `/v1/messages/render` - Render Anthropic-compatible messages (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) - `/v1/messages/count_tokens` - Count tokens for Anthropic messages - `/v1/responses` - Create a response +- `/v1/responses/render` - Render a self-contained response request (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) - `/v1/responses/{response_id}` - Retrieve a response - `/v1/responses/{response_id}/cancel` - Cancel a response - `/v1/score` - Scoring API diff --git a/tests/entrypoints/launchers/test_cli_args.py b/tests/entrypoints/launchers/test_cli_args.py index a9c714ada0..e84abe5d26 100644 --- a/tests/entrypoints/launchers/test_cli_args.py +++ b/tests/entrypoints/launchers/test_cli_args.py @@ -2,9 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json +from types import SimpleNamespace import pytest +import vllm.entrypoints.launchers.cli_args as cli_args_module from tests.utils import VLLM_PATH from vllm.entrypoints.launchers.cli_args import ( make_arg_parser, @@ -332,6 +334,26 @@ def test_default_chat_template_kwargs_default_none(serve_parser): assert args.default_chat_template_kwargs is None +@pytest.mark.parametrize( + "default_kwargs,cohere_format,expected", + [ + (None, "cmd3", {"cohere_format": "cmd3"}), + ( + {"cohere_format": "cmd4", "custom": True}, + "cmd3", + {"cohere_format": "cmd4", "custom": True}, + ), + ], +) +def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, expected): + args = SimpleNamespace( + default_chat_template_kwargs=default_kwargs, + cohere_format=cohere_format, + ) + + assert cli_args_module.resolve_default_chat_template_kwargs(args) == expected + + def test_default_chat_template_kwargs_invalid_json(serve_parser): """Ensure invalid JSON raises an error""" with pytest.raises(SystemExit): diff --git a/tests/entrypoints/openai/responses/test_mcp_tools.py b/tests/entrypoints/openai/responses/test_mcp_tools.py index 7a6e432e66..2aad5b790e 100644 --- a/tests/entrypoints/openai/responses/test_mcp_tools.py +++ b/tests/entrypoints/openai/responses/test_mcp_tools.py @@ -4,14 +4,19 @@ from __future__ import annotations +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + import pytest import pytest_asyncio from openai import OpenAI from openai_harmony import Message, ToolDescription, ToolNamespaceConfig +import vllm.entrypoints.mcp.tool_server as tool_server_module from tests.utils import RemoteOpenAIServer from vllm.entrypoints.mcp.tool_server import ( MCPToolServer, + init_tool_server, post_process_tools_description, ) @@ -39,6 +44,25 @@ _PYTHON_TOOL_INSTRUCTION = ( ) +@pytest.mark.asyncio +async def test_init_tool_server_without_configuration(): + args = SimpleNamespace(tool_server=None) + + assert await init_tool_server(args) is None + + +@pytest.mark.asyncio +async def test_init_tool_server_with_mcp_url(monkeypatch: pytest.MonkeyPatch): + server = MagicMock() + server.add_tool_server = AsyncMock() + monkeypatch.setattr(tool_server_module, "MCPToolServer", lambda: server) + + result = await init_tool_server(SimpleNamespace(tool_server="http://mcp.test")) + + assert result is server + server.add_tool_server.assert_awaited_once_with("http://mcp.test") + + @pytest.mark.skip_global_cleanup class TestMCPToolServerUnit: """Test MCP tool description processing and filtering. diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 723c59cac5..041e517e29 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -2,12 +2,16 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from contextlib import AsyncExitStack -from unittest.mock import MagicMock +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock import pytest import pytest_asyncio from openai.types.responses import ( + ResponseFunctionToolCall, ResponseOutputItemDoneEvent, + ResponseOutputMessage, + ResponseOutputText, ResponseReasoningItem, ResponseReasoningTextDeltaEvent, ResponseReasoningTextDoneEvent, @@ -23,6 +27,8 @@ from openai.types.responses.tool import ( Mcp, Tool, ) +from openai_harmony import Message as OpenAIHarmonyMessage +from openai_harmony import Role import vllm.envs as envs from vllm.entrypoints.generate.base.protocol import ( @@ -32,7 +38,11 @@ from vllm.entrypoints.generate.base.protocol import ( RequestResponseMetadata, ) from vllm.entrypoints.mcp.tool_server import ToolServer -from vllm.entrypoints.openai.responses.context import ConversationContext, SimpleContext +from vllm.entrypoints.openai.responses.context import ( + ConversationContext, + HarmonyContext, + SimpleContext, +) from vllm.entrypoints.openai.responses.protocol import ( ResponseCreatedEvent, ResponseRawMessageAndToken, @@ -42,7 +52,6 @@ from vllm.entrypoints.openai.responses.protocol import ( ) from vllm.entrypoints.openai.responses.serving import ( OpenAIServingResponses, - _extract_allowed_tools_from_mcp_requests, extract_tool_types, ) from vllm.entrypoints.openai.responses.streaming_events import ( @@ -52,8 +61,23 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.inputs import tokens_input from vllm.outputs import CompletionOutput, RequestOutput from vllm.parser.harmony import Segment +from vllm.renderers import TokenizeParams +from vllm.renderers.online_renderer import ( + OnlineRenderer, + _extract_allowed_tools_from_mcp_requests, +) from vllm.sampling_params import SamplingParams +pytestmark = pytest.mark.skip_global_cleanup + + +def _new_online_renderer() -> OnlineRenderer: + renderer = OnlineRenderer.__new__(OnlineRenderer) + renderer.exclude_tools_when_tool_choice_none = False + renderer.parser = None + renderer.trust_request_chat_template = True + return renderer + class MockConversationContext(ConversationContext): """Mock conversation context for testing""" @@ -205,6 +229,325 @@ def test_response_created_event_uses_public_json_schema_alias() -> None: assert event.response.text.format.model_dump(by_alias=True)["schema"] == schema +@pytest.mark.asyncio +async def test_online_renderer_renders_non_harmony_responses_with_explicit_history(): + renderer = _new_online_renderer() + renderer.use_harmony = False + renderer.chat_template = "server-template" + renderer.chat_template_content_format = "string" + renderer.default_chat_template_kwargs = {"server_default": "kept"} + renderer.parser = None + renderer.preprocess_chat = AsyncMock( + return_value=( + [], + [tokens_input([11, 12, 13], cache_salt="request-salt")], + ) + ) + previous_messages = [ + {"role": "system", "content": "old instructions"}, + {"role": "user", "content": "first"}, + ] + previous_outputs = [ + ResponseOutputMessage( + id="msg_1", + content=[ + ResponseOutputText( + annotations=[], + text="prior answer", + type="output_text", + ) + ], + role="assistant", + status="completed", + type="message", + ) + ] + request = ResponsesRequest( + input="next", + instructions="new instructions", + tools=[], + cache_salt="request-salt", + chat_template_kwargs={"request_default": "kept"}, + ) + + result = await renderer.render_responses( + request, + previous_messages=previous_messages, + previous_response_outputs=previous_outputs, + ) + + assert not isinstance(result, ErrorResponse) + assert result.messages == [ + {"role": "system", "content": "new instructions"}, + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "prior answer"}, + {"role": "user", "content": "next"}, + ] + assert result.engine_input["prompt_token_ids"] == [11, 12, 13] + assert result.engine_input["cache_salt"] == "request-salt" + assert previous_messages[0]["content"] == "old instructions" + assert previous_outputs[0].content[0].text == "prior answer" + + preprocess_kwargs = renderer.preprocess_chat.call_args.kwargs + assert preprocess_kwargs["default_template"] == "server-template" + assert preprocess_kwargs["default_template_content_format"] == "string" + assert preprocess_kwargs["default_template_kwargs"]["server_default"] == "kept" + assert preprocess_kwargs["default_template_kwargs"]["request_default"] == "kept" + + +@pytest.mark.asyncio +async def test_online_renderer_rejects_untrusted_responses_chat_template(): + renderer = _new_online_renderer() + renderer.use_harmony = False + renderer.chat_template = "server-template" + renderer.chat_template_content_format = "string" + renderer.default_chat_template_kwargs = {} + renderer.exclude_tools_when_tool_choice_none = False + renderer.trust_request_chat_template = False + renderer.parser = None + renderer.preprocess_chat = AsyncMock( + return_value=([], [tokens_input([1])]), + ) + request = ResponsesRequest( + input="hello", + chat_template_kwargs={"chat_template": "{{ messages }}"}, + ) + + result = await renderer.render_responses(request) + + assert isinstance(result, ErrorResponse) + assert result.error.code == 400 + assert "untrusted chat template" in result.error.message.lower() + renderer.preprocess_chat.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_online_renderer_rejects_harmony_history_for_non_harmony_model(): + renderer = _new_online_renderer() + renderer.use_harmony = False + request = ResponsesRequest(input="next") + + result = await renderer.render_responses( + request, + previous_messages=[ + OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first") + ], + ) + + assert isinstance(result, ErrorResponse) + assert result.error.param == "previous_response_id" + + +@pytest.mark.asyncio +async def test_online_renderer_rejects_chat_history_for_harmony_model(): + renderer = _new_online_renderer() + renderer.use_harmony = True + request = ResponsesRequest(input="next") + + result = await renderer.render_responses( + request, + previous_messages=[{"role": "user", "content": "first"}], + ) + + assert isinstance(result, ErrorResponse) + assert result.error.param == "previous_response_id" + + +@pytest.mark.asyncio +async def test_online_renderer_adjusts_harmony_tool_choice( + monkeypatch: pytest.MonkeyPatch, +): + class StubParser: + def __init__(self, tokenizer, tools, *, model_config): + pass + + def adjust_request(self, request): + request.cache_salt = "adjusted" + return request + + renderer = _new_online_renderer() + renderer.use_harmony = True + renderer.parser = StubParser + renderer.model_config = SimpleNamespace(max_model_len=100) + tokenizer = SimpleNamespace(truncation_side="left") + renderer.renderer = SimpleNamespace( + tokenizer=tokenizer, + get_tokenizer=lambda: tokenizer, + ) + monkeypatch.setattr( + "vllm.renderers.online_renderer.render_for_completion", + lambda messages: [1], + ) + request = ResponsesRequest( + input="next", + tools=[ + { + "type": "function", + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + } + ], + tool_choice={"type": "function", "name": "lookup"}, + ) + + result = await renderer.render_responses(request) + + assert not isinstance(result, ErrorResponse) + assert result.engine_input["cache_salt"] == "adjusted" + + +@pytest.mark.asyncio +async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt( + monkeypatch: pytest.MonkeyPatch, +): + renderer = _new_online_renderer() + renderer.use_harmony = True + renderer.model_config = SimpleNamespace(max_model_len=5) + renderer.renderer = SimpleNamespace( + tokenizer=SimpleNamespace(truncation_side="left") + ) + monkeypatch.setattr( + "vllm.renderers.online_renderer.render_for_completion", + lambda messages: [1, 2, 3, 4, 5, 6], + ) + request = ResponsesRequest( + input="hello", + max_output_tokens=2, + truncation="auto", + ) + + result = await renderer.render_responses(request) + + assert not isinstance(result, ErrorResponse) + assert result.engine_input["prompt_token_ids"] == [4, 5, 6] + + +@pytest.mark.asyncio +async def test_online_renderer_renders_harmony_continuation_with_call_linkage( + monkeypatch: pytest.MonkeyPatch, +): + renderer = _new_online_renderer() + renderer.use_harmony = True + renderer.model_config = SimpleNamespace(max_model_len=100) + renderer.renderer = SimpleNamespace( + tokenizer=SimpleNamespace(truncation_side="left") + ) + monkeypatch.setattr( + "vllm.renderers.online_renderer.render_for_completion", + lambda messages: [1], + ) + previous_messages = [ + OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first"), + ] + previous_outputs = [ + ResponseFunctionToolCall( + arguments='{"city":"Paris"}', + call_id="call_1", + name="lookup_weather", + type="function_call", + ) + ] + request = ResponsesRequest( + input=[ + { + "type": "function_call_output", + "call_id": "call_1", + "output": "sunny", + } + ], + instructions="replacement instructions are ignored on continuations", + reasoning={"effort": "high"}, + tools=[ + { + "type": "function", + "name": "lookup_weather", + "parameters": {"type": "object", "properties": {}}, + } + ], + cache_salt="request-salt", + ) + + result = await renderer.render_responses( + request, + previous_messages=previous_messages, + previous_response_outputs=previous_outputs, + ) + + assert not isinstance(result, ErrorResponse) + assert result.messages[0] is previous_messages[0] + tool_output = result.messages[-1] + assert tool_output.author.role == "tool" + assert tool_output.author.name == "functions.lookup_weather" + assert tool_output.channel == "commentary" + assert tool_output.recipient == "assistant" + assert result.engine_input["cache_salt"] == "request-salt" + assert "arrival_time" in result.engine_input + assert previous_messages == [ + OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first"), + ] + assert previous_outputs[0].call_id == "call_1" + + +@pytest.mark.asyncio +async def test_online_renderer_returns_typed_error_for_bad_harmony_call_reference(): + renderer = _new_online_renderer() + renderer.use_harmony = True + request = ResponsesRequest( + input=[ + { + "type": "function_call_output", + "call_id": "missing", + "output": "orphaned", + } + ], + tools=[], + ) + + result = await renderer.render_responses(request) + + assert isinstance(result, ErrorResponse) + assert result.error.type == "invalid_request_error" + assert result.error.param == "input" + assert "No call message found for missing" in result.error.message + + +@pytest.mark.asyncio +async def test_online_renderer_preserves_missing_harmony_builtin_tool_behavior( + monkeypatch: pytest.MonkeyPatch, +): + renderer = _new_online_renderer() + renderer.use_harmony = True + renderer.model_config = SimpleNamespace(max_model_len=100) + renderer.renderer = SimpleNamespace( + tokenizer=SimpleNamespace(truncation_side="left") + ) + monkeypatch.setattr( + "vllm.renderers.online_renderer.render_for_completion", + lambda messages: [1], + ) + request = ResponsesRequest( + input="search", + tools=[{"type": "web_search_preview"}], + ) + + result = await renderer.render_responses(request, tool_server=None) + + assert not isinstance(result, ErrorResponse) + + +@pytest.mark.parametrize( + "engine_inputs", + [[], [tokens_input([1]), tokens_input([2])]], +) +def test_responses_render_result_rejects_non_single_prompt(engine_inputs): + renderer = _new_online_renderer() + + result = renderer._responses_render_result([], engine_inputs) + + assert isinstance(result, ErrorResponse) + assert f"got {len(engine_inputs)}" in result.error.message + + class TestInitializeToolSessions: """Test class for _initialize_tool_sessions method""" @@ -240,6 +583,139 @@ class TestInitializeToolSessions: return instance + @pytest.mark.asyncio + async def test_stateful_harmony_delegates_named_tool_choice_to_renderer( + self, + serving_responses_instance, + ): + serving_responses_instance.use_harmony = True + serving_responses_instance.online_renderer.render_responses = AsyncMock( + return_value=SimpleNamespace( + messages=[], + engine_input=tokens_input([1]), + ) + ) + request = ResponsesRequest( + input="hello", + tool_choice={"type": "function", "name": "lookup"}, + tools=[ + { + "type": "function", + "name": "lookup", + "parameters": {"type": "object", "properties": {}}, + } + ], + ) + + result = await serving_responses_instance._render_resolved_response_inputs( + request, + None, + ) + + assert result.engine_input["prompt_token_ids"] == [1] + serving_responses_instance.online_renderer.render_responses.assert_awaited_once() + + @pytest.mark.asyncio + async def test_render_next_turn_reuses_shared_responses_renderer( + self, serving_responses_instance + ): + serving_responses_instance.online_renderer.render_responses = AsyncMock( + return_value=SimpleNamespace( + messages=[], + engine_input=tokens_input([8, 9], cache_salt="request-salt"), + ) + ) + serving_responses_instance.online_renderer.preprocess_chat = AsyncMock( + return_value=([], [tokens_input([1])]) + ) + request = ResponsesRequest( + input="first", + instructions="initial instructions", + cache_salt="request-salt", + ) + turn_messages = [ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "tool call"}, + {"role": "tool", "content": "tool output"}, + ] + + engine_inputs = await serving_responses_instance._render_next_turn( + request, + turn_messages, + ) + + assert engine_inputs[0]["prompt_token_ids"] == [8, 9] + render_request = ( + serving_responses_instance.online_renderer.render_responses.call_args.args[ + 0 + ] + ) + assert render_request.input == turn_messages + assert render_request.instructions is None + assert render_request.cache_salt == "request-salt" + + @pytest.mark.asyncio + async def test_harmony_tool_followup_preserves_render_params( + self, serving_responses_instance, monkeypatch: pytest.MonkeyPatch + ): + class ToolCallingHarmonyContext(HarmonyContext): + def __init__(self): + self._messages = [] + self._needs_tool_call = True + + def append_output(self, output) -> None: + pass + + def need_builtin_tool_call(self) -> bool: + return self._needs_tool_call + + async def call_tool(self): + self._needs_tool_call = False + return [] + + def append_tool_output(self, output) -> None: + self._messages.extend(output) + + async def generate_output(): + yield MagicMock() + + context = ToolCallingHarmonyContext() + serving_responses_instance.engine_client.generate.side_effect = ( + lambda *args, **kwargs: generate_output() + ) + monkeypatch.setattr( + "vllm.renderers.online_renderer.render_for_completion", + lambda messages: [1, 2, 3, 4], + ) + online_renderer = serving_responses_instance.online_renderer + online_renderer.renderer = SimpleNamespace( + tokenizer=SimpleNamespace(truncation_side="left") + ) + serving_responses_instance.online_renderer.render_responses_harmony_messages = ( + OnlineRenderer.render_responses_harmony_messages.__get__(online_renderer) + ) + serving_responses_instance._extract_prompt_len = MagicMock(return_value=2) + + async for _ in serving_responses_instance._generate_with_builtin_tools( + request_id="req", + engine_input=tokens_input([1], cache_salt="request-salt"), + sampling_params=MagicMock(), + context=context, + tok_params=TokenizeParams( + max_total_tokens=3, + max_output_tokens=1, + truncate_prompt_tokens=-1, + ), + ): + pass + + assert serving_responses_instance.engine_client.generate.call_count == 2 + followup_engine_input = ( + serving_responses_instance.engine_client.generate.call_args_list[1].args[0] + ) + assert followup_engine_input.get("cache_salt") == "request-salt" + assert followup_engine_input["prompt_token_ids"] == [3, 4] + @pytest.mark.asyncio async def test_initialize_tool_sessions( self, serving_responses_instance, mock_context, mock_exit_stack diff --git a/tests/entrypoints/openai/test_render_parity.py b/tests/entrypoints/openai/test_render_parity.py index 18db2ceb32..ef7e68ae59 100644 --- a/tests/entrypoints/openai/test_render_parity.py +++ b/tests/entrypoints/openai/test_render_parity.py @@ -151,7 +151,8 @@ async def _capture_responses( request: ResponsesRequest, ) -> CapturedRenderInputs: capture = RenderCapture(serving.online_renderer) - await serving._make_request(request, prev_response=None) + result = await serving.online_renderer.render_responses(request) + assert not isinstance(result, ErrorResponse), result return capture.take() diff --git a/tests/entrypoints/scale_out/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py index 40e338c7f0..6aaf119f16 100644 --- a/tests/entrypoints/scale_out/render/test_render.py +++ b/tests/entrypoints/scale_out/render/test_render.py @@ -3,15 +3,138 @@ """Tests for the /render endpoints that expose prompt preprocessing.""" +from http import HTTPStatus +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + import httpx import pytest import pytest_asyncio from tests.utils import RemoteLaunchRenderServer +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.entrypoints.scale_out.render.api_router import router +from vllm.entrypoints.scale_out.render.serving import ServingRender +from vllm.entrypoints.serve.engine.protocol import ErrorResponse +from vllm.renderers.online_renderer import OnlineRenderer MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" +def _build_responses_serving_render() -> ServingRender: + serving = ServingRender.__new__(ServingRender) + serving.model_config = SimpleNamespace( + max_model_len=100, + is_encoder_decoder=False, + ) + serving.default_sampling_params = {} + serving.override_max_tokens = None + serving.tool_server = MagicMock() + serving.online_renderer = MagicMock() + serving.online_renderer.create_error_response = ( + OnlineRenderer.create_error_response.__get__(serving.online_renderer) + ) + serving.online_renderer.validate_chat_template = ( + OnlineRenderer.validate_chat_template.__get__(serving.online_renderer) + ) + serving.online_renderer.trust_request_chat_template = True + serving._check_model = AsyncMock(return_value=None) + return serving + + +@pytest.mark.skip_global_cleanup +def test_responses_render_route_is_registered(): + assert any(route.path == "/v1/responses/render" for route in router.routes) + + +@pytest.mark.skip_global_cleanup +def test_responses_render_route_documents_not_implemented(): + route = next( + route for route in router.routes if route.path == "/v1/responses/render" + ) + + assert HTTPStatus.NOT_IMPLEMENTED.value in route.responses + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_render_responses_returns_generate_request_without_stored_state(): + serving = _build_responses_serving_render() + serving.online_renderer.render_responses = AsyncMock( + return_value=MagicMock( + messages=[{"role": "user", "content": "Test prompt"}], + engine_input={"prompt_token_ids": [7, 8, 9]}, + ) + ) + request = ResponsesRequest( + model=MODEL_NAME, + input="Test prompt", + max_output_tokens=12, + stream=True, + cache_salt="request-salt", + priority=3, + kv_transfer_params={"do_remote_prefill": True}, + ec_transfer_params={"remote": True}, + ) + + response = await serving.render_responses_request(request) + + assert response.token_ids == [7, 8, 9] + assert response.request_id == request.request_id + assert response.sampling_params.max_tokens == 12 + assert response.model == MODEL_NAME + assert response.stream is True + assert response.cache_salt == "request-salt" + assert response.priority == 3 + assert response.kv_transfer_params == {"do_remote_prefill": True} + assert response.ec_transfer_params == {"remote": True} + serving.online_renderer.render_responses.assert_awaited_once_with( + request, + previous_messages=None, + previous_response_outputs=None, + tool_server=serving.tool_server, + skip_mm_cache=True, + ) + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_render_responses_rejects_previous_response_id(): + serving = _build_responses_serving_render() + serving.online_renderer.render_responses = AsyncMock() + request = ResponsesRequest( + model=MODEL_NAME, + input="Test prompt", + previous_response_id="resp_previous", + ) + + response = await serving.render_responses_request(request) + + assert isinstance(response, ErrorResponse) + assert response.error.code == 400 + assert response.error.param == "previous_response_id" + serving.online_renderer.render_responses.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_render_responses_rejects_empty_token_ids(): + serving = _build_responses_serving_render() + serving.online_renderer.render_responses = AsyncMock( + return_value=MagicMock( + messages=[], + engine_input={"prompt_token_ids": []}, + ) + ) + + response = await serving.render_responses_request( + ResponsesRequest(model=MODEL_NAME, input="Test prompt") + ) + + assert isinstance(response, ErrorResponse) + assert response.error.message == "No token_ids rendered" + + @pytest.fixture(scope="module") def server(): args: list[str] = ["--trust-request-chat-template"] @@ -28,6 +151,71 @@ async def client(server): yield http_client +@pytest.mark.asyncio +async def test_responses_render_basic(client): + response = await client.post( + "/v1/responses/render", + json={ + "model": MODEL_NAME, + "input": "When should a Responses handler return an empty string?", + "max_output_tokens": 7, + }, + ) + + assert response.status_code == 200 + data = response.json() + assert data["model"] == MODEL_NAME + assert data["request_id"].startswith("resp_") + assert data["sampling_params"]["max_tokens"] == 7 + assert data["token_ids"] + + +@pytest.mark.asyncio +async def test_responses_render_includes_prior_input_items(client): + current_turn = { + "role": "user", + "content": "Which color should I remember?", + } + current_turn_response = await client.post( + "/v1/responses/render", + json={"model": MODEL_NAME, "input": [current_turn]}, + ) + multi_turn_response = await client.post( + "/v1/responses/render", + json={ + "model": MODEL_NAME, + "input": [ + {"role": "user", "content": "Remember the color cobalt."}, + {"role": "assistant", "content": "I will remember cobalt."}, + current_turn, + ], + }, + ) + + assert current_turn_response.status_code == 200 + assert multi_turn_response.status_code == 200 + current_turn_token_ids = current_turn_response.json()["token_ids"] + multi_turn_token_ids = multi_turn_response.json()["token_ids"] + assert len(multi_turn_token_ids) > len(current_turn_token_ids) + + +@pytest.mark.asyncio +async def test_responses_render_rejects_previous_response_id_over_http(client): + response = await client.post( + "/v1/responses/render", + json={ + "model": MODEL_NAME, + "input": "Continue the previous response.", + "previous_response_id": "resp_previous", + }, + ) + + assert response.status_code == 400 + error = response.json()["error"] + assert error["type"] == "invalid_request_error" + assert error["param"] == "previous_response_id" + + @pytest.mark.asyncio async def test_completion_render_basic(client): """Test basic completion render endpoint.""" diff --git a/tests/entrypoints/scale_out/test_factories.py b/tests/entrypoints/scale_out/test_factories.py index 76b066ccf1..9811bca8a2 100644 --- a/tests/entrypoints/scale_out/test_factories.py +++ b/tests/entrypoints/scale_out/test_factories.py @@ -15,6 +15,7 @@ RENDER_PATHS = { "/v1/completions/render", "/v1/completions/derender", "/v1/messages/render", + "/v1/responses/render", } GENERATE_PATH = "/inference/v1/generate" SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH} diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 92c564ac31..12c70e37d5 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from fastapi import FastAPI @@ -60,6 +60,7 @@ async def init_generate_state( args: "Namespace", request_logger: RequestLogger | None, supported_tasks: tuple["SupportedTask", ...], + default_chat_template_kwargs: dict[str, Any], ): from vllm.entrypoints.anthropic.serving import AnthropicServingMessages from vllm.entrypoints.chat_utils import load_chat_template @@ -79,12 +80,6 @@ async def init_generate_state( CohereServingChatV2 = None # type: ignore[assignment,misc] else: CohereServingChatV2 = None # type: ignore[assignment,misc] - - from vllm.entrypoints.mcp.tool_server import ( - DemoToolServer, - MCPToolServer, - ToolServer, - ) from vllm.entrypoints.openai.chat_completion.batch_serving import ( OpenAIServingChatBatch, ) @@ -100,33 +95,11 @@ async def init_generate_state( getattr(args, "fingerprint_value", None), ) - if args.tool_server == "demo": - tool_server: ToolServer | None = DemoToolServer() - assert isinstance(tool_server, DemoToolServer) - await tool_server.init_and_validate() - elif args.tool_server: - tool_server = MCPToolServer() - await tool_server.add_tool_server(args.tool_server) - else: - tool_server = None resolved_chat_template = load_chat_template(args.chat_template) - # Fold the dedicated ``--cohere-format`` CLI flag into the renderer's - # default chat-template kwargs. The cohere renderer reads - # ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4 - # rendering; making this a first-class flag keeps the right format - # discoverable for ``vllm serve --tokenizer-mode cohere`` users - # without forcing them to hand-construct a JSON dict for - # ``--default-chat-template-kwargs``. Per-request overrides still - # take precedence (see ``merge_kwargs`` in - # ``ChatCompletionRequest.build_chat_params``). - default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {}) - if getattr(args, "cohere_format", None): - default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format) - - # Render endpoints are always backed by OnlineRenderer so that - # /v1/chat/completions/render and /v1/completions/render work on both - # generate-mode and render-only servers. Created in init_app_state. + # Render endpoints are always backed by OnlineRenderer so that chat, + # completion, and Responses rendering work on both generate-mode and + # render-only servers. Created in init_app_state. state.openai_serving_responses = ( OpenAIServingResponses( @@ -139,7 +112,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, - tool_server=tool_server, + tool_server=state.tool_server, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, diff --git a/vllm/entrypoints/launchers/api_server/app_state.py b/vllm/entrypoints/launchers/api_server/app_state.py index c896db4392..09bb0ec630 100644 --- a/vllm/entrypoints/launchers/api_server/app_state.py +++ b/vllm/entrypoints/launchers/api_server/app_state.py @@ -8,6 +8,8 @@ from starlette.datastructures import State from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import load_chat_template +from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs +from vllm.entrypoints.mcp.tool_server import init_tool_server from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.tokenize.serving import ServingTokenization @@ -63,6 +65,10 @@ async def init_app_state( state.vllm_config = vllm_config state.args = args resolved_chat_template = load_chat_template(args.chat_template) + default_chat_template_kwargs = resolve_default_chat_template_kwargs(args) + state.tool_server = ( + await init_tool_server(args) if "generate" in supported_tasks else None + ) # Merge default_mm_loras into the static lora_modules default_mm_loras = ( @@ -90,7 +96,7 @@ async def init_app_state( exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, reasoning_parser=args.structured_outputs_config.reasoning_parser, - default_chat_template_kwargs=args.default_chat_template_kwargs, + default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, ) state.online_renderer.warmup() @@ -106,7 +112,7 @@ async def init_app_state( exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, reasoning_parser=args.structured_outputs_config.reasoning_parser, - default_chat_template_kwargs=args.default_chat_template_kwargs, + default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, ) @@ -116,7 +122,7 @@ async def init_app_state( request_logger=request_logger, chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, - default_chat_template_kwargs=args.default_chat_template_kwargs, + default_chat_template_kwargs=default_chat_template_kwargs, trust_request_chat_template=args.trust_request_chat_template, ) @@ -124,7 +130,12 @@ async def init_app_state( from vllm.entrypoints.generate.api_router import init_generate_state await init_generate_state( - engine_client, state, args, request_logger, supported_tasks + engine_client, + state, + args, + request_logger, + supported_tasks, + default_chat_template_kwargs, ) from vllm.entrypoints.scale_out.factories import init_scale_out_state diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 615c65d7fb..c0dd64164e 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -370,6 +370,16 @@ class FrontendArgs(BaseFrontendArgs): return frontend_kwargs +def resolve_default_chat_template_kwargs( + args: argparse.Namespace, +) -> dict[str, Any]: + """Resolve renderer defaults, including the dedicated Cohere format flag.""" + kwargs = dict(args.default_chat_template_kwargs or {}) + if getattr(args, "cohere_format", None): + kwargs.setdefault("cohere_format", args.cohere_format) + return kwargs + + def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: """Create the CLI argument parser used by the OpenAI API server. diff --git a/vllm/entrypoints/launchers/render/app_state.py b/vllm/entrypoints/launchers/render/app_state.py index 2b0b030b4e..6fb4123c9b 100644 --- a/vllm/entrypoints/launchers/render/app_state.py +++ b/vllm/entrypoints/launchers/render/app_state.py @@ -6,6 +6,8 @@ from starlette.datastructures import State from vllm.config import VllmConfig from vllm.entrypoints.chat_utils import load_chat_template +from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs +from vllm.entrypoints.mcp.tool_server import init_tool_server from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry from vllm.entrypoints.scale_out.factories import init_render_state @@ -46,6 +48,8 @@ async def init_render_app_state( renderer = renderer_from_config(vllm_config) resolved_chat_template = load_chat_template(args.chat_template) + default_chat_template_kwargs = resolve_default_chat_template_kwargs(args) + state.tool_server = await init_tool_server(args) state.online_renderer = OnlineRenderer( model_config=vllm_config.model_config, @@ -58,7 +62,7 @@ async def init_render_app_state( exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, reasoning_parser=args.reasoning_parser, - default_chat_template_kwargs=args.default_chat_template_kwargs, + default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, ) state.online_renderer.warmup() @@ -74,7 +78,7 @@ async def init_render_app_state( exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, reasoning_parser=args.reasoning_parser, - default_chat_template_kwargs=args.default_chat_template_kwargs, + default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, ) @@ -85,7 +89,7 @@ async def init_render_app_state( request_logger=request_logger, chat_template=resolved_chat_template, chat_template_content_format=args.chat_template_content_format, - default_chat_template_kwargs=args.default_chat_template_kwargs, + default_chat_template_kwargs=default_chat_template_kwargs, trust_request_chat_template=args.trust_request_chat_template, ) diff --git a/vllm/entrypoints/mcp/tool_server.py b/vllm/entrypoints/mcp/tool_server.py index 46725e251c..ed519ce4a5 100644 --- a/vllm/entrypoints/mcp/tool_server.py +++ b/vllm/entrypoints/mcp/tool_server.py @@ -12,6 +12,8 @@ from vllm.logger import init_logger logger = init_logger(__name__) if TYPE_CHECKING: + from argparse import Namespace + from mcp.types import ListToolsResult @@ -232,3 +234,16 @@ class DemoToolServer(ToolServer): if tool_name not in self.tools: raise KeyError(f"Tool '{tool_name}' is not supported") yield self.tools[tool_name] + + +async def init_tool_server(args: "Namespace") -> ToolServer | None: + tool_server = getattr(args, "tool_server", None) + if tool_server == "demo": + demo_server = DemoToolServer() + await demo_server.init_and_validate() + return demo_server + if tool_server: + mcp_server = MCPToolServer() + await mcp_server.add_tool_server(tool_server) + return mcp_server + return None diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 7053aba292..2c61205e22 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -6,13 +6,11 @@ import time from collections import deque from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, Sequence from contextlib import AsyncExitStack -from copy import copy from http import HTTPStatus -from typing import Any, Final +from typing import Any, Final, cast from fastapi import Request from openai.types.responses import ( - ResponseFunctionToolCall, ResponseOutputItem, ResponseOutputMessage, ResponseOutputText, @@ -20,15 +18,12 @@ from openai.types.responses import ( response_text_delta_event, ) from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob -from openai.types.responses.tool import Mcp, Tool -from openai_harmony import Message as OpenAIHarmonyMessage from pydantic import TypeAdapter from vllm import envs from vllm.config.utils import replace from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import ( - ChatCompletionMessageParam, ChatTemplateContentFormatOption, ) from vllm.entrypoints.generate.base.protocol import ( @@ -38,24 +33,13 @@ from vllm.entrypoints.generate.base.protocol import ( from vllm.entrypoints.generate.base.serving import GenerateBaseServing from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.openai.parser.harmony_utils import ( - build_harmony_preamble, - extract_instructions_from_messages, - get_user_message, - has_custom_tools, - render_for_completion, -) from vllm.entrypoints.openai.responses.context import ( ConversationContext, HarmonyContext, ParsableContext, SimpleContext, ) -from vllm.entrypoints.openai.responses.harmony import ( - construct_harmony_previous_input_messages, - harmony_to_response_output, - response_input_to_harmony, -) +from vllm.entrypoints.openai.responses.harmony import harmony_to_response_output from vllm.entrypoints.openai.responses.protocol import ( InputTokensDetails, OutputTokensDetails, @@ -80,8 +64,6 @@ from vllm.entrypoints.openai.responses.streaming_events import ( ) from vllm.entrypoints.openai.responses.utils import ( build_response_output_items, - construct_input_messages, - construct_tool_dicts, extract_function_tool_names, extract_tool_types, ) @@ -89,14 +71,19 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.utils.api_utils import get_max_tokens from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.exceptions import GenerationError, VLLMValidationError -from vllm.inputs import EngineInput, tokens_input +from vllm.inputs import EngineInput from vllm.logger import init_logger from vllm.logprobs import Logprob as SampleLogprob from vllm.logprobs import SampleLogprobs from vllm.lora.request import LoRARequest from vllm.outputs import CompletionOutput from vllm.parser import Parser, ParserManager -from vllm.renderers.online_renderer import OnlineRenderer +from vllm.renderers import TokenizeParams +from vllm.renderers.online_renderer import ( + OnlineRenderer, + ResponsesPreviousMessages, + ResponsesRenderResult, +) from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.tokenizers import TokenizerLike from vllm.utils import random_uuid @@ -105,45 +92,6 @@ from vllm.utils.collection_utils import as_list logger = init_logger(__name__) -def _extract_allowed_tools_from_mcp_requests( - tools: list[Tool], -) -> dict[str, list[str] | None]: - """ - Extract allowed_tools mapping from MCP tool requests. - - Returns a dictionary mapping server_label to allowed_tools list. - Handles both list format and McpAllowedToolsMcpToolFilter object format. - - Special handling: - - If allowed_tools is None, returns None (allows all tools) - - If allowed_tools contains "*", returns None (allows all tools) - - Otherwise, returns the list of specific tool names - - This function can be reused for both harmony and non-harmony MCP calls. - """ - allowed_tools_map: dict[str, list[str] | None] = {} - for tool in tools: - if not isinstance(tool, Mcp): - continue - - # allowed_tools can be a list or an object with tool_names - # Extract the actual list of tool names - allowed_tools_val = None - if tool.allowed_tools is not None: - if isinstance(tool.allowed_tools, list): - allowed_tools_val = tool.allowed_tools - elif hasattr(tool.allowed_tools, "tool_names"): - # It's an McpAllowedToolsMcpToolFilter object - allowed_tools_val = tool.allowed_tools.tool_names - - # Normalize "*" to None (both mean "allow all tools") - if allowed_tools_val is not None and "*" in allowed_tools_val: - allowed_tools_val = None - - allowed_tools_map[tool.server_label] = allowed_tools_val - return allowed_tools_map - - class OpenAIServingResponses(GenerateBaseServing): def __init__( self, @@ -226,7 +174,7 @@ class OpenAIServingResponses(GenerateBaseServing): # HACK(woosuk): This is a hack. We should use a better store. # FIXME: If enable_store=True, this may cause a memory leak since we # never remove messages from the store. - self.msg_store: dict[str, list[ChatCompletionMessageParam]] = {} + self.msg_store: dict[str, ResponsesPreviousMessages] = {} # HACK(wuhang): This is a hack. We should use a better store. # FIXME: If enable_store=True, this may cause a memory leak since we @@ -322,6 +270,33 @@ class OpenAIServingResponses(GenerateBaseServing): ) return None + async def _render_resolved_response_inputs( + self, + request: ResponsesRequest, + prev_response: ResponsesResponse | None, + *, + skip_mm_cache: bool = False, + ) -> ResponsesRenderResult | ErrorResponse: + previous_messages: ResponsesPreviousMessages | None = None + previous_response_outputs: list[ResponseOutputItem] | None = None + if prev_response is not None: + stored_messages = ( + self.msg_store[prev_response.id] + if self.use_harmony + else self.msg_store.get(prev_response.id) + ) + if stored_messages is not None: + previous_messages = stored_messages.copy() + previous_response_outputs = list(prev_response.output) + + return await self.online_renderer.render_responses( + request, + previous_messages=previous_messages, + previous_response_outputs=previous_response_outputs, + tool_server=self.tool_server, + skip_mm_cache=skip_mm_cache, + ) + async def create_responses( self, request: ResponsesRequest, @@ -374,12 +349,14 @@ class OpenAIServingResponses(GenerateBaseServing): lora_request = self._maybe_get_adapters(request) model_name = self.models.model_name(lora_request) - if self.use_harmony: - messages, engine_inputs = self._make_request_with_harmony( - request, prev_response - ) - else: - messages, engine_inputs = await self._make_request(request, prev_response) + render_result = await self._render_resolved_response_inputs( + request, + prev_response, + ) + if isinstance(render_result, ErrorResponse): + return render_result + messages = render_result.messages + engine_inputs = [render_result.engine_input] request_metadata = RequestResponseMetadata(request_id=request.request_id) if raw_request: @@ -508,6 +485,11 @@ class OpenAIServingResponses(GenerateBaseServing): engine_input=engine_input, sampling_params=sampling_params, context=context, + tok_params=( + request.build_tok_params(self.model_config) + if isinstance(context, HarmonyContext) + else None + ), lora_request=lora_request, priority=self._get_priority(request, raw_request), trace_headers=trace_headers, @@ -599,60 +581,28 @@ class OpenAIServingResponses(GenerateBaseServing): request_metadata, ) - async def _make_request( - self, - request: ResponsesRequest, - prev_response: ResponsesResponse | None, - ): - tool_dicts = construct_tool_dicts( - request.tools, - request.tool_choice, - exclude_tools_when_tool_choice_none=( - self.online_renderer.exclude_tools_when_tool_choice_none - ), - ) - # Construct the input messages. - messages = construct_input_messages( - request_instructions=request.instructions, - request_input=request.input, - prev_msg=self.msg_store.get(prev_response.id) if prev_response else None, - prev_response_output=prev_response.output if prev_response else None, - ) - chat_template_kwargs = self._effective_chat_template_kwargs(request) - _, engine_inputs = await self.online_renderer.preprocess_chat( - request, - messages, - default_template=self.chat_template, - default_template_content_format=self.chat_template_content_format, - default_template_kwargs=chat_template_kwargs, - tool_dicts=tool_dicts, - parser=self.parser, - ) - return messages, engine_inputs - async def _render_next_turn( self, request: ResponsesRequest, messages: list[ResponseInputOutputItem], - tool_dicts: list[dict[str, Any]] | None, - parser: type[Parser] | None, - chat_template: str | None, - chat_template_content_format: ChatTemplateContentFormatOption, - ): - new_messages = construct_input_messages( - request_input=messages, + ) -> list[EngineInput]: + render_request = request.model_copy( + update={ + "input": list(messages), + "instructions": None, + "previous_input_messages": None, + "previous_response_id": None, + } ) - chat_template_kwargs = self._effective_chat_template_kwargs(request) - _, engine_inputs = await self.online_renderer.preprocess_chat( - request, - new_messages, - default_template=chat_template, - default_template_content_format=chat_template_content_format, - default_template_kwargs=chat_template_kwargs, - tool_dicts=tool_dicts, - parser=parser, + render_result = await self.online_renderer.render_responses( + render_request, + previous_messages=None, + previous_response_outputs=None, + tool_server=self.tool_server, ) - return engine_inputs + if isinstance(render_result, ErrorResponse): + raise VLLMValidationError(render_result.error.message) + return [render_result.engine_input] async def _generate_with_builtin_tools( self, @@ -660,6 +610,7 @@ class OpenAIServingResponses(GenerateBaseServing): engine_input: EngineInput, sampling_params: SamplingParams, context: ConversationContext, + tok_params: TokenizeParams | None = None, lora_request: LoRARequest | None = None, priority: int = 0, trace_headers: Mapping[str, str] | None = None, @@ -667,6 +618,7 @@ class OpenAIServingResponses(GenerateBaseServing): reasoning_parser_kwargs: dict[str, Any] | None = None, ): max_model_len = self.model_config.max_model_len + cache_salt = cast(str | None, engine_input.get("cache_salt")) orig_priority = priority sub_request = 0 @@ -711,18 +663,19 @@ class OpenAIServingResponses(GenerateBaseServing): # Create inputs for the next turn. # Render the next prompt token ids and update sampling_params. if isinstance(context, HarmonyContext): - token_ids = context.render_for_completion() - engine_input = tokens_input(token_ids) + engine_input = self.online_renderer.render_responses_harmony_messages( + context.messages, + cache_salt=cache_salt, + tok_params=tok_params, + ) - sampling_params.max_tokens = max_model_len - len(token_ids) + sampling_params.max_tokens = max_model_len - self._extract_prompt_len( + engine_input + ) elif isinstance(context, ParsableContext): (engine_input,) = await self._render_next_turn( context.request, context.response_messages, - context.tool_dicts, - context.parser_cls, - context.chat_template, - context.chat_template_content_format, ) sampling_params.max_tokens = get_max_tokens( @@ -740,28 +693,6 @@ class OpenAIServingResponses(GenerateBaseServing): priority = orig_priority - 1 sub_request += 1 - def _make_request_with_harmony( - self, - request: ResponsesRequest, - prev_response: ResponsesResponse | None, - ): - if self.parser is not None: - # HarmonyParser doesn't need chat_template_kwargs - # TODO: Unify adjust_request() call with non-harmony branch - self.parser( - self.renderer.get_tokenizer(), - request.tools, - model_config=self.model_config, - ).adjust_request(request=request) - - arrival_time = time.time() - messages = self._construct_input_messages_with_harmony(request, prev_response) - prompt_token_ids = render_for_completion(messages) - engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt) - engine_input["arrival_time"] = arrival_time - - return messages, [engine_input] - async def _initialize_tool_sessions( self, request: ResponsesRequest, @@ -1104,113 +1035,6 @@ class OpenAIServingResponses(GenerateBaseServing): ) ] - def _get_harmony_builtin_tool_descriptions( - self, request: ResponsesRequest, tool_types: set[str] - ) -> dict[str, str | None]: - # Extract allowed_tools from MCP tool requests - allowed_tools_map = _extract_allowed_tools_from_mcp_requests(request.tools) - - # Get filtered tool descriptions first. - # If get_tool_description returns None (due to filtering), the tool is disabled. - browser_description = ( - self.tool_server.get_tool_description( - "browser", allowed_tools_map.get("web_search_preview") - ) - if "web_search_preview" in tool_types - and self.tool_server is not None - and self.tool_server.has_tool("browser") - else None - ) - python_description = ( - self.tool_server.get_tool_description( - "python", allowed_tools_map.get("code_interpreter") - ) - if "code_interpreter" in tool_types - and self.tool_server is not None - and self.tool_server.has_tool("python") - else None - ) - container_description = ( - self.tool_server.get_tool_description( - "container", allowed_tools_map.get("container") - ) - if "container" in tool_types - and self.tool_server is not None - and self.tool_server.has_tool("container") - else None - ) - return { - "browser_description": browser_description, - "python_description": python_description, - "container_description": container_description, - } - - def _construct_input_messages_with_harmony( - self, - request: ResponsesRequest, - prev_response: ResponsesResponse | None, - ) -> list[OpenAIHarmonyMessage]: - messages: list[OpenAIHarmonyMessage] = [] - request_input = request.input - if prev_response is None: - # New conversation. - tool_types = extract_tool_types(request.tools) - with_custom_tools = has_custom_tools(tool_types) - instructions = request.instructions - if instructions is None and isinstance(request_input, list): - instructions, request_input = extract_instructions_from_messages( - request_input - ) - tool_descriptions = self._get_harmony_builtin_tool_descriptions( - request, tool_types - ) - tools = request.tools if with_custom_tools else None - messages.extend( - build_harmony_preamble( - instructions=instructions, - tools=tools, - reasoning_effort=( - request.reasoning.effort if request.reasoning else None - ), - with_custom_tools=with_custom_tools, - **tool_descriptions, - ) - ) - messages += construct_harmony_previous_input_messages(request) - - else: - # Continue the previous conversation. - # FIXME(woosuk): Currently, request params like reasoning and - # instructions are ignored. - prev_msgs = self.msg_store[prev_response.id] - - messages.extend(prev_msgs) - # Append the new input. - # Responses API supports simple text inputs without chat format. - if isinstance(request_input, str): - # Skip empty string input when previous_input_messages supplies - # the full conversation history --- an empty trailing user message - # confuses the model into thinking nothing was sent. - if request_input or not request.previous_input_messages: - messages.append(get_user_message(request_input)) - else: - if prev_response is not None: - prev_outputs = copy(prev_response.output) - else: - prev_outputs = [] - for response_msg in request_input: - new_msg = response_input_to_harmony(response_msg, prev_outputs) - if new_msg is not None: - messages.append(new_msg) - - # User passes in a tool call request and its output. We need - # to add the tool call request to prev_outputs so that - # response_input_to_harmony can find the tool call request when - # parsing the tool call output. - if isinstance(response_msg, ResponseFunctionToolCall): - prev_outputs.append(response_msg) - return messages - async def _run_background_request_stream( self, request: ResponsesRequest, diff --git a/vllm/entrypoints/scale_out/factories.py b/vllm/entrypoints/scale_out/factories.py index 1af682c31e..30c329c51f 100644 --- a/vllm/entrypoints/scale_out/factories.py +++ b/vllm/entrypoints/scale_out/factories.py @@ -31,6 +31,7 @@ def init_render_state( state.openai_serving_models, state.online_renderer, request_logger=request_logger, + tool_server=state.tool_server, ) state.serving_derender = ServingDerender( diff --git a/vllm/entrypoints/scale_out/render/api_router.py b/vllm/entrypoints/scale_out/render/api_router.py index 634dcef4e3..97df983d2e 100644 --- a/vllm/entrypoints/scale_out/render/api_router.py +++ b/vllm/entrypoints/scale_out/render/api_router.py @@ -8,6 +8,7 @@ from fastapi.responses import JSONResponse from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.completion.protocol import CompletionRequest +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.utils.api_utils import validate_json_request from vllm.logger import init_logger @@ -95,3 +96,25 @@ async def render_completion(request: CompletionRequest, raw_request: Request): return JSONResponse(content=result.model_dump(), status_code=result.error.code) return JSONResponse(content=[item.model_dump() for item in result]) + + +@router.post( + "/v1/responses/render", + dependencies=[Depends(validate_json_request)], + response_model=GenerateRequest, + responses={ + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, + HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, + }, +) +async def render_responses(request: ResponsesRequest, raw_request: Request): + handler = render(raw_request) + if handler is None: + raise NotImplementedError("The model does not support Responses Render API") + + result = await handler.render_responses_request(request) + if isinstance(result, ErrorResponse): + return JSONResponse(content=result.model_dump(), status_code=result.error.code) + return JSONResponse(content=result.model_dump()) diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index f36b1a3363..b79138664f 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import TYPE_CHECKING + from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest from vllm.entrypoints.anthropic.serving import AnthropicServingMessages from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest @@ -8,6 +10,7 @@ from vllm.entrypoints.openai.models.serving import ( OpenAIModelRegistry, OpenAIServingModels, ) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.entrypoints.scale_out.token_in_token_out.mm_features import ( extract_mm_features, ) @@ -31,6 +34,9 @@ from vllm.utils import random_uuid logger = init_logger(__name__) +if TYPE_CHECKING: + from vllm.entrypoints.mcp.tool_server import ToolServer + class ServingRender(BaseServing): def __init__( @@ -39,6 +45,7 @@ class ServingRender(BaseServing): online_renderer: "OnlineRenderer", *, request_logger: RequestLogger | None = None, + tool_server: "ToolServer | None" = None, ) -> None: super().__init__( models=models, @@ -47,6 +54,7 @@ class ServingRender(BaseServing): ) self.online_renderer = online_renderer + self.tool_server = tool_server self._merge_inline_system = ( AnthropicServingMessages._detect_merge_inline_system( @@ -229,6 +237,64 @@ class ServingRender(BaseServing): return generate_requests + async def render_responses_request( + self, + request: ResponsesRequest, + ) -> GenerateRequest | ErrorResponse: + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + if request.previous_response_id is not None: + return self.create_error_response( + message=( + "previous_response_id is not supported by the stateless " + "render endpoint." + ), + err_type="invalid_request_error", + param="previous_response_id", + ) + + result = await self.online_renderer.render_responses( + request, + previous_messages=None, + previous_response_outputs=None, + tool_server=self.tool_server, + skip_mm_cache=True, + ) + if isinstance(result, ErrorResponse): + return result + + engine_input = result.engine_input + prompt_components = extract_prompt_components(self.model_config, engine_input) + token_ids = prompt_components.token_ids + if not token_ids: + return self.create_error_response("No token_ids rendered") + + input_length = extract_prompt_len(self.model_config, engine_input) + max_tokens = get_max_tokens( + self.model_config.max_model_len, + request.max_output_tokens, + input_length, + self.default_sampling_params, + self.override_max_tokens, + truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None), + ) + params = request.to_sampling_params(max_tokens, self.default_sampling_params) + + return GenerateRequest( + request_id=request.request_id, + token_ids=list(token_ids), + features=self._extract_mm_features(engine_input), + sampling_params=params, + model=request.model, + stream=bool(request.stream), + cache_salt=request.cache_salt, + priority=request.priority, + kv_transfer_params=request.kv_transfer_params, + ec_transfer_params=request.ec_transfer_params, + token_offsets=engine_input.get("prompt_token_offsets"), + ) + def _placeholder_metadata_fields(self, modality: str) -> set[str]: """Fields the EC consumer still requires after embeddings transfer.""" if self._placeholder_metadata_parser_failed: diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index db7cc551c8..b6075d9bd1 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -1,13 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import time from collections.abc import Sequence +from dataclasses import dataclass from http import HTTPStatus -from typing import Any +from typing import TYPE_CHECKING, Any, TypeAlias +from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem +from openai.types.responses.tool import Mcp, Tool from openai_harmony import Message as OpenAIMessage +from openai_harmony import ToolNamespaceConfig from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ( + ChatCompletionMessageParam, ChatTemplateContentFormatOption, ConversationMessage, ) @@ -19,12 +25,24 @@ from vllm.entrypoints.openai.completion.protocol import ( CompletionRequest, ) from vllm.entrypoints.openai.parser.harmony_utils import ( + BUILTIN_TOOL_TO_MCP_SERVER_LABEL, build_harmony_preamble, extract_instructions_from_messages, + get_user_message, + has_custom_tools, parse_chat_inputs_to_harmony_messages, render_for_completion, ) +from vllm.entrypoints.openai.responses.harmony import ( + construct_harmony_previous_input_messages, + response_input_to_harmony, +) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.entrypoints.openai.responses.utils import ( + construct_input_messages, + construct_tool_dicts, + extract_tool_types, +) from vllm.entrypoints.serve import create_error_response from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.utils.request_logger import RequestLogger @@ -32,11 +50,12 @@ from vllm.inputs import ( EngineInput, PromptType, SingletonPrompt, + TokensPrompt, tokens_input, ) from vllm.logger import init_logger from vllm.parser import Parser, ParserManager -from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs +from vllm.renderers import BaseRenderer, ChatParams, TokenizeParams, merge_kwargs from vllm.renderers.inputs.preprocess import ( parse_model_prompt, prompt_to_seq, @@ -46,6 +65,39 @@ from vllm.utils.mistral import mt as _mt logger = init_logger(__name__) +if TYPE_CHECKING: + from vllm.entrypoints.mcp.tool_server import ToolServer + + +ResponsesPreviousMessages: TypeAlias = ( + list[ChatCompletionMessageParam] | list[OpenAIMessage] +) + + +@dataclass(frozen=True) +class ResponsesRenderResult: + messages: ResponsesPreviousMessages + engine_input: EngineInput + + +def _extract_allowed_tools_from_mcp_requests( + tools: list[Tool], +) -> dict[str, list[str] | None]: + allowed_tools_map: dict[str, list[str] | None] = {} + for tool in tools: + if not isinstance(tool, Mcp): + continue + + allowed_tools = None + if isinstance(tool.allowed_tools, list): + allowed_tools = tool.allowed_tools + elif tool.allowed_tools is not None: + allowed_tools = tool.allowed_tools.tool_names + if allowed_tools is not None and "*" in allowed_tools: + allowed_tools = None + allowed_tools_map[tool.server_label] = allowed_tools + return allowed_tools_map + def _reused_prompt_token_ids(request: Any) -> list[int] | None: """Pop prompt token ids forwarded for decode-side reuse, if any. @@ -217,6 +269,219 @@ class OnlineRenderer: return conversation, engine_inputs + async def render_responses( + self, + request: ResponsesRequest, + *, + previous_messages: ResponsesPreviousMessages | None = None, + previous_response_outputs: list[ResponseOutputItem] | None = None, + tool_server: "ToolServer | None" = None, + skip_mm_cache: bool = False, + ) -> ResponsesRenderResult | ErrorResponse: + """Render a Responses request using only explicitly supplied history.""" + template_error = self.validate_chat_template( + request_chat_template=None, + chat_template_kwargs=request.chat_template_kwargs, + trust_request_chat_template=self.trust_request_chat_template, + ) + if template_error is not None: + return template_error + + if self.use_harmony: + return self._render_responses_with_harmony( + request, + previous_messages=previous_messages, + previous_response_outputs=previous_response_outputs, + tool_server=tool_server, + ) + + if previous_messages is not None and any( + isinstance(message, OpenAIMessage) for message in previous_messages + ): + return self.create_error_response( + "Non-Harmony Responses history must use chat messages.", + err_type="invalid_request_error", + param="previous_response_id", + ) + + tool_dicts = construct_tool_dicts( + request.tools, + request.tool_choice, + exclude_tools_when_tool_choice_none=( + self.exclude_tools_when_tool_choice_none + ), + ) + messages = construct_input_messages( + request_instructions=request.instructions, + request_input=request.input, + prev_msg=list(previous_messages) if previous_messages is not None else None, + prev_response_output=( + list(previous_response_outputs) + if previous_response_outputs is not None + else None + ), + ) + chat_template_kwargs = ( + request.build_chat_params( + self.chat_template, + self.chat_template_content_format, + ) + .with_defaults(self.default_chat_template_kwargs) + .chat_template_kwargs + ) + _, engine_inputs = await self.preprocess_chat( + request, + messages, + default_template=self.chat_template, + default_template_content_format=self.chat_template_content_format, + default_template_kwargs=chat_template_kwargs, + tool_dicts=tool_dicts, + parser=self.parser, + skip_mm_cache=skip_mm_cache, + ) + return self._responses_render_result(messages, engine_inputs) + + def _render_responses_with_harmony( + self, + request: ResponsesRequest, + *, + previous_messages: ResponsesPreviousMessages | None, + previous_response_outputs: list[ResponseOutputItem] | None, + tool_server: "ToolServer | None", + ) -> ResponsesRenderResult | ErrorResponse: + if self.parser is not None: + # HarmonyParser doesn't need chat_template_kwargs + # TODO: Unify adjust_request() call with non-harmony branch + self.parser( + self.renderer.get_tokenizer(), + request.tools, + model_config=self.model_config, + ).adjust_request(request=request) + + if previous_messages is not None and any( + not isinstance(message, OpenAIMessage) for message in previous_messages + ): + return self.create_error_response( + "Harmony Responses history must use Harmony messages.", + err_type="invalid_request_error", + param="previous_response_id", + ) + + messages: list[OpenAIMessage] = [] + request_input = request.input + if previous_messages is None: + tool_types = extract_tool_types(request.tools) + with_custom_tools = has_custom_tools(tool_types) + instructions = request.instructions + if instructions is None and isinstance(request_input, list): + instructions, request_input = extract_instructions_from_messages( + request_input + ) + descriptions = self._get_harmony_builtin_tool_descriptions( + request.tools, + tool_types, + tool_server, + ) + messages.extend( + build_harmony_preamble( + instructions=instructions, + tools=request.tools if with_custom_tools else None, + reasoning_effort=( + request.reasoning.effort if request.reasoning else None + ), + with_custom_tools=with_custom_tools, + **descriptions, + ) + ) + messages.extend(construct_harmony_previous_input_messages(request)) + else: + messages.extend(previous_messages) + + previous_outputs = list(previous_response_outputs or ()) + try: + if isinstance(request_input, str): + if request_input or not request.previous_input_messages: + messages.append(get_user_message(request_input)) + else: + for response_message in request_input: + new_message = response_input_to_harmony( + response_message, + previous_outputs, + ) + if new_message is not None: + messages.append(new_message) + if isinstance(response_message, ResponseFunctionToolCall): + previous_outputs.append(response_message) + except ValueError as exc: + return self.create_error_response( + str(exc), + err_type="invalid_request_error", + param="input", + ) + + return ResponsesRenderResult( + messages=messages, + engine_input=self.render_responses_harmony_messages( + messages, + cache_salt=request.cache_salt, + tok_params=request.build_tok_params(self.model_config), + ), + ) + + def _get_harmony_builtin_tool_descriptions( + self, + tools: list[Tool], + tool_types: set[str], + tool_server: "ToolServer | None", + ) -> dict[str, ToolNamespaceConfig | None]: + allowed_tools = _extract_allowed_tools_from_mcp_requests(tools) + descriptions: dict[str, ToolNamespaceConfig | None] = {} + for server_name, request_name in BUILTIN_TOOL_TO_MCP_SERVER_LABEL.items(): + description = ( + tool_server.get_tool_description( + server_name, + allowed_tools.get(request_name), + ) + if request_name in tool_types + and tool_server is not None + and tool_server.has_tool(server_name) + else None + ) + descriptions[f"{server_name}_description"] = description + return descriptions + + def render_responses_harmony_messages( + self, + messages: list[OpenAIMessage], + *, + cache_salt: str | None, + tok_params: TokenizeParams | None = None, + ) -> EngineInput: + arrival_time = time.time() + prompt = TokensPrompt(prompt_token_ids=render_for_completion(messages)) + if tok_params is not None: + tok_params.apply_post_tokenization( + self.renderer.tokenizer, + prompt, + ) + engine_input = tokens_input(prompt["prompt_token_ids"], cache_salt=cache_salt) + engine_input["arrival_time"] = arrival_time + return engine_input + + def _responses_render_result( + self, + messages: list[ChatCompletionMessageParam], + engine_inputs: list[EngineInput], + ) -> ResponsesRenderResult | ErrorResponse: + if len(engine_inputs) != 1: + return self.create_error_response( + f"Expected exactly 1 engine prompt, got {len(engine_inputs)}" + ) + return ResponsesRenderResult( + messages=messages, + engine_input=engine_inputs[0], + ) + def _make_request_with_harmony( self, request: ChatCompletionRequest,