[Frontend] Add stateless /v1/responses/render endpoint (#50195)

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Francisco Javier Arceo
2026-09-07 12:50:42 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 5e6f6a8ed4
commit 6a2a2bb02b
19 changed files with 1228 additions and 298 deletions
+2
View File
@@ -141,6 +141,8 @@ For further details on renderer APIs, please refer to [this page](renderer.md).
- Render completion requests - Render completion requests
- [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`) - [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
- Render chat completions - Render chat completions
- [Responses Render API](renderer.md) (`/v1/responses/render`)
- Render self-contained Responses requests
### Derenderer APIs ### Derenderer APIs
+23
View File
@@ -25,6 +25,29 @@ VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve <model>
- Render completion requests - Render completion requests
- [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`) - [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
- Render chat completions - Render chat completions
- [Responses Render API](renderer.md) (`/v1/responses/render`)
- Render a self-contained Responses request
The Responses render endpoint uses the same prompt construction as
`/v1/responses` and returns one token-in `GenerateRequest`. It is stateless:
inline history is supported, but `previous_response_id` is not. Callers must
resolve stored response state and include the resulting history in the request
before rendering.
For multimodal requests, the `GenerateRequest` contains the model-processed
multimodal payload, which can be substantially larger than the source image or
video. The caller must forward that payload unchanged to the generation service
and provision transport limits and memory accordingly.
```bash
curl http://localhost:8000/v1/responses/render \
-H "Content-Type: application/json" \
-d '{
"model": "meta-llama/Llama-3.1-8B-Instruct",
"input": "Explain prefix caching in one sentence.",
"max_output_tokens": 32
}'
```
For the post processing counterpart that turns generated token IDs back into OpenAI compatible responses, see the [Derenderer APIs](derenderer.md). For the post processing counterpart that turns generated token IDs back into OpenAI compatible responses, see the [Derenderer APIs](derenderer.md).
+1
View File
@@ -170,6 +170,7 @@ When `--api-key` is configured, the following `/v1` endpoints require Bearer tok
- `/v1/messages/render` - Render Anthropic-compatible messages (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) - `/v1/messages/render` - Render Anthropic-compatible messages (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
- `/v1/messages/count_tokens` - Count tokens for Anthropic messages - `/v1/messages/count_tokens` - Count tokens for Anthropic messages
- `/v1/responses` - Create a response - `/v1/responses` - Create a response
- `/v1/responses/render` - Render a self-contained response request (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
- `/v1/responses/{response_id}` - Retrieve a response - `/v1/responses/{response_id}` - Retrieve a response
- `/v1/responses/{response_id}/cancel` - Cancel a response - `/v1/responses/{response_id}/cancel` - Cancel a response
- `/v1/score` - Scoring API - `/v1/score` - Scoring API
@@ -2,9 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json import json
from types import SimpleNamespace
import pytest import pytest
import vllm.entrypoints.launchers.cli_args as cli_args_module
from tests.utils import VLLM_PATH from tests.utils import VLLM_PATH
from vllm.entrypoints.launchers.cli_args import ( from vllm.entrypoints.launchers.cli_args import (
make_arg_parser, make_arg_parser,
@@ -332,6 +334,26 @@ def test_default_chat_template_kwargs_default_none(serve_parser):
assert args.default_chat_template_kwargs is None assert args.default_chat_template_kwargs is None
@pytest.mark.parametrize(
"default_kwargs,cohere_format,expected",
[
(None, "cmd3", {"cohere_format": "cmd3"}),
(
{"cohere_format": "cmd4", "custom": True},
"cmd3",
{"cohere_format": "cmd4", "custom": True},
),
],
)
def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, expected):
args = SimpleNamespace(
default_chat_template_kwargs=default_kwargs,
cohere_format=cohere_format,
)
assert cli_args_module.resolve_default_chat_template_kwargs(args) == expected
def test_default_chat_template_kwargs_invalid_json(serve_parser): def test_default_chat_template_kwargs_invalid_json(serve_parser):
"""Ensure invalid JSON raises an error""" """Ensure invalid JSON raises an error"""
with pytest.raises(SystemExit): with pytest.raises(SystemExit):
@@ -4,14 +4,19 @@
from __future__ import annotations from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest import pytest
import pytest_asyncio import pytest_asyncio
from openai import OpenAI from openai import OpenAI
from openai_harmony import Message, ToolDescription, ToolNamespaceConfig from openai_harmony import Message, ToolDescription, ToolNamespaceConfig
import vllm.entrypoints.mcp.tool_server as tool_server_module
from tests.utils import RemoteOpenAIServer from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.mcp.tool_server import ( from vllm.entrypoints.mcp.tool_server import (
MCPToolServer, MCPToolServer,
init_tool_server,
post_process_tools_description, post_process_tools_description,
) )
@@ -39,6 +44,25 @@ _PYTHON_TOOL_INSTRUCTION = (
) )
@pytest.mark.asyncio
async def test_init_tool_server_without_configuration():
args = SimpleNamespace(tool_server=None)
assert await init_tool_server(args) is None
@pytest.mark.asyncio
async def test_init_tool_server_with_mcp_url(monkeypatch: pytest.MonkeyPatch):
server = MagicMock()
server.add_tool_server = AsyncMock()
monkeypatch.setattr(tool_server_module, "MCPToolServer", lambda: server)
result = await init_tool_server(SimpleNamespace(tool_server="http://mcp.test"))
assert result is server
server.add_tool_server.assert_awaited_once_with("http://mcp.test")
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
class TestMCPToolServerUnit: class TestMCPToolServerUnit:
"""Test MCP tool description processing and filtering. """Test MCP tool description processing and filtering.
@@ -2,12 +2,16 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import AsyncExitStack from contextlib import AsyncExitStack
from unittest.mock import MagicMock from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest import pytest
import pytest_asyncio import pytest_asyncio
from openai.types.responses import ( from openai.types.responses import (
ResponseFunctionToolCall,
ResponseOutputItemDoneEvent, ResponseOutputItemDoneEvent,
ResponseOutputMessage,
ResponseOutputText,
ResponseReasoningItem, ResponseReasoningItem,
ResponseReasoningTextDeltaEvent, ResponseReasoningTextDeltaEvent,
ResponseReasoningTextDoneEvent, ResponseReasoningTextDoneEvent,
@@ -23,6 +27,8 @@ from openai.types.responses.tool import (
Mcp, Mcp,
Tool, Tool,
) )
from openai_harmony import Message as OpenAIHarmonyMessage
from openai_harmony import Role
import vllm.envs as envs import vllm.envs as envs
from vllm.entrypoints.generate.base.protocol import ( from vllm.entrypoints.generate.base.protocol import (
@@ -32,7 +38,11 @@ from vllm.entrypoints.generate.base.protocol import (
RequestResponseMetadata, RequestResponseMetadata,
) )
from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.responses.context import ConversationContext, SimpleContext from vllm.entrypoints.openai.responses.context import (
ConversationContext,
HarmonyContext,
SimpleContext,
)
from vllm.entrypoints.openai.responses.protocol import ( from vllm.entrypoints.openai.responses.protocol import (
ResponseCreatedEvent, ResponseCreatedEvent,
ResponseRawMessageAndToken, ResponseRawMessageAndToken,
@@ -42,7 +52,6 @@ from vllm.entrypoints.openai.responses.protocol import (
) )
from vllm.entrypoints.openai.responses.serving import ( from vllm.entrypoints.openai.responses.serving import (
OpenAIServingResponses, OpenAIServingResponses,
_extract_allowed_tools_from_mcp_requests,
extract_tool_types, extract_tool_types,
) )
from vllm.entrypoints.openai.responses.streaming_events import ( from vllm.entrypoints.openai.responses.streaming_events import (
@@ -52,8 +61,23 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.inputs import tokens_input from vllm.inputs import tokens_input
from vllm.outputs import CompletionOutput, RequestOutput from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser.harmony import Segment from vllm.parser.harmony import Segment
from vllm.renderers import TokenizeParams
from vllm.renderers.online_renderer import (
OnlineRenderer,
_extract_allowed_tools_from_mcp_requests,
)
from vllm.sampling_params import SamplingParams from vllm.sampling_params import SamplingParams
pytestmark = pytest.mark.skip_global_cleanup
def _new_online_renderer() -> OnlineRenderer:
renderer = OnlineRenderer.__new__(OnlineRenderer)
renderer.exclude_tools_when_tool_choice_none = False
renderer.parser = None
renderer.trust_request_chat_template = True
return renderer
class MockConversationContext(ConversationContext): class MockConversationContext(ConversationContext):
"""Mock conversation context for testing""" """Mock conversation context for testing"""
@@ -205,6 +229,325 @@ def test_response_created_event_uses_public_json_schema_alias() -> None:
assert event.response.text.format.model_dump(by_alias=True)["schema"] == schema assert event.response.text.format.model_dump(by_alias=True)["schema"] == schema
@pytest.mark.asyncio
async def test_online_renderer_renders_non_harmony_responses_with_explicit_history():
renderer = _new_online_renderer()
renderer.use_harmony = False
renderer.chat_template = "server-template"
renderer.chat_template_content_format = "string"
renderer.default_chat_template_kwargs = {"server_default": "kept"}
renderer.parser = None
renderer.preprocess_chat = AsyncMock(
return_value=(
[],
[tokens_input([11, 12, 13], cache_salt="request-salt")],
)
)
previous_messages = [
{"role": "system", "content": "old instructions"},
{"role": "user", "content": "first"},
]
previous_outputs = [
ResponseOutputMessage(
id="msg_1",
content=[
ResponseOutputText(
annotations=[],
text="prior answer",
type="output_text",
)
],
role="assistant",
status="completed",
type="message",
)
]
request = ResponsesRequest(
input="next",
instructions="new instructions",
tools=[],
cache_salt="request-salt",
chat_template_kwargs={"request_default": "kept"},
)
result = await renderer.render_responses(
request,
previous_messages=previous_messages,
previous_response_outputs=previous_outputs,
)
assert not isinstance(result, ErrorResponse)
assert result.messages == [
{"role": "system", "content": "new instructions"},
{"role": "user", "content": "first"},
{"role": "assistant", "content": "prior answer"},
{"role": "user", "content": "next"},
]
assert result.engine_input["prompt_token_ids"] == [11, 12, 13]
assert result.engine_input["cache_salt"] == "request-salt"
assert previous_messages[0]["content"] == "old instructions"
assert previous_outputs[0].content[0].text == "prior answer"
preprocess_kwargs = renderer.preprocess_chat.call_args.kwargs
assert preprocess_kwargs["default_template"] == "server-template"
assert preprocess_kwargs["default_template_content_format"] == "string"
assert preprocess_kwargs["default_template_kwargs"]["server_default"] == "kept"
assert preprocess_kwargs["default_template_kwargs"]["request_default"] == "kept"
@pytest.mark.asyncio
async def test_online_renderer_rejects_untrusted_responses_chat_template():
renderer = _new_online_renderer()
renderer.use_harmony = False
renderer.chat_template = "server-template"
renderer.chat_template_content_format = "string"
renderer.default_chat_template_kwargs = {}
renderer.exclude_tools_when_tool_choice_none = False
renderer.trust_request_chat_template = False
renderer.parser = None
renderer.preprocess_chat = AsyncMock(
return_value=([], [tokens_input([1])]),
)
request = ResponsesRequest(
input="hello",
chat_template_kwargs={"chat_template": "{{ messages }}"},
)
result = await renderer.render_responses(request)
assert isinstance(result, ErrorResponse)
assert result.error.code == 400
assert "untrusted chat template" in result.error.message.lower()
renderer.preprocess_chat.assert_not_awaited()
@pytest.mark.asyncio
async def test_online_renderer_rejects_harmony_history_for_non_harmony_model():
renderer = _new_online_renderer()
renderer.use_harmony = False
request = ResponsesRequest(input="next")
result = await renderer.render_responses(
request,
previous_messages=[
OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first")
],
)
assert isinstance(result, ErrorResponse)
assert result.error.param == "previous_response_id"
@pytest.mark.asyncio
async def test_online_renderer_rejects_chat_history_for_harmony_model():
renderer = _new_online_renderer()
renderer.use_harmony = True
request = ResponsesRequest(input="next")
result = await renderer.render_responses(
request,
previous_messages=[{"role": "user", "content": "first"}],
)
assert isinstance(result, ErrorResponse)
assert result.error.param == "previous_response_id"
@pytest.mark.asyncio
async def test_online_renderer_adjusts_harmony_tool_choice(
monkeypatch: pytest.MonkeyPatch,
):
class StubParser:
def __init__(self, tokenizer, tools, *, model_config):
pass
def adjust_request(self, request):
request.cache_salt = "adjusted"
return request
renderer = _new_online_renderer()
renderer.use_harmony = True
renderer.parser = StubParser
renderer.model_config = SimpleNamespace(max_model_len=100)
tokenizer = SimpleNamespace(truncation_side="left")
renderer.renderer = SimpleNamespace(
tokenizer=tokenizer,
get_tokenizer=lambda: tokenizer,
)
monkeypatch.setattr(
"vllm.renderers.online_renderer.render_for_completion",
lambda messages: [1],
)
request = ResponsesRequest(
input="next",
tools=[
{
"type": "function",
"name": "lookup",
"parameters": {"type": "object", "properties": {}},
}
],
tool_choice={"type": "function", "name": "lookup"},
)
result = await renderer.render_responses(request)
assert not isinstance(result, ErrorResponse)
assert result.engine_input["cache_salt"] == "adjusted"
@pytest.mark.asyncio
async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt(
monkeypatch: pytest.MonkeyPatch,
):
renderer = _new_online_renderer()
renderer.use_harmony = True
renderer.model_config = SimpleNamespace(max_model_len=5)
renderer.renderer = SimpleNamespace(
tokenizer=SimpleNamespace(truncation_side="left")
)
monkeypatch.setattr(
"vllm.renderers.online_renderer.render_for_completion",
lambda messages: [1, 2, 3, 4, 5, 6],
)
request = ResponsesRequest(
input="hello",
max_output_tokens=2,
truncation="auto",
)
result = await renderer.render_responses(request)
assert not isinstance(result, ErrorResponse)
assert result.engine_input["prompt_token_ids"] == [4, 5, 6]
@pytest.mark.asyncio
async def test_online_renderer_renders_harmony_continuation_with_call_linkage(
monkeypatch: pytest.MonkeyPatch,
):
renderer = _new_online_renderer()
renderer.use_harmony = True
renderer.model_config = SimpleNamespace(max_model_len=100)
renderer.renderer = SimpleNamespace(
tokenizer=SimpleNamespace(truncation_side="left")
)
monkeypatch.setattr(
"vllm.renderers.online_renderer.render_for_completion",
lambda messages: [1],
)
previous_messages = [
OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first"),
]
previous_outputs = [
ResponseFunctionToolCall(
arguments='{"city":"Paris"}',
call_id="call_1",
name="lookup_weather",
type="function_call",
)
]
request = ResponsesRequest(
input=[
{
"type": "function_call_output",
"call_id": "call_1",
"output": "sunny",
}
],
instructions="replacement instructions are ignored on continuations",
reasoning={"effort": "high"},
tools=[
{
"type": "function",
"name": "lookup_weather",
"parameters": {"type": "object", "properties": {}},
}
],
cache_salt="request-salt",
)
result = await renderer.render_responses(
request,
previous_messages=previous_messages,
previous_response_outputs=previous_outputs,
)
assert not isinstance(result, ErrorResponse)
assert result.messages[0] is previous_messages[0]
tool_output = result.messages[-1]
assert tool_output.author.role == "tool"
assert tool_output.author.name == "functions.lookup_weather"
assert tool_output.channel == "commentary"
assert tool_output.recipient == "assistant"
assert result.engine_input["cache_salt"] == "request-salt"
assert "arrival_time" in result.engine_input
assert previous_messages == [
OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first"),
]
assert previous_outputs[0].call_id == "call_1"
@pytest.mark.asyncio
async def test_online_renderer_returns_typed_error_for_bad_harmony_call_reference():
renderer = _new_online_renderer()
renderer.use_harmony = True
request = ResponsesRequest(
input=[
{
"type": "function_call_output",
"call_id": "missing",
"output": "orphaned",
}
],
tools=[],
)
result = await renderer.render_responses(request)
assert isinstance(result, ErrorResponse)
assert result.error.type == "invalid_request_error"
assert result.error.param == "input"
assert "No call message found for missing" in result.error.message
@pytest.mark.asyncio
async def test_online_renderer_preserves_missing_harmony_builtin_tool_behavior(
monkeypatch: pytest.MonkeyPatch,
):
renderer = _new_online_renderer()
renderer.use_harmony = True
renderer.model_config = SimpleNamespace(max_model_len=100)
renderer.renderer = SimpleNamespace(
tokenizer=SimpleNamespace(truncation_side="left")
)
monkeypatch.setattr(
"vllm.renderers.online_renderer.render_for_completion",
lambda messages: [1],
)
request = ResponsesRequest(
input="search",
tools=[{"type": "web_search_preview"}],
)
result = await renderer.render_responses(request, tool_server=None)
assert not isinstance(result, ErrorResponse)
@pytest.mark.parametrize(
"engine_inputs",
[[], [tokens_input([1]), tokens_input([2])]],
)
def test_responses_render_result_rejects_non_single_prompt(engine_inputs):
renderer = _new_online_renderer()
result = renderer._responses_render_result([], engine_inputs)
assert isinstance(result, ErrorResponse)
assert f"got {len(engine_inputs)}" in result.error.message
class TestInitializeToolSessions: class TestInitializeToolSessions:
"""Test class for _initialize_tool_sessions method""" """Test class for _initialize_tool_sessions method"""
@@ -240,6 +583,139 @@ class TestInitializeToolSessions:
return instance return instance
@pytest.mark.asyncio
async def test_stateful_harmony_delegates_named_tool_choice_to_renderer(
self,
serving_responses_instance,
):
serving_responses_instance.use_harmony = True
serving_responses_instance.online_renderer.render_responses = AsyncMock(
return_value=SimpleNamespace(
messages=[],
engine_input=tokens_input([1]),
)
)
request = ResponsesRequest(
input="hello",
tool_choice={"type": "function", "name": "lookup"},
tools=[
{
"type": "function",
"name": "lookup",
"parameters": {"type": "object", "properties": {}},
}
],
)
result = await serving_responses_instance._render_resolved_response_inputs(
request,
None,
)
assert result.engine_input["prompt_token_ids"] == [1]
serving_responses_instance.online_renderer.render_responses.assert_awaited_once()
@pytest.mark.asyncio
async def test_render_next_turn_reuses_shared_responses_renderer(
self, serving_responses_instance
):
serving_responses_instance.online_renderer.render_responses = AsyncMock(
return_value=SimpleNamespace(
messages=[],
engine_input=tokens_input([8, 9], cache_salt="request-salt"),
)
)
serving_responses_instance.online_renderer.preprocess_chat = AsyncMock(
return_value=([], [tokens_input([1])])
)
request = ResponsesRequest(
input="first",
instructions="initial instructions",
cache_salt="request-salt",
)
turn_messages = [
{"role": "user", "content": "first"},
{"role": "assistant", "content": "tool call"},
{"role": "tool", "content": "tool output"},
]
engine_inputs = await serving_responses_instance._render_next_turn(
request,
turn_messages,
)
assert engine_inputs[0]["prompt_token_ids"] == [8, 9]
render_request = (
serving_responses_instance.online_renderer.render_responses.call_args.args[
0
]
)
assert render_request.input == turn_messages
assert render_request.instructions is None
assert render_request.cache_salt == "request-salt"
@pytest.mark.asyncio
async def test_harmony_tool_followup_preserves_render_params(
self, serving_responses_instance, monkeypatch: pytest.MonkeyPatch
):
class ToolCallingHarmonyContext(HarmonyContext):
def __init__(self):
self._messages = []
self._needs_tool_call = True
def append_output(self, output) -> None:
pass
def need_builtin_tool_call(self) -> bool:
return self._needs_tool_call
async def call_tool(self):
self._needs_tool_call = False
return []
def append_tool_output(self, output) -> None:
self._messages.extend(output)
async def generate_output():
yield MagicMock()
context = ToolCallingHarmonyContext()
serving_responses_instance.engine_client.generate.side_effect = (
lambda *args, **kwargs: generate_output()
)
monkeypatch.setattr(
"vllm.renderers.online_renderer.render_for_completion",
lambda messages: [1, 2, 3, 4],
)
online_renderer = serving_responses_instance.online_renderer
online_renderer.renderer = SimpleNamespace(
tokenizer=SimpleNamespace(truncation_side="left")
)
serving_responses_instance.online_renderer.render_responses_harmony_messages = (
OnlineRenderer.render_responses_harmony_messages.__get__(online_renderer)
)
serving_responses_instance._extract_prompt_len = MagicMock(return_value=2)
async for _ in serving_responses_instance._generate_with_builtin_tools(
request_id="req",
engine_input=tokens_input([1], cache_salt="request-salt"),
sampling_params=MagicMock(),
context=context,
tok_params=TokenizeParams(
max_total_tokens=3,
max_output_tokens=1,
truncate_prompt_tokens=-1,
),
):
pass
assert serving_responses_instance.engine_client.generate.call_count == 2
followup_engine_input = (
serving_responses_instance.engine_client.generate.call_args_list[1].args[0]
)
assert followup_engine_input.get("cache_salt") == "request-salt"
assert followup_engine_input["prompt_token_ids"] == [3, 4]
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_initialize_tool_sessions( async def test_initialize_tool_sessions(
self, serving_responses_instance, mock_context, mock_exit_stack self, serving_responses_instance, mock_context, mock_exit_stack
@@ -151,7 +151,8 @@ async def _capture_responses(
request: ResponsesRequest, request: ResponsesRequest,
) -> CapturedRenderInputs: ) -> CapturedRenderInputs:
capture = RenderCapture(serving.online_renderer) capture = RenderCapture(serving.online_renderer)
await serving._make_request(request, prev_response=None) result = await serving.online_renderer.render_responses(request)
assert not isinstance(result, ErrorResponse), result
return capture.take() return capture.take()
@@ -3,15 +3,138 @@
"""Tests for the /render endpoints that expose prompt preprocessing.""" """Tests for the /render endpoints that expose prompt preprocessing."""
from http import HTTPStatus
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import httpx import httpx
import pytest import pytest
import pytest_asyncio import pytest_asyncio
from tests.utils import RemoteLaunchRenderServer from tests.utils import RemoteLaunchRenderServer
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.render.api_router import router
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.renderers.online_renderer import OnlineRenderer
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
def _build_responses_serving_render() -> ServingRender:
serving = ServingRender.__new__(ServingRender)
serving.model_config = SimpleNamespace(
max_model_len=100,
is_encoder_decoder=False,
)
serving.default_sampling_params = {}
serving.override_max_tokens = None
serving.tool_server = MagicMock()
serving.online_renderer = MagicMock()
serving.online_renderer.create_error_response = (
OnlineRenderer.create_error_response.__get__(serving.online_renderer)
)
serving.online_renderer.validate_chat_template = (
OnlineRenderer.validate_chat_template.__get__(serving.online_renderer)
)
serving.online_renderer.trust_request_chat_template = True
serving._check_model = AsyncMock(return_value=None)
return serving
@pytest.mark.skip_global_cleanup
def test_responses_render_route_is_registered():
assert any(route.path == "/v1/responses/render" for route in router.routes)
@pytest.mark.skip_global_cleanup
def test_responses_render_route_documents_not_implemented():
route = next(
route for route in router.routes if route.path == "/v1/responses/render"
)
assert HTTPStatus.NOT_IMPLEMENTED.value in route.responses
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_render_responses_returns_generate_request_without_stored_state():
serving = _build_responses_serving_render()
serving.online_renderer.render_responses = AsyncMock(
return_value=MagicMock(
messages=[{"role": "user", "content": "Test prompt"}],
engine_input={"prompt_token_ids": [7, 8, 9]},
)
)
request = ResponsesRequest(
model=MODEL_NAME,
input="Test prompt",
max_output_tokens=12,
stream=True,
cache_salt="request-salt",
priority=3,
kv_transfer_params={"do_remote_prefill": True},
ec_transfer_params={"remote": True},
)
response = await serving.render_responses_request(request)
assert response.token_ids == [7, 8, 9]
assert response.request_id == request.request_id
assert response.sampling_params.max_tokens == 12
assert response.model == MODEL_NAME
assert response.stream is True
assert response.cache_salt == "request-salt"
assert response.priority == 3
assert response.kv_transfer_params == {"do_remote_prefill": True}
assert response.ec_transfer_params == {"remote": True}
serving.online_renderer.render_responses.assert_awaited_once_with(
request,
previous_messages=None,
previous_response_outputs=None,
tool_server=serving.tool_server,
skip_mm_cache=True,
)
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_render_responses_rejects_previous_response_id():
serving = _build_responses_serving_render()
serving.online_renderer.render_responses = AsyncMock()
request = ResponsesRequest(
model=MODEL_NAME,
input="Test prompt",
previous_response_id="resp_previous",
)
response = await serving.render_responses_request(request)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
assert response.error.param == "previous_response_id"
serving.online_renderer.render_responses.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.skip_global_cleanup
async def test_render_responses_rejects_empty_token_ids():
serving = _build_responses_serving_render()
serving.online_renderer.render_responses = AsyncMock(
return_value=MagicMock(
messages=[],
engine_input={"prompt_token_ids": []},
)
)
response = await serving.render_responses_request(
ResponsesRequest(model=MODEL_NAME, input="Test prompt")
)
assert isinstance(response, ErrorResponse)
assert response.error.message == "No token_ids rendered"
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def server(): def server():
args: list[str] = ["--trust-request-chat-template"] args: list[str] = ["--trust-request-chat-template"]
@@ -28,6 +151,71 @@ async def client(server):
yield http_client yield http_client
@pytest.mark.asyncio
async def test_responses_render_basic(client):
response = await client.post(
"/v1/responses/render",
json={
"model": MODEL_NAME,
"input": "When should a Responses handler return an empty string?",
"max_output_tokens": 7,
},
)
assert response.status_code == 200
data = response.json()
assert data["model"] == MODEL_NAME
assert data["request_id"].startswith("resp_")
assert data["sampling_params"]["max_tokens"] == 7
assert data["token_ids"]
@pytest.mark.asyncio
async def test_responses_render_includes_prior_input_items(client):
current_turn = {
"role": "user",
"content": "Which color should I remember?",
}
current_turn_response = await client.post(
"/v1/responses/render",
json={"model": MODEL_NAME, "input": [current_turn]},
)
multi_turn_response = await client.post(
"/v1/responses/render",
json={
"model": MODEL_NAME,
"input": [
{"role": "user", "content": "Remember the color cobalt."},
{"role": "assistant", "content": "I will remember cobalt."},
current_turn,
],
},
)
assert current_turn_response.status_code == 200
assert multi_turn_response.status_code == 200
current_turn_token_ids = current_turn_response.json()["token_ids"]
multi_turn_token_ids = multi_turn_response.json()["token_ids"]
assert len(multi_turn_token_ids) > len(current_turn_token_ids)
@pytest.mark.asyncio
async def test_responses_render_rejects_previous_response_id_over_http(client):
response = await client.post(
"/v1/responses/render",
json={
"model": MODEL_NAME,
"input": "Continue the previous response.",
"previous_response_id": "resp_previous",
},
)
assert response.status_code == 400
error = response.json()["error"]
assert error["type"] == "invalid_request_error"
assert error["param"] == "previous_response_id"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_completion_render_basic(client): async def test_completion_render_basic(client):
"""Test basic completion render endpoint.""" """Test basic completion render endpoint."""
@@ -15,6 +15,7 @@ RENDER_PATHS = {
"/v1/completions/render", "/v1/completions/render",
"/v1/completions/derender", "/v1/completions/derender",
"/v1/messages/render", "/v1/messages/render",
"/v1/responses/render",
} }
GENERATE_PATH = "/inference/v1/generate" GENERATE_PATH = "/inference/v1/generate"
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH} SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
+6 -33
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING from typing import TYPE_CHECKING, Any
from fastapi import FastAPI from fastapi import FastAPI
@@ -60,6 +60,7 @@ async def init_generate_state(
args: "Namespace", args: "Namespace",
request_logger: RequestLogger | None, request_logger: RequestLogger | None,
supported_tasks: tuple["SupportedTask", ...], supported_tasks: tuple["SupportedTask", ...],
default_chat_template_kwargs: dict[str, Any],
): ):
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.chat_utils import load_chat_template
@@ -79,12 +80,6 @@ async def init_generate_state(
CohereServingChatV2 = None # type: ignore[assignment,misc] CohereServingChatV2 = None # type: ignore[assignment,misc]
else: else:
CohereServingChatV2 = None # type: ignore[assignment,misc] CohereServingChatV2 = None # type: ignore[assignment,misc]
from vllm.entrypoints.mcp.tool_server import (
DemoToolServer,
MCPToolServer,
ToolServer,
)
from vllm.entrypoints.openai.chat_completion.batch_serving import ( from vllm.entrypoints.openai.chat_completion.batch_serving import (
OpenAIServingChatBatch, OpenAIServingChatBatch,
) )
@@ -100,33 +95,11 @@ async def init_generate_state(
getattr(args, "fingerprint_value", None), getattr(args, "fingerprint_value", None),
) )
if args.tool_server == "demo":
tool_server: ToolServer | None = DemoToolServer()
assert isinstance(tool_server, DemoToolServer)
await tool_server.init_and_validate()
elif args.tool_server:
tool_server = MCPToolServer()
await tool_server.add_tool_server(args.tool_server)
else:
tool_server = None
resolved_chat_template = load_chat_template(args.chat_template) resolved_chat_template = load_chat_template(args.chat_template)
# Fold the dedicated ``--cohere-format`` CLI flag into the renderer's # Render endpoints are always backed by OnlineRenderer so that chat,
# default chat-template kwargs. The cohere renderer reads # completion, and Responses rendering work on both generate-mode and
# ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4 # render-only servers. Created in init_app_state.
# rendering; making this a first-class flag keeps the right format
# discoverable for ``vllm serve --tokenizer-mode cohere`` users
# without forcing them to hand-construct a JSON dict for
# ``--default-chat-template-kwargs``. Per-request overrides still
# take precedence (see ``merge_kwargs`` in
# ``ChatCompletionRequest.build_chat_params``).
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
if getattr(args, "cohere_format", None):
default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
# Render endpoints are always backed by OnlineRenderer so that
# /v1/chat/completions/render and /v1/completions/render work on both
# generate-mode and render-only servers. Created in init_app_state.
state.openai_serving_responses = ( state.openai_serving_responses = (
OpenAIServingResponses( OpenAIServingResponses(
@@ -139,7 +112,7 @@ async def init_generate_state(
return_tokens_as_token_ids=args.return_tokens_as_token_ids, return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice, enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser, tool_parser=args.tool_call_parser,
tool_server=tool_server, tool_server=state.tool_server,
reasoning_parser=args.structured_outputs_config.reasoning_parser, reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage, enable_force_include_usage=args.enable_force_include_usage,
@@ -8,6 +8,8 @@ from starlette.datastructures import State
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
@@ -63,6 +65,10 @@ async def init_app_state(
state.vllm_config = vllm_config state.vllm_config = vllm_config
state.args = args state.args = args
resolved_chat_template = load_chat_template(args.chat_template) resolved_chat_template = load_chat_template(args.chat_template)
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
state.tool_server = (
await init_tool_server(args) if "generate" in supported_tasks else None
)
# Merge default_mm_loras into the static lora_modules # Merge default_mm_loras into the static lora_modules
default_mm_loras = ( default_mm_loras = (
@@ -90,7 +96,7 @@ async def init_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser, tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser, reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs, default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack, log_error_stack=args.log_error_stack,
) )
state.online_renderer.warmup() state.online_renderer.warmup()
@@ -106,7 +112,7 @@ async def init_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser, tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser, reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs, default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack, log_error_stack=args.log_error_stack,
) )
@@ -116,7 +122,7 @@ async def init_app_state(
request_logger=request_logger, request_logger=request_logger,
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format, chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs, default_chat_template_kwargs=default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
) )
@@ -124,7 +130,12 @@ async def init_app_state(
from vllm.entrypoints.generate.api_router import init_generate_state from vllm.entrypoints.generate.api_router import init_generate_state
await init_generate_state( await init_generate_state(
engine_client, state, args, request_logger, supported_tasks engine_client,
state,
args,
request_logger,
supported_tasks,
default_chat_template_kwargs,
) )
from vllm.entrypoints.scale_out.factories import init_scale_out_state from vllm.entrypoints.scale_out.factories import init_scale_out_state
+10
View File
@@ -370,6 +370,16 @@ class FrontendArgs(BaseFrontendArgs):
return frontend_kwargs return frontend_kwargs
def resolve_default_chat_template_kwargs(
args: argparse.Namespace,
) -> dict[str, Any]:
"""Resolve renderer defaults, including the dedicated Cohere format flag."""
kwargs = dict(args.default_chat_template_kwargs or {})
if getattr(args, "cohere_format", None):
kwargs.setdefault("cohere_format", args.cohere_format)
return kwargs
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
"""Create the CLI argument parser used by the OpenAI API server. """Create the CLI argument parser used by the OpenAI API server.
@@ -6,6 +6,8 @@ from starlette.datastructures import State
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
from vllm.entrypoints.scale_out.factories import init_render_state from vllm.entrypoints.scale_out.factories import init_render_state
@@ -46,6 +48,8 @@ async def init_render_app_state(
renderer = renderer_from_config(vllm_config) renderer = renderer_from_config(vllm_config)
resolved_chat_template = load_chat_template(args.chat_template) resolved_chat_template = load_chat_template(args.chat_template)
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
state.tool_server = await init_tool_server(args)
state.online_renderer = OnlineRenderer( state.online_renderer = OnlineRenderer(
model_config=vllm_config.model_config, model_config=vllm_config.model_config,
@@ -58,7 +62,7 @@ async def init_render_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser, tool_parser=args.tool_call_parser,
reasoning_parser=args.reasoning_parser, reasoning_parser=args.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs, default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack, log_error_stack=args.log_error_stack,
) )
state.online_renderer.warmup() state.online_renderer.warmup()
@@ -74,7 +78,7 @@ async def init_render_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser, tool_parser=args.tool_call_parser,
reasoning_parser=args.reasoning_parser, reasoning_parser=args.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs, default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack, log_error_stack=args.log_error_stack,
) )
@@ -85,7 +89,7 @@ async def init_render_app_state(
request_logger=request_logger, request_logger=request_logger,
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format, chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs, default_chat_template_kwargs=default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template, trust_request_chat_template=args.trust_request_chat_template,
) )
+15
View File
@@ -12,6 +12,8 @@ from vllm.logger import init_logger
logger = init_logger(__name__) logger = init_logger(__name__)
if TYPE_CHECKING: if TYPE_CHECKING:
from argparse import Namespace
from mcp.types import ListToolsResult from mcp.types import ListToolsResult
@@ -232,3 +234,16 @@ class DemoToolServer(ToolServer):
if tool_name not in self.tools: if tool_name not in self.tools:
raise KeyError(f"Tool '{tool_name}' is not supported") raise KeyError(f"Tool '{tool_name}' is not supported")
yield self.tools[tool_name] yield self.tools[tool_name]
async def init_tool_server(args: "Namespace") -> ToolServer | None:
tool_server = getattr(args, "tool_server", None)
if tool_server == "demo":
demo_server = DemoToolServer()
await demo_server.init_and_validate()
return demo_server
if tool_server:
mcp_server = MCPToolServer()
await mcp_server.add_tool_server(tool_server)
return mcp_server
return None
+76 -252
View File
@@ -6,13 +6,11 @@ import time
from collections import deque from collections import deque
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, Sequence from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, Sequence
from contextlib import AsyncExitStack from contextlib import AsyncExitStack
from copy import copy
from http import HTTPStatus from http import HTTPStatus
from typing import Any, Final from typing import Any, Final, cast
from fastapi import Request from fastapi import Request
from openai.types.responses import ( from openai.types.responses import (
ResponseFunctionToolCall,
ResponseOutputItem, ResponseOutputItem,
ResponseOutputMessage, ResponseOutputMessage,
ResponseOutputText, ResponseOutputText,
@@ -20,15 +18,12 @@ from openai.types.responses import (
response_text_delta_event, response_text_delta_event,
) )
from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
from openai.types.responses.tool import Mcp, Tool
from openai_harmony import Message as OpenAIHarmonyMessage
from pydantic import TypeAdapter from pydantic import TypeAdapter
from vllm import envs from vllm import envs
from vllm.config.utils import replace from vllm.config.utils import replace
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import ( from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption, ChatTemplateContentFormatOption,
) )
from vllm.entrypoints.generate.base.protocol import ( from vllm.entrypoints.generate.base.protocol import (
@@ -38,24 +33,13 @@ from vllm.entrypoints.generate.base.protocol import (
from vllm.entrypoints.generate.base.serving import GenerateBaseServing from vllm.entrypoints.generate.base.serving import GenerateBaseServing
from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.parser.harmony_utils import (
build_harmony_preamble,
extract_instructions_from_messages,
get_user_message,
has_custom_tools,
render_for_completion,
)
from vllm.entrypoints.openai.responses.context import ( from vllm.entrypoints.openai.responses.context import (
ConversationContext, ConversationContext,
HarmonyContext, HarmonyContext,
ParsableContext, ParsableContext,
SimpleContext, SimpleContext,
) )
from vllm.entrypoints.openai.responses.harmony import ( from vllm.entrypoints.openai.responses.harmony import harmony_to_response_output
construct_harmony_previous_input_messages,
harmony_to_response_output,
response_input_to_harmony,
)
from vllm.entrypoints.openai.responses.protocol import ( from vllm.entrypoints.openai.responses.protocol import (
InputTokensDetails, InputTokensDetails,
OutputTokensDetails, OutputTokensDetails,
@@ -80,8 +64,6 @@ from vllm.entrypoints.openai.responses.streaming_events import (
) )
from vllm.entrypoints.openai.responses.utils import ( from vllm.entrypoints.openai.responses.utils import (
build_response_output_items, build_response_output_items,
construct_input_messages,
construct_tool_dicts,
extract_function_tool_names, extract_function_tool_names,
extract_tool_types, extract_tool_types,
) )
@@ -89,14 +71,19 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import GenerationError, VLLMValidationError from vllm.exceptions import GenerationError, VLLMValidationError
from vllm.inputs import EngineInput, tokens_input from vllm.inputs import EngineInput
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.logprobs import Logprob as SampleLogprob from vllm.logprobs import Logprob as SampleLogprob
from vllm.logprobs import SampleLogprobs from vllm.logprobs import SampleLogprobs
from vllm.lora.request import LoRARequest from vllm.lora.request import LoRARequest
from vllm.outputs import CompletionOutput from vllm.outputs import CompletionOutput
from vllm.parser import Parser, ParserManager from vllm.parser import Parser, ParserManager
from vllm.renderers.online_renderer import OnlineRenderer from vllm.renderers import TokenizeParams
from vllm.renderers.online_renderer import (
OnlineRenderer,
ResponsesPreviousMessages,
ResponsesRenderResult,
)
from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.tokenizers import TokenizerLike from vllm.tokenizers import TokenizerLike
from vllm.utils import random_uuid from vllm.utils import random_uuid
@@ -105,45 +92,6 @@ from vllm.utils.collection_utils import as_list
logger = init_logger(__name__) logger = init_logger(__name__)
def _extract_allowed_tools_from_mcp_requests(
tools: list[Tool],
) -> dict[str, list[str] | None]:
"""
Extract allowed_tools mapping from MCP tool requests.
Returns a dictionary mapping server_label to allowed_tools list.
Handles both list format and McpAllowedToolsMcpToolFilter object format.
Special handling:
- If allowed_tools is None, returns None (allows all tools)
- If allowed_tools contains "*", returns None (allows all tools)
- Otherwise, returns the list of specific tool names
This function can be reused for both harmony and non-harmony MCP calls.
"""
allowed_tools_map: dict[str, list[str] | None] = {}
for tool in tools:
if not isinstance(tool, Mcp):
continue
# allowed_tools can be a list or an object with tool_names
# Extract the actual list of tool names
allowed_tools_val = None
if tool.allowed_tools is not None:
if isinstance(tool.allowed_tools, list):
allowed_tools_val = tool.allowed_tools
elif hasattr(tool.allowed_tools, "tool_names"):
# It's an McpAllowedToolsMcpToolFilter object
allowed_tools_val = tool.allowed_tools.tool_names
# Normalize "*" to None (both mean "allow all tools")
if allowed_tools_val is not None and "*" in allowed_tools_val:
allowed_tools_val = None
allowed_tools_map[tool.server_label] = allowed_tools_val
return allowed_tools_map
class OpenAIServingResponses(GenerateBaseServing): class OpenAIServingResponses(GenerateBaseServing):
def __init__( def __init__(
self, self,
@@ -226,7 +174,7 @@ class OpenAIServingResponses(GenerateBaseServing):
# HACK(woosuk): This is a hack. We should use a better store. # HACK(woosuk): This is a hack. We should use a better store.
# FIXME: If enable_store=True, this may cause a memory leak since we # FIXME: If enable_store=True, this may cause a memory leak since we
# never remove messages from the store. # never remove messages from the store.
self.msg_store: dict[str, list[ChatCompletionMessageParam]] = {} self.msg_store: dict[str, ResponsesPreviousMessages] = {}
# HACK(wuhang): This is a hack. We should use a better store. # HACK(wuhang): This is a hack. We should use a better store.
# FIXME: If enable_store=True, this may cause a memory leak since we # FIXME: If enable_store=True, this may cause a memory leak since we
@@ -322,6 +270,33 @@ class OpenAIServingResponses(GenerateBaseServing):
) )
return None return None
async def _render_resolved_response_inputs(
self,
request: ResponsesRequest,
prev_response: ResponsesResponse | None,
*,
skip_mm_cache: bool = False,
) -> ResponsesRenderResult | ErrorResponse:
previous_messages: ResponsesPreviousMessages | None = None
previous_response_outputs: list[ResponseOutputItem] | None = None
if prev_response is not None:
stored_messages = (
self.msg_store[prev_response.id]
if self.use_harmony
else self.msg_store.get(prev_response.id)
)
if stored_messages is not None:
previous_messages = stored_messages.copy()
previous_response_outputs = list(prev_response.output)
return await self.online_renderer.render_responses(
request,
previous_messages=previous_messages,
previous_response_outputs=previous_response_outputs,
tool_server=self.tool_server,
skip_mm_cache=skip_mm_cache,
)
async def create_responses( async def create_responses(
self, self,
request: ResponsesRequest, request: ResponsesRequest,
@@ -374,12 +349,14 @@ class OpenAIServingResponses(GenerateBaseServing):
lora_request = self._maybe_get_adapters(request) lora_request = self._maybe_get_adapters(request)
model_name = self.models.model_name(lora_request) model_name = self.models.model_name(lora_request)
if self.use_harmony: render_result = await self._render_resolved_response_inputs(
messages, engine_inputs = self._make_request_with_harmony( request,
request, prev_response prev_response,
) )
else: if isinstance(render_result, ErrorResponse):
messages, engine_inputs = await self._make_request(request, prev_response) return render_result
messages = render_result.messages
engine_inputs = [render_result.engine_input]
request_metadata = RequestResponseMetadata(request_id=request.request_id) request_metadata = RequestResponseMetadata(request_id=request.request_id)
if raw_request: if raw_request:
@@ -508,6 +485,11 @@ class OpenAIServingResponses(GenerateBaseServing):
engine_input=engine_input, engine_input=engine_input,
sampling_params=sampling_params, sampling_params=sampling_params,
context=context, context=context,
tok_params=(
request.build_tok_params(self.model_config)
if isinstance(context, HarmonyContext)
else None
),
lora_request=lora_request, lora_request=lora_request,
priority=self._get_priority(request, raw_request), priority=self._get_priority(request, raw_request),
trace_headers=trace_headers, trace_headers=trace_headers,
@@ -599,60 +581,28 @@ class OpenAIServingResponses(GenerateBaseServing):
request_metadata, request_metadata,
) )
async def _make_request(
self,
request: ResponsesRequest,
prev_response: ResponsesResponse | None,
):
tool_dicts = construct_tool_dicts(
request.tools,
request.tool_choice,
exclude_tools_when_tool_choice_none=(
self.online_renderer.exclude_tools_when_tool_choice_none
),
)
# Construct the input messages.
messages = construct_input_messages(
request_instructions=request.instructions,
request_input=request.input,
prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,
prev_response_output=prev_response.output if prev_response else None,
)
chat_template_kwargs = self._effective_chat_template_kwargs(request)
_, engine_inputs = await self.online_renderer.preprocess_chat(
request,
messages,
default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=chat_template_kwargs,
tool_dicts=tool_dicts,
parser=self.parser,
)
return messages, engine_inputs
async def _render_next_turn( async def _render_next_turn(
self, self,
request: ResponsesRequest, request: ResponsesRequest,
messages: list[ResponseInputOutputItem], messages: list[ResponseInputOutputItem],
tool_dicts: list[dict[str, Any]] | None, ) -> list[EngineInput]:
parser: type[Parser] | None, render_request = request.model_copy(
chat_template: str | None, update={
chat_template_content_format: ChatTemplateContentFormatOption, "input": list(messages),
): "instructions": None,
new_messages = construct_input_messages( "previous_input_messages": None,
request_input=messages, "previous_response_id": None,
}
) )
chat_template_kwargs = self._effective_chat_template_kwargs(request) render_result = await self.online_renderer.render_responses(
_, engine_inputs = await self.online_renderer.preprocess_chat( render_request,
request, previous_messages=None,
new_messages, previous_response_outputs=None,
default_template=chat_template, tool_server=self.tool_server,
default_template_content_format=chat_template_content_format,
default_template_kwargs=chat_template_kwargs,
tool_dicts=tool_dicts,
parser=parser,
) )
return engine_inputs if isinstance(render_result, ErrorResponse):
raise VLLMValidationError(render_result.error.message)
return [render_result.engine_input]
async def _generate_with_builtin_tools( async def _generate_with_builtin_tools(
self, self,
@@ -660,6 +610,7 @@ class OpenAIServingResponses(GenerateBaseServing):
engine_input: EngineInput, engine_input: EngineInput,
sampling_params: SamplingParams, sampling_params: SamplingParams,
context: ConversationContext, context: ConversationContext,
tok_params: TokenizeParams | None = None,
lora_request: LoRARequest | None = None, lora_request: LoRARequest | None = None,
priority: int = 0, priority: int = 0,
trace_headers: Mapping[str, str] | None = None, trace_headers: Mapping[str, str] | None = None,
@@ -667,6 +618,7 @@ class OpenAIServingResponses(GenerateBaseServing):
reasoning_parser_kwargs: dict[str, Any] | None = None, reasoning_parser_kwargs: dict[str, Any] | None = None,
): ):
max_model_len = self.model_config.max_model_len max_model_len = self.model_config.max_model_len
cache_salt = cast(str | None, engine_input.get("cache_salt"))
orig_priority = priority orig_priority = priority
sub_request = 0 sub_request = 0
@@ -711,18 +663,19 @@ class OpenAIServingResponses(GenerateBaseServing):
# Create inputs for the next turn. # Create inputs for the next turn.
# Render the next prompt token ids and update sampling_params. # Render the next prompt token ids and update sampling_params.
if isinstance(context, HarmonyContext): if isinstance(context, HarmonyContext):
token_ids = context.render_for_completion() engine_input = self.online_renderer.render_responses_harmony_messages(
engine_input = tokens_input(token_ids) context.messages,
cache_salt=cache_salt,
tok_params=tok_params,
)
sampling_params.max_tokens = max_model_len - len(token_ids) sampling_params.max_tokens = max_model_len - self._extract_prompt_len(
engine_input
)
elif isinstance(context, ParsableContext): elif isinstance(context, ParsableContext):
(engine_input,) = await self._render_next_turn( (engine_input,) = await self._render_next_turn(
context.request, context.request,
context.response_messages, context.response_messages,
context.tool_dicts,
context.parser_cls,
context.chat_template,
context.chat_template_content_format,
) )
sampling_params.max_tokens = get_max_tokens( sampling_params.max_tokens = get_max_tokens(
@@ -740,28 +693,6 @@ class OpenAIServingResponses(GenerateBaseServing):
priority = orig_priority - 1 priority = orig_priority - 1
sub_request += 1 sub_request += 1
def _make_request_with_harmony(
self,
request: ResponsesRequest,
prev_response: ResponsesResponse | None,
):
if self.parser is not None:
# HarmonyParser doesn't need chat_template_kwargs
# TODO: Unify adjust_request() call with non-harmony branch
self.parser(
self.renderer.get_tokenizer(),
request.tools,
model_config=self.model_config,
).adjust_request(request=request)
arrival_time = time.time()
messages = self._construct_input_messages_with_harmony(request, prev_response)
prompt_token_ids = render_for_completion(messages)
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
engine_input["arrival_time"] = arrival_time
return messages, [engine_input]
async def _initialize_tool_sessions( async def _initialize_tool_sessions(
self, self,
request: ResponsesRequest, request: ResponsesRequest,
@@ -1104,113 +1035,6 @@ class OpenAIServingResponses(GenerateBaseServing):
) )
] ]
def _get_harmony_builtin_tool_descriptions(
self, request: ResponsesRequest, tool_types: set[str]
) -> dict[str, str | None]:
# Extract allowed_tools from MCP tool requests
allowed_tools_map = _extract_allowed_tools_from_mcp_requests(request.tools)
# Get filtered tool descriptions first.
# If get_tool_description returns None (due to filtering), the tool is disabled.
browser_description = (
self.tool_server.get_tool_description(
"browser", allowed_tools_map.get("web_search_preview")
)
if "web_search_preview" in tool_types
and self.tool_server is not None
and self.tool_server.has_tool("browser")
else None
)
python_description = (
self.tool_server.get_tool_description(
"python", allowed_tools_map.get("code_interpreter")
)
if "code_interpreter" in tool_types
and self.tool_server is not None
and self.tool_server.has_tool("python")
else None
)
container_description = (
self.tool_server.get_tool_description(
"container", allowed_tools_map.get("container")
)
if "container" in tool_types
and self.tool_server is not None
and self.tool_server.has_tool("container")
else None
)
return {
"browser_description": browser_description,
"python_description": python_description,
"container_description": container_description,
}
def _construct_input_messages_with_harmony(
self,
request: ResponsesRequest,
prev_response: ResponsesResponse | None,
) -> list[OpenAIHarmonyMessage]:
messages: list[OpenAIHarmonyMessage] = []
request_input = request.input
if prev_response is None:
# New conversation.
tool_types = extract_tool_types(request.tools)
with_custom_tools = has_custom_tools(tool_types)
instructions = request.instructions
if instructions is None and isinstance(request_input, list):
instructions, request_input = extract_instructions_from_messages(
request_input
)
tool_descriptions = self._get_harmony_builtin_tool_descriptions(
request, tool_types
)
tools = request.tools if with_custom_tools else None
messages.extend(
build_harmony_preamble(
instructions=instructions,
tools=tools,
reasoning_effort=(
request.reasoning.effort if request.reasoning else None
),
with_custom_tools=with_custom_tools,
**tool_descriptions,
)
)
messages += construct_harmony_previous_input_messages(request)
else:
# Continue the previous conversation.
# FIXME(woosuk): Currently, request params like reasoning and
# instructions are ignored.
prev_msgs = self.msg_store[prev_response.id]
messages.extend(prev_msgs)
# Append the new input.
# Responses API supports simple text inputs without chat format.
if isinstance(request_input, str):
# Skip empty string input when previous_input_messages supplies
# the full conversation history --- an empty trailing user message
# confuses the model into thinking nothing was sent.
if request_input or not request.previous_input_messages:
messages.append(get_user_message(request_input))
else:
if prev_response is not None:
prev_outputs = copy(prev_response.output)
else:
prev_outputs = []
for response_msg in request_input:
new_msg = response_input_to_harmony(response_msg, prev_outputs)
if new_msg is not None:
messages.append(new_msg)
# User passes in a tool call request and its output. We need
# to add the tool call request to prev_outputs so that
# response_input_to_harmony can find the tool call request when
# parsing the tool call output.
if isinstance(response_msg, ResponseFunctionToolCall):
prev_outputs.append(response_msg)
return messages
async def _run_background_request_stream( async def _run_background_request_stream(
self, self,
request: ResponsesRequest, request: ResponsesRequest,
+1
View File
@@ -31,6 +31,7 @@ def init_render_state(
state.openai_serving_models, state.openai_serving_models,
state.online_renderer, state.online_renderer,
request_logger=request_logger, request_logger=request_logger,
tool_server=state.tool_server,
) )
state.serving_derender = ServingDerender( state.serving_derender = ServingDerender(
@@ -8,6 +8,7 @@ from fastapi.responses import JSONResponse
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger from vllm.logger import init_logger
@@ -95,3 +96,25 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
return JSONResponse(content=result.model_dump(), status_code=result.error.code) return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=[item.model_dump() for item in result]) return JSONResponse(content=[item.model_dump() for item in result])
@router.post(
"/v1/responses/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def render_responses(request: ResponsesRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Responses Render API")
result = await handler.render_responses_request(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
@@ -8,6 +10,7 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry, OpenAIModelRegistry,
OpenAIServingModels, OpenAIServingModels,
) )
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import ( from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
extract_mm_features, extract_mm_features,
) )
@@ -31,6 +34,9 @@ from vllm.utils import random_uuid
logger = init_logger(__name__) logger = init_logger(__name__)
if TYPE_CHECKING:
from vllm.entrypoints.mcp.tool_server import ToolServer
class ServingRender(BaseServing): class ServingRender(BaseServing):
def __init__( def __init__(
@@ -39,6 +45,7 @@ class ServingRender(BaseServing):
online_renderer: "OnlineRenderer", online_renderer: "OnlineRenderer",
*, *,
request_logger: RequestLogger | None = None, request_logger: RequestLogger | None = None,
tool_server: "ToolServer | None" = None,
) -> None: ) -> None:
super().__init__( super().__init__(
models=models, models=models,
@@ -47,6 +54,7 @@ class ServingRender(BaseServing):
) )
self.online_renderer = online_renderer self.online_renderer = online_renderer
self.tool_server = tool_server
self._merge_inline_system = ( self._merge_inline_system = (
AnthropicServingMessages._detect_merge_inline_system( AnthropicServingMessages._detect_merge_inline_system(
@@ -229,6 +237,64 @@ class ServingRender(BaseServing):
return generate_requests return generate_requests
async def render_responses_request(
self,
request: ResponsesRequest,
) -> GenerateRequest | ErrorResponse:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if request.previous_response_id is not None:
return self.create_error_response(
message=(
"previous_response_id is not supported by the stateless "
"render endpoint."
),
err_type="invalid_request_error",
param="previous_response_id",
)
result = await self.online_renderer.render_responses(
request,
previous_messages=None,
previous_response_outputs=None,
tool_server=self.tool_server,
skip_mm_cache=True,
)
if isinstance(result, ErrorResponse):
return result
engine_input = result.engine_input
prompt_components = extract_prompt_components(self.model_config, engine_input)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_output_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
return GenerateRequest(
request_id=request.request_id,
token_ids=list(token_ids),
features=self._extract_mm_features(engine_input),
sampling_params=params,
model=request.model,
stream=bool(request.stream),
cache_salt=request.cache_salt,
priority=request.priority,
kv_transfer_params=request.kv_transfer_params,
ec_transfer_params=request.ec_transfer_params,
token_offsets=engine_input.get("prompt_token_offsets"),
)
def _placeholder_metadata_fields(self, modality: str) -> set[str]: def _placeholder_metadata_fields(self, modality: str) -> set[str]:
"""Fields the EC consumer still requires after embeddings transfer.""" """Fields the EC consumer still requires after embeddings transfer."""
if self._placeholder_metadata_parser_failed: if self._placeholder_metadata_parser_failed:
+267 -2
View File
@@ -1,13 +1,19 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Sequence from collections.abc import Sequence
from dataclasses import dataclass
from http import HTTPStatus from http import HTTPStatus
from typing import Any from typing import TYPE_CHECKING, Any, TypeAlias
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
from openai.types.responses.tool import Mcp, Tool
from openai_harmony import Message as OpenAIMessage from openai_harmony import Message as OpenAIMessage
from openai_harmony import ToolNamespaceConfig
from vllm.config import ModelConfig from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import ( from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption, ChatTemplateContentFormatOption,
ConversationMessage, ConversationMessage,
) )
@@ -19,12 +25,24 @@ from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest, CompletionRequest,
) )
from vllm.entrypoints.openai.parser.harmony_utils import ( from vllm.entrypoints.openai.parser.harmony_utils import (
BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
build_harmony_preamble, build_harmony_preamble,
extract_instructions_from_messages, extract_instructions_from_messages,
get_user_message,
has_custom_tools,
parse_chat_inputs_to_harmony_messages, parse_chat_inputs_to_harmony_messages,
render_for_completion, render_for_completion,
) )
from vllm.entrypoints.openai.responses.harmony import (
construct_harmony_previous_input_messages,
response_input_to_harmony,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.openai.responses.utils import (
construct_input_messages,
construct_tool_dicts,
extract_tool_types,
)
from vllm.entrypoints.serve import create_error_response from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.request_logger import RequestLogger
@@ -32,11 +50,12 @@ from vllm.inputs import (
EngineInput, EngineInput,
PromptType, PromptType,
SingletonPrompt, SingletonPrompt,
TokensPrompt,
tokens_input, tokens_input,
) )
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs from vllm.renderers import BaseRenderer, ChatParams, TokenizeParams, merge_kwargs
from vllm.renderers.inputs.preprocess import ( from vllm.renderers.inputs.preprocess import (
parse_model_prompt, parse_model_prompt,
prompt_to_seq, prompt_to_seq,
@@ -46,6 +65,39 @@ from vllm.utils.mistral import mt as _mt
logger = init_logger(__name__) logger = init_logger(__name__)
if TYPE_CHECKING:
from vllm.entrypoints.mcp.tool_server import ToolServer
ResponsesPreviousMessages: TypeAlias = (
list[ChatCompletionMessageParam] | list[OpenAIMessage]
)
@dataclass(frozen=True)
class ResponsesRenderResult:
messages: ResponsesPreviousMessages
engine_input: EngineInput
def _extract_allowed_tools_from_mcp_requests(
tools: list[Tool],
) -> dict[str, list[str] | None]:
allowed_tools_map: dict[str, list[str] | None] = {}
for tool in tools:
if not isinstance(tool, Mcp):
continue
allowed_tools = None
if isinstance(tool.allowed_tools, list):
allowed_tools = tool.allowed_tools
elif tool.allowed_tools is not None:
allowed_tools = tool.allowed_tools.tool_names
if allowed_tools is not None and "*" in allowed_tools:
allowed_tools = None
allowed_tools_map[tool.server_label] = allowed_tools
return allowed_tools_map
def _reused_prompt_token_ids(request: Any) -> list[int] | None: def _reused_prompt_token_ids(request: Any) -> list[int] | None:
"""Pop prompt token ids forwarded for decode-side reuse, if any. """Pop prompt token ids forwarded for decode-side reuse, if any.
@@ -217,6 +269,219 @@ class OnlineRenderer:
return conversation, engine_inputs return conversation, engine_inputs
async def render_responses(
self,
request: ResponsesRequest,
*,
previous_messages: ResponsesPreviousMessages | None = None,
previous_response_outputs: list[ResponseOutputItem] | None = None,
tool_server: "ToolServer | None" = None,
skip_mm_cache: bool = False,
) -> ResponsesRenderResult | ErrorResponse:
"""Render a Responses request using only explicitly supplied history."""
template_error = self.validate_chat_template(
request_chat_template=None,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if template_error is not None:
return template_error
if self.use_harmony:
return self._render_responses_with_harmony(
request,
previous_messages=previous_messages,
previous_response_outputs=previous_response_outputs,
tool_server=tool_server,
)
if previous_messages is not None and any(
isinstance(message, OpenAIMessage) for message in previous_messages
):
return self.create_error_response(
"Non-Harmony Responses history must use chat messages.",
err_type="invalid_request_error",
param="previous_response_id",
)
tool_dicts = construct_tool_dicts(
request.tools,
request.tool_choice,
exclude_tools_when_tool_choice_none=(
self.exclude_tools_when_tool_choice_none
),
)
messages = construct_input_messages(
request_instructions=request.instructions,
request_input=request.input,
prev_msg=list(previous_messages) if previous_messages is not None else None,
prev_response_output=(
list(previous_response_outputs)
if previous_response_outputs is not None
else None
),
)
chat_template_kwargs = (
request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)
_, engine_inputs = await self.preprocess_chat(
request,
messages,
default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=chat_template_kwargs,
tool_dicts=tool_dicts,
parser=self.parser,
skip_mm_cache=skip_mm_cache,
)
return self._responses_render_result(messages, engine_inputs)
def _render_responses_with_harmony(
self,
request: ResponsesRequest,
*,
previous_messages: ResponsesPreviousMessages | None,
previous_response_outputs: list[ResponseOutputItem] | None,
tool_server: "ToolServer | None",
) -> ResponsesRenderResult | ErrorResponse:
if self.parser is not None:
# HarmonyParser doesn't need chat_template_kwargs
# TODO: Unify adjust_request() call with non-harmony branch
self.parser(
self.renderer.get_tokenizer(),
request.tools,
model_config=self.model_config,
).adjust_request(request=request)
if previous_messages is not None and any(
not isinstance(message, OpenAIMessage) for message in previous_messages
):
return self.create_error_response(
"Harmony Responses history must use Harmony messages.",
err_type="invalid_request_error",
param="previous_response_id",
)
messages: list[OpenAIMessage] = []
request_input = request.input
if previous_messages is None:
tool_types = extract_tool_types(request.tools)
with_custom_tools = has_custom_tools(tool_types)
instructions = request.instructions
if instructions is None and isinstance(request_input, list):
instructions, request_input = extract_instructions_from_messages(
request_input
)
descriptions = self._get_harmony_builtin_tool_descriptions(
request.tools,
tool_types,
tool_server,
)
messages.extend(
build_harmony_preamble(
instructions=instructions,
tools=request.tools if with_custom_tools else None,
reasoning_effort=(
request.reasoning.effort if request.reasoning else None
),
with_custom_tools=with_custom_tools,
**descriptions,
)
)
messages.extend(construct_harmony_previous_input_messages(request))
else:
messages.extend(previous_messages)
previous_outputs = list(previous_response_outputs or ())
try:
if isinstance(request_input, str):
if request_input or not request.previous_input_messages:
messages.append(get_user_message(request_input))
else:
for response_message in request_input:
new_message = response_input_to_harmony(
response_message,
previous_outputs,
)
if new_message is not None:
messages.append(new_message)
if isinstance(response_message, ResponseFunctionToolCall):
previous_outputs.append(response_message)
except ValueError as exc:
return self.create_error_response(
str(exc),
err_type="invalid_request_error",
param="input",
)
return ResponsesRenderResult(
messages=messages,
engine_input=self.render_responses_harmony_messages(
messages,
cache_salt=request.cache_salt,
tok_params=request.build_tok_params(self.model_config),
),
)
def _get_harmony_builtin_tool_descriptions(
self,
tools: list[Tool],
tool_types: set[str],
tool_server: "ToolServer | None",
) -> dict[str, ToolNamespaceConfig | None]:
allowed_tools = _extract_allowed_tools_from_mcp_requests(tools)
descriptions: dict[str, ToolNamespaceConfig | None] = {}
for server_name, request_name in BUILTIN_TOOL_TO_MCP_SERVER_LABEL.items():
description = (
tool_server.get_tool_description(
server_name,
allowed_tools.get(request_name),
)
if request_name in tool_types
and tool_server is not None
and tool_server.has_tool(server_name)
else None
)
descriptions[f"{server_name}_description"] = description
return descriptions
def render_responses_harmony_messages(
self,
messages: list[OpenAIMessage],
*,
cache_salt: str | None,
tok_params: TokenizeParams | None = None,
) -> EngineInput:
arrival_time = time.time()
prompt = TokensPrompt(prompt_token_ids=render_for_completion(messages))
if tok_params is not None:
tok_params.apply_post_tokenization(
self.renderer.tokenizer,
prompt,
)
engine_input = tokens_input(prompt["prompt_token_ids"], cache_salt=cache_salt)
engine_input["arrival_time"] = arrival_time
return engine_input
def _responses_render_result(
self,
messages: list[ChatCompletionMessageParam],
engine_inputs: list[EngineInput],
) -> ResponsesRenderResult | ErrorResponse:
if len(engine_inputs) != 1:
return self.create_error_response(
f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
)
return ResponsesRenderResult(
messages=messages,
engine_input=engine_inputs[0],
)
def _make_request_with_harmony( def _make_request_with_harmony(
self, self,
request: ChatCompletionRequest, request: ChatCompletionRequest,