[Frontend] Add stateless /v1/responses/render endpoint (#50195)
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
5e6f6a8ed4
commit
6a2a2bb02b
@@ -2,9 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.entrypoints.launchers.cli_args as cli_args_module
|
||||
from tests.utils import VLLM_PATH
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
make_arg_parser,
|
||||
@@ -332,6 +334,26 @@ def test_default_chat_template_kwargs_default_none(serve_parser):
|
||||
assert args.default_chat_template_kwargs is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"default_kwargs,cohere_format,expected",
|
||||
[
|
||||
(None, "cmd3", {"cohere_format": "cmd3"}),
|
||||
(
|
||||
{"cohere_format": "cmd4", "custom": True},
|
||||
"cmd3",
|
||||
{"cohere_format": "cmd4", "custom": True},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, expected):
|
||||
args = SimpleNamespace(
|
||||
default_chat_template_kwargs=default_kwargs,
|
||||
cohere_format=cohere_format,
|
||||
)
|
||||
|
||||
assert cli_args_module.resolve_default_chat_template_kwargs(args) == expected
|
||||
|
||||
|
||||
def test_default_chat_template_kwargs_invalid_json(serve_parser):
|
||||
"""Ensure invalid JSON raises an error"""
|
||||
with pytest.raises(SystemExit):
|
||||
|
||||
@@ -4,14 +4,19 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from openai import OpenAI
|
||||
from openai_harmony import Message, ToolDescription, ToolNamespaceConfig
|
||||
|
||||
import vllm.entrypoints.mcp.tool_server as tool_server_module
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.mcp.tool_server import (
|
||||
MCPToolServer,
|
||||
init_tool_server,
|
||||
post_process_tools_description,
|
||||
)
|
||||
|
||||
@@ -39,6 +44,25 @@ _PYTHON_TOOL_INSTRUCTION = (
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_init_tool_server_without_configuration():
|
||||
args = SimpleNamespace(tool_server=None)
|
||||
|
||||
assert await init_tool_server(args) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_init_tool_server_with_mcp_url(monkeypatch: pytest.MonkeyPatch):
|
||||
server = MagicMock()
|
||||
server.add_tool_server = AsyncMock()
|
||||
monkeypatch.setattr(tool_server_module, "MCPToolServer", lambda: server)
|
||||
|
||||
result = await init_tool_server(SimpleNamespace(tool_server="http://mcp.test"))
|
||||
|
||||
assert result is server
|
||||
server.add_tool_server.assert_awaited_once_with("http://mcp.test")
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
class TestMCPToolServerUnit:
|
||||
"""Test MCP tool description processing and filtering.
|
||||
|
||||
@@ -2,12 +2,16 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import AsyncExitStack
|
||||
from unittest.mock import MagicMock
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
ResponseOutputItemDoneEvent,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
ResponseReasoningItem,
|
||||
ResponseReasoningTextDeltaEvent,
|
||||
ResponseReasoningTextDoneEvent,
|
||||
@@ -23,6 +27,8 @@ from openai.types.responses.tool import (
|
||||
Mcp,
|
||||
Tool,
|
||||
)
|
||||
from openai_harmony import Message as OpenAIHarmonyMessage
|
||||
from openai_harmony import Role
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
@@ -32,7 +38,11 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
RequestResponseMetadata,
|
||||
)
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.responses.context import ConversationContext, SimpleContext
|
||||
from vllm.entrypoints.openai.responses.context import (
|
||||
ConversationContext,
|
||||
HarmonyContext,
|
||||
SimpleContext,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponseCreatedEvent,
|
||||
ResponseRawMessageAndToken,
|
||||
@@ -42,7 +52,6 @@ from vllm.entrypoints.openai.responses.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.serving import (
|
||||
OpenAIServingResponses,
|
||||
_extract_allowed_tools_from_mcp_requests,
|
||||
extract_tool_types,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.streaming_events import (
|
||||
@@ -52,8 +61,23 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.parser.harmony import Segment
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.online_renderer import (
|
||||
OnlineRenderer,
|
||||
_extract_allowed_tools_from_mcp_requests,
|
||||
)
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def _new_online_renderer() -> OnlineRenderer:
|
||||
renderer = OnlineRenderer.__new__(OnlineRenderer)
|
||||
renderer.exclude_tools_when_tool_choice_none = False
|
||||
renderer.parser = None
|
||||
renderer.trust_request_chat_template = True
|
||||
return renderer
|
||||
|
||||
|
||||
class MockConversationContext(ConversationContext):
|
||||
"""Mock conversation context for testing"""
|
||||
@@ -205,6 +229,325 @@ def test_response_created_event_uses_public_json_schema_alias() -> None:
|
||||
assert event.response.text.format.model_dump(by_alias=True)["schema"] == schema
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_renders_non_harmony_responses_with_explicit_history():
|
||||
renderer = _new_online_renderer()
|
||||
renderer.use_harmony = False
|
||||
renderer.chat_template = "server-template"
|
||||
renderer.chat_template_content_format = "string"
|
||||
renderer.default_chat_template_kwargs = {"server_default": "kept"}
|
||||
renderer.parser = None
|
||||
renderer.preprocess_chat = AsyncMock(
|
||||
return_value=(
|
||||
[],
|
||||
[tokens_input([11, 12, 13], cache_salt="request-salt")],
|
||||
)
|
||||
)
|
||||
previous_messages = [
|
||||
{"role": "system", "content": "old instructions"},
|
||||
{"role": "user", "content": "first"},
|
||||
]
|
||||
previous_outputs = [
|
||||
ResponseOutputMessage(
|
||||
id="msg_1",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
annotations=[],
|
||||
text="prior answer",
|
||||
type="output_text",
|
||||
)
|
||||
],
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
)
|
||||
]
|
||||
request = ResponsesRequest(
|
||||
input="next",
|
||||
instructions="new instructions",
|
||||
tools=[],
|
||||
cache_salt="request-salt",
|
||||
chat_template_kwargs={"request_default": "kept"},
|
||||
)
|
||||
|
||||
result = await renderer.render_responses(
|
||||
request,
|
||||
previous_messages=previous_messages,
|
||||
previous_response_outputs=previous_outputs,
|
||||
)
|
||||
|
||||
assert not isinstance(result, ErrorResponse)
|
||||
assert result.messages == [
|
||||
{"role": "system", "content": "new instructions"},
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": "prior answer"},
|
||||
{"role": "user", "content": "next"},
|
||||
]
|
||||
assert result.engine_input["prompt_token_ids"] == [11, 12, 13]
|
||||
assert result.engine_input["cache_salt"] == "request-salt"
|
||||
assert previous_messages[0]["content"] == "old instructions"
|
||||
assert previous_outputs[0].content[0].text == "prior answer"
|
||||
|
||||
preprocess_kwargs = renderer.preprocess_chat.call_args.kwargs
|
||||
assert preprocess_kwargs["default_template"] == "server-template"
|
||||
assert preprocess_kwargs["default_template_content_format"] == "string"
|
||||
assert preprocess_kwargs["default_template_kwargs"]["server_default"] == "kept"
|
||||
assert preprocess_kwargs["default_template_kwargs"]["request_default"] == "kept"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_untrusted_responses_chat_template():
|
||||
renderer = _new_online_renderer()
|
||||
renderer.use_harmony = False
|
||||
renderer.chat_template = "server-template"
|
||||
renderer.chat_template_content_format = "string"
|
||||
renderer.default_chat_template_kwargs = {}
|
||||
renderer.exclude_tools_when_tool_choice_none = False
|
||||
renderer.trust_request_chat_template = False
|
||||
renderer.parser = None
|
||||
renderer.preprocess_chat = AsyncMock(
|
||||
return_value=([], [tokens_input([1])]),
|
||||
)
|
||||
request = ResponsesRequest(
|
||||
input="hello",
|
||||
chat_template_kwargs={"chat_template": "{{ messages }}"},
|
||||
)
|
||||
|
||||
result = await renderer.render_responses(request)
|
||||
|
||||
assert isinstance(result, ErrorResponse)
|
||||
assert result.error.code == 400
|
||||
assert "untrusted chat template" in result.error.message.lower()
|
||||
renderer.preprocess_chat.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_harmony_history_for_non_harmony_model():
|
||||
renderer = _new_online_renderer()
|
||||
renderer.use_harmony = False
|
||||
request = ResponsesRequest(input="next")
|
||||
|
||||
result = await renderer.render_responses(
|
||||
request,
|
||||
previous_messages=[
|
||||
OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first")
|
||||
],
|
||||
)
|
||||
|
||||
assert isinstance(result, ErrorResponse)
|
||||
assert result.error.param == "previous_response_id"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_rejects_chat_history_for_harmony_model():
|
||||
renderer = _new_online_renderer()
|
||||
renderer.use_harmony = True
|
||||
request = ResponsesRequest(input="next")
|
||||
|
||||
result = await renderer.render_responses(
|
||||
request,
|
||||
previous_messages=[{"role": "user", "content": "first"}],
|
||||
)
|
||||
|
||||
assert isinstance(result, ErrorResponse)
|
||||
assert result.error.param == "previous_response_id"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_adjusts_harmony_tool_choice(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
class StubParser:
|
||||
def __init__(self, tokenizer, tools, *, model_config):
|
||||
pass
|
||||
|
||||
def adjust_request(self, request):
|
||||
request.cache_salt = "adjusted"
|
||||
return request
|
||||
|
||||
renderer = _new_online_renderer()
|
||||
renderer.use_harmony = True
|
||||
renderer.parser = StubParser
|
||||
renderer.model_config = SimpleNamespace(max_model_len=100)
|
||||
tokenizer = SimpleNamespace(truncation_side="left")
|
||||
renderer.renderer = SimpleNamespace(
|
||||
tokenizer=tokenizer,
|
||||
get_tokenizer=lambda: tokenizer,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.renderers.online_renderer.render_for_completion",
|
||||
lambda messages: [1],
|
||||
)
|
||||
request = ResponsesRequest(
|
||||
input="next",
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
}
|
||||
],
|
||||
tool_choice={"type": "function", "name": "lookup"},
|
||||
)
|
||||
|
||||
result = await renderer.render_responses(request)
|
||||
|
||||
assert not isinstance(result, ErrorResponse)
|
||||
assert result.engine_input["cache_salt"] == "adjusted"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
renderer = _new_online_renderer()
|
||||
renderer.use_harmony = True
|
||||
renderer.model_config = SimpleNamespace(max_model_len=5)
|
||||
renderer.renderer = SimpleNamespace(
|
||||
tokenizer=SimpleNamespace(truncation_side="left")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.renderers.online_renderer.render_for_completion",
|
||||
lambda messages: [1, 2, 3, 4, 5, 6],
|
||||
)
|
||||
request = ResponsesRequest(
|
||||
input="hello",
|
||||
max_output_tokens=2,
|
||||
truncation="auto",
|
||||
)
|
||||
|
||||
result = await renderer.render_responses(request)
|
||||
|
||||
assert not isinstance(result, ErrorResponse)
|
||||
assert result.engine_input["prompt_token_ids"] == [4, 5, 6]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_renders_harmony_continuation_with_call_linkage(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
renderer = _new_online_renderer()
|
||||
renderer.use_harmony = True
|
||||
renderer.model_config = SimpleNamespace(max_model_len=100)
|
||||
renderer.renderer = SimpleNamespace(
|
||||
tokenizer=SimpleNamespace(truncation_side="left")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.renderers.online_renderer.render_for_completion",
|
||||
lambda messages: [1],
|
||||
)
|
||||
previous_messages = [
|
||||
OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first"),
|
||||
]
|
||||
previous_outputs = [
|
||||
ResponseFunctionToolCall(
|
||||
arguments='{"city":"Paris"}',
|
||||
call_id="call_1",
|
||||
name="lookup_weather",
|
||||
type="function_call",
|
||||
)
|
||||
]
|
||||
request = ResponsesRequest(
|
||||
input=[
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_1",
|
||||
"output": "sunny",
|
||||
}
|
||||
],
|
||||
instructions="replacement instructions are ignored on continuations",
|
||||
reasoning={"effort": "high"},
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "lookup_weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
}
|
||||
],
|
||||
cache_salt="request-salt",
|
||||
)
|
||||
|
||||
result = await renderer.render_responses(
|
||||
request,
|
||||
previous_messages=previous_messages,
|
||||
previous_response_outputs=previous_outputs,
|
||||
)
|
||||
|
||||
assert not isinstance(result, ErrorResponse)
|
||||
assert result.messages[0] is previous_messages[0]
|
||||
tool_output = result.messages[-1]
|
||||
assert tool_output.author.role == "tool"
|
||||
assert tool_output.author.name == "functions.lookup_weather"
|
||||
assert tool_output.channel == "commentary"
|
||||
assert tool_output.recipient == "assistant"
|
||||
assert result.engine_input["cache_salt"] == "request-salt"
|
||||
assert "arrival_time" in result.engine_input
|
||||
assert previous_messages == [
|
||||
OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first"),
|
||||
]
|
||||
assert previous_outputs[0].call_id == "call_1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_returns_typed_error_for_bad_harmony_call_reference():
|
||||
renderer = _new_online_renderer()
|
||||
renderer.use_harmony = True
|
||||
request = ResponsesRequest(
|
||||
input=[
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "missing",
|
||||
"output": "orphaned",
|
||||
}
|
||||
],
|
||||
tools=[],
|
||||
)
|
||||
|
||||
result = await renderer.render_responses(request)
|
||||
|
||||
assert isinstance(result, ErrorResponse)
|
||||
assert result.error.type == "invalid_request_error"
|
||||
assert result.error.param == "input"
|
||||
assert "No call message found for missing" in result.error.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_online_renderer_preserves_missing_harmony_builtin_tool_behavior(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
renderer = _new_online_renderer()
|
||||
renderer.use_harmony = True
|
||||
renderer.model_config = SimpleNamespace(max_model_len=100)
|
||||
renderer.renderer = SimpleNamespace(
|
||||
tokenizer=SimpleNamespace(truncation_side="left")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.renderers.online_renderer.render_for_completion",
|
||||
lambda messages: [1],
|
||||
)
|
||||
request = ResponsesRequest(
|
||||
input="search",
|
||||
tools=[{"type": "web_search_preview"}],
|
||||
)
|
||||
|
||||
result = await renderer.render_responses(request, tool_server=None)
|
||||
|
||||
assert not isinstance(result, ErrorResponse)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"engine_inputs",
|
||||
[[], [tokens_input([1]), tokens_input([2])]],
|
||||
)
|
||||
def test_responses_render_result_rejects_non_single_prompt(engine_inputs):
|
||||
renderer = _new_online_renderer()
|
||||
|
||||
result = renderer._responses_render_result([], engine_inputs)
|
||||
|
||||
assert isinstance(result, ErrorResponse)
|
||||
assert f"got {len(engine_inputs)}" in result.error.message
|
||||
|
||||
|
||||
class TestInitializeToolSessions:
|
||||
"""Test class for _initialize_tool_sessions method"""
|
||||
|
||||
@@ -240,6 +583,139 @@ class TestInitializeToolSessions:
|
||||
|
||||
return instance
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stateful_harmony_delegates_named_tool_choice_to_renderer(
|
||||
self,
|
||||
serving_responses_instance,
|
||||
):
|
||||
serving_responses_instance.use_harmony = True
|
||||
serving_responses_instance.online_renderer.render_responses = AsyncMock(
|
||||
return_value=SimpleNamespace(
|
||||
messages=[],
|
||||
engine_input=tokens_input([1]),
|
||||
)
|
||||
)
|
||||
request = ResponsesRequest(
|
||||
input="hello",
|
||||
tool_choice={"type": "function", "name": "lookup"},
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
result = await serving_responses_instance._render_resolved_response_inputs(
|
||||
request,
|
||||
None,
|
||||
)
|
||||
|
||||
assert result.engine_input["prompt_token_ids"] == [1]
|
||||
serving_responses_instance.online_renderer.render_responses.assert_awaited_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_render_next_turn_reuses_shared_responses_renderer(
|
||||
self, serving_responses_instance
|
||||
):
|
||||
serving_responses_instance.online_renderer.render_responses = AsyncMock(
|
||||
return_value=SimpleNamespace(
|
||||
messages=[],
|
||||
engine_input=tokens_input([8, 9], cache_salt="request-salt"),
|
||||
)
|
||||
)
|
||||
serving_responses_instance.online_renderer.preprocess_chat = AsyncMock(
|
||||
return_value=([], [tokens_input([1])])
|
||||
)
|
||||
request = ResponsesRequest(
|
||||
input="first",
|
||||
instructions="initial instructions",
|
||||
cache_salt="request-salt",
|
||||
)
|
||||
turn_messages = [
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": "tool call"},
|
||||
{"role": "tool", "content": "tool output"},
|
||||
]
|
||||
|
||||
engine_inputs = await serving_responses_instance._render_next_turn(
|
||||
request,
|
||||
turn_messages,
|
||||
)
|
||||
|
||||
assert engine_inputs[0]["prompt_token_ids"] == [8, 9]
|
||||
render_request = (
|
||||
serving_responses_instance.online_renderer.render_responses.call_args.args[
|
||||
0
|
||||
]
|
||||
)
|
||||
assert render_request.input == turn_messages
|
||||
assert render_request.instructions is None
|
||||
assert render_request.cache_salt == "request-salt"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_harmony_tool_followup_preserves_render_params(
|
||||
self, serving_responses_instance, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
class ToolCallingHarmonyContext(HarmonyContext):
|
||||
def __init__(self):
|
||||
self._messages = []
|
||||
self._needs_tool_call = True
|
||||
|
||||
def append_output(self, output) -> None:
|
||||
pass
|
||||
|
||||
def need_builtin_tool_call(self) -> bool:
|
||||
return self._needs_tool_call
|
||||
|
||||
async def call_tool(self):
|
||||
self._needs_tool_call = False
|
||||
return []
|
||||
|
||||
def append_tool_output(self, output) -> None:
|
||||
self._messages.extend(output)
|
||||
|
||||
async def generate_output():
|
||||
yield MagicMock()
|
||||
|
||||
context = ToolCallingHarmonyContext()
|
||||
serving_responses_instance.engine_client.generate.side_effect = (
|
||||
lambda *args, **kwargs: generate_output()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.renderers.online_renderer.render_for_completion",
|
||||
lambda messages: [1, 2, 3, 4],
|
||||
)
|
||||
online_renderer = serving_responses_instance.online_renderer
|
||||
online_renderer.renderer = SimpleNamespace(
|
||||
tokenizer=SimpleNamespace(truncation_side="left")
|
||||
)
|
||||
serving_responses_instance.online_renderer.render_responses_harmony_messages = (
|
||||
OnlineRenderer.render_responses_harmony_messages.__get__(online_renderer)
|
||||
)
|
||||
serving_responses_instance._extract_prompt_len = MagicMock(return_value=2)
|
||||
|
||||
async for _ in serving_responses_instance._generate_with_builtin_tools(
|
||||
request_id="req",
|
||||
engine_input=tokens_input([1], cache_salt="request-salt"),
|
||||
sampling_params=MagicMock(),
|
||||
context=context,
|
||||
tok_params=TokenizeParams(
|
||||
max_total_tokens=3,
|
||||
max_output_tokens=1,
|
||||
truncate_prompt_tokens=-1,
|
||||
),
|
||||
):
|
||||
pass
|
||||
|
||||
assert serving_responses_instance.engine_client.generate.call_count == 2
|
||||
followup_engine_input = (
|
||||
serving_responses_instance.engine_client.generate.call_args_list[1].args[0]
|
||||
)
|
||||
assert followup_engine_input.get("cache_salt") == "request-salt"
|
||||
assert followup_engine_input["prompt_token_ids"] == [3, 4]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_tool_sessions(
|
||||
self, serving_responses_instance, mock_context, mock_exit_stack
|
||||
|
||||
@@ -151,7 +151,8 @@ async def _capture_responses(
|
||||
request: ResponsesRequest,
|
||||
) -> CapturedRenderInputs:
|
||||
capture = RenderCapture(serving.online_renderer)
|
||||
await serving._make_request(request, prev_response=None)
|
||||
result = await serving.online_renderer.render_responses(request)
|
||||
assert not isinstance(result, ErrorResponse), result
|
||||
return capture.take()
|
||||
|
||||
|
||||
|
||||
@@ -3,15 +3,138 @@
|
||||
|
||||
"""Tests for the /render endpoints that expose prompt preprocessing."""
|
||||
|
||||
from http import HTTPStatus
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.scale_out.render.api_router import router
|
||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||
|
||||
|
||||
def _build_responses_serving_render() -> ServingRender:
|
||||
serving = ServingRender.__new__(ServingRender)
|
||||
serving.model_config = SimpleNamespace(
|
||||
max_model_len=100,
|
||||
is_encoder_decoder=False,
|
||||
)
|
||||
serving.default_sampling_params = {}
|
||||
serving.override_max_tokens = None
|
||||
serving.tool_server = MagicMock()
|
||||
serving.online_renderer = MagicMock()
|
||||
serving.online_renderer.create_error_response = (
|
||||
OnlineRenderer.create_error_response.__get__(serving.online_renderer)
|
||||
)
|
||||
serving.online_renderer.validate_chat_template = (
|
||||
OnlineRenderer.validate_chat_template.__get__(serving.online_renderer)
|
||||
)
|
||||
serving.online_renderer.trust_request_chat_template = True
|
||||
serving._check_model = AsyncMock(return_value=None)
|
||||
return serving
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_responses_render_route_is_registered():
|
||||
assert any(route.path == "/v1/responses/render" for route in router.routes)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_responses_render_route_documents_not_implemented():
|
||||
route = next(
|
||||
route for route in router.routes if route.path == "/v1/responses/render"
|
||||
)
|
||||
|
||||
assert HTTPStatus.NOT_IMPLEMENTED.value in route.responses
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_responses_returns_generate_request_without_stored_state():
|
||||
serving = _build_responses_serving_render()
|
||||
serving.online_renderer.render_responses = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
engine_input={"prompt_token_ids": [7, 8, 9]},
|
||||
)
|
||||
)
|
||||
request = ResponsesRequest(
|
||||
model=MODEL_NAME,
|
||||
input="Test prompt",
|
||||
max_output_tokens=12,
|
||||
stream=True,
|
||||
cache_salt="request-salt",
|
||||
priority=3,
|
||||
kv_transfer_params={"do_remote_prefill": True},
|
||||
ec_transfer_params={"remote": True},
|
||||
)
|
||||
|
||||
response = await serving.render_responses_request(request)
|
||||
|
||||
assert response.token_ids == [7, 8, 9]
|
||||
assert response.request_id == request.request_id
|
||||
assert response.sampling_params.max_tokens == 12
|
||||
assert response.model == MODEL_NAME
|
||||
assert response.stream is True
|
||||
assert response.cache_salt == "request-salt"
|
||||
assert response.priority == 3
|
||||
assert response.kv_transfer_params == {"do_remote_prefill": True}
|
||||
assert response.ec_transfer_params == {"remote": True}
|
||||
serving.online_renderer.render_responses.assert_awaited_once_with(
|
||||
request,
|
||||
previous_messages=None,
|
||||
previous_response_outputs=None,
|
||||
tool_server=serving.tool_server,
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_responses_rejects_previous_response_id():
|
||||
serving = _build_responses_serving_render()
|
||||
serving.online_renderer.render_responses = AsyncMock()
|
||||
request = ResponsesRequest(
|
||||
model=MODEL_NAME,
|
||||
input="Test prompt",
|
||||
previous_response_id="resp_previous",
|
||||
)
|
||||
|
||||
response = await serving.render_responses_request(request)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
assert response.error.param == "previous_response_id"
|
||||
serving.online_renderer.render_responses.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip_global_cleanup
|
||||
async def test_render_responses_rejects_empty_token_ids():
|
||||
serving = _build_responses_serving_render()
|
||||
serving.online_renderer.render_responses = AsyncMock(
|
||||
return_value=MagicMock(
|
||||
messages=[],
|
||||
engine_input={"prompt_token_ids": []},
|
||||
)
|
||||
)
|
||||
|
||||
response = await serving.render_responses_request(
|
||||
ResponsesRequest(model=MODEL_NAME, input="Test prompt")
|
||||
)
|
||||
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.message == "No token_ids rendered"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args: list[str] = ["--trust-request-chat-template"]
|
||||
@@ -28,6 +151,71 @@ async def client(server):
|
||||
yield http_client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_render_basic(client):
|
||||
response = await client.post(
|
||||
"/v1/responses/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": "When should a Responses handler return an empty string?",
|
||||
"max_output_tokens": 7,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["model"] == MODEL_NAME
|
||||
assert data["request_id"].startswith("resp_")
|
||||
assert data["sampling_params"]["max_tokens"] == 7
|
||||
assert data["token_ids"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_render_includes_prior_input_items(client):
|
||||
current_turn = {
|
||||
"role": "user",
|
||||
"content": "Which color should I remember?",
|
||||
}
|
||||
current_turn_response = await client.post(
|
||||
"/v1/responses/render",
|
||||
json={"model": MODEL_NAME, "input": [current_turn]},
|
||||
)
|
||||
multi_turn_response = await client.post(
|
||||
"/v1/responses/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": [
|
||||
{"role": "user", "content": "Remember the color cobalt."},
|
||||
{"role": "assistant", "content": "I will remember cobalt."},
|
||||
current_turn,
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert current_turn_response.status_code == 200
|
||||
assert multi_turn_response.status_code == 200
|
||||
current_turn_token_ids = current_turn_response.json()["token_ids"]
|
||||
multi_turn_token_ids = multi_turn_response.json()["token_ids"]
|
||||
assert len(multi_turn_token_ids) > len(current_turn_token_ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_render_rejects_previous_response_id_over_http(client):
|
||||
response = await client.post(
|
||||
"/v1/responses/render",
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": "Continue the previous response.",
|
||||
"previous_response_id": "resp_previous",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
error = response.json()["error"]
|
||||
assert error["type"] == "invalid_request_error"
|
||||
assert error["param"] == "previous_response_id"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_render_basic(client):
|
||||
"""Test basic completion render endpoint."""
|
||||
|
||||
@@ -15,6 +15,7 @@ RENDER_PATHS = {
|
||||
"/v1/completions/render",
|
||||
"/v1/completions/derender",
|
||||
"/v1/messages/render",
|
||||
"/v1/responses/render",
|
||||
}
|
||||
GENERATE_PATH = "/inference/v1/generate"
|
||||
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
|
||||
|
||||
Reference in New Issue
Block a user