[Frontend] Add stateless /v1/responses/render endpoint (#50195)
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
5e6f6a8ed4
commit
6a2a2bb02b
@@ -141,6 +141,8 @@ For further details on renderer APIs, please refer to [this page](renderer.md).
|
|||||||
- Render completion requests
|
- Render completion requests
|
||||||
- [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
|
- [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
|
||||||
- Render chat completions
|
- Render chat completions
|
||||||
|
- [Responses Render API](renderer.md) (`/v1/responses/render`)
|
||||||
|
- Render self-contained Responses requests
|
||||||
|
|
||||||
### Derenderer APIs
|
### Derenderer APIs
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,29 @@ VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve <model>
|
|||||||
- Render completion requests
|
- Render completion requests
|
||||||
- [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
|
- [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
|
||||||
- Render chat completions
|
- Render chat completions
|
||||||
|
- [Responses Render API](renderer.md) (`/v1/responses/render`)
|
||||||
|
- Render a self-contained Responses request
|
||||||
|
|
||||||
|
The Responses render endpoint uses the same prompt construction as
|
||||||
|
`/v1/responses` and returns one token-in `GenerateRequest`. It is stateless:
|
||||||
|
inline history is supported, but `previous_response_id` is not. Callers must
|
||||||
|
resolve stored response state and include the resulting history in the request
|
||||||
|
before rendering.
|
||||||
|
|
||||||
|
For multimodal requests, the `GenerateRequest` contains the model-processed
|
||||||
|
multimodal payload, which can be substantially larger than the source image or
|
||||||
|
video. The caller must forward that payload unchanged to the generation service
|
||||||
|
and provision transport limits and memory accordingly.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:8000/v1/responses/render \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{
|
||||||
|
"model": "meta-llama/Llama-3.1-8B-Instruct",
|
||||||
|
"input": "Explain prefix caching in one sentence.",
|
||||||
|
"max_output_tokens": 32
|
||||||
|
}'
|
||||||
|
```
|
||||||
|
|
||||||
For the post processing counterpart that turns generated token IDs back into OpenAI compatible responses, see the [Derenderer APIs](derenderer.md).
|
For the post processing counterpart that turns generated token IDs back into OpenAI compatible responses, see the [Derenderer APIs](derenderer.md).
|
||||||
|
|
||||||
|
|||||||
@@ -170,6 +170,7 @@ When `--api-key` is configured, the following `/v1` endpoints require Bearer tok
|
|||||||
- `/v1/messages/render` - Render Anthropic-compatible messages (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
|
- `/v1/messages/render` - Render Anthropic-compatible messages (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
|
||||||
- `/v1/messages/count_tokens` - Count tokens for Anthropic messages
|
- `/v1/messages/count_tokens` - Count tokens for Anthropic messages
|
||||||
- `/v1/responses` - Create a response
|
- `/v1/responses` - Create a response
|
||||||
|
- `/v1/responses/render` - Render a self-contained response request (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
|
||||||
- `/v1/responses/{response_id}` - Retrieve a response
|
- `/v1/responses/{response_id}` - Retrieve a response
|
||||||
- `/v1/responses/{response_id}/cancel` - Cancel a response
|
- `/v1/responses/{response_id}/cancel` - Cancel a response
|
||||||
- `/v1/score` - Scoring API
|
- `/v1/score` - Scoring API
|
||||||
|
|||||||
@@ -2,9 +2,11 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
import vllm.entrypoints.launchers.cli_args as cli_args_module
|
||||||
from tests.utils import VLLM_PATH
|
from tests.utils import VLLM_PATH
|
||||||
from vllm.entrypoints.launchers.cli_args import (
|
from vllm.entrypoints.launchers.cli_args import (
|
||||||
make_arg_parser,
|
make_arg_parser,
|
||||||
@@ -332,6 +334,26 @@ def test_default_chat_template_kwargs_default_none(serve_parser):
|
|||||||
assert args.default_chat_template_kwargs is None
|
assert args.default_chat_template_kwargs is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"default_kwargs,cohere_format,expected",
|
||||||
|
[
|
||||||
|
(None, "cmd3", {"cohere_format": "cmd3"}),
|
||||||
|
(
|
||||||
|
{"cohere_format": "cmd4", "custom": True},
|
||||||
|
"cmd3",
|
||||||
|
{"cohere_format": "cmd4", "custom": True},
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, expected):
|
||||||
|
args = SimpleNamespace(
|
||||||
|
default_chat_template_kwargs=default_kwargs,
|
||||||
|
cohere_format=cohere_format,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cli_args_module.resolve_default_chat_template_kwargs(args) == expected
|
||||||
|
|
||||||
|
|
||||||
def test_default_chat_template_kwargs_invalid_json(serve_parser):
|
def test_default_chat_template_kwargs_invalid_json(serve_parser):
|
||||||
"""Ensure invalid JSON raises an error"""
|
"""Ensure invalid JSON raises an error"""
|
||||||
with pytest.raises(SystemExit):
|
with pytest.raises(SystemExit):
|
||||||
|
|||||||
@@ -4,14 +4,19 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
from openai import OpenAI
|
from openai import OpenAI
|
||||||
from openai_harmony import Message, ToolDescription, ToolNamespaceConfig
|
from openai_harmony import Message, ToolDescription, ToolNamespaceConfig
|
||||||
|
|
||||||
|
import vllm.entrypoints.mcp.tool_server as tool_server_module
|
||||||
from tests.utils import RemoteOpenAIServer
|
from tests.utils import RemoteOpenAIServer
|
||||||
from vllm.entrypoints.mcp.tool_server import (
|
from vllm.entrypoints.mcp.tool_server import (
|
||||||
MCPToolServer,
|
MCPToolServer,
|
||||||
|
init_tool_server,
|
||||||
post_process_tools_description,
|
post_process_tools_description,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -39,6 +44,25 @@ _PYTHON_TOOL_INSTRUCTION = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_init_tool_server_without_configuration():
|
||||||
|
args = SimpleNamespace(tool_server=None)
|
||||||
|
|
||||||
|
assert await init_tool_server(args) is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_init_tool_server_with_mcp_url(monkeypatch: pytest.MonkeyPatch):
|
||||||
|
server = MagicMock()
|
||||||
|
server.add_tool_server = AsyncMock()
|
||||||
|
monkeypatch.setattr(tool_server_module, "MCPToolServer", lambda: server)
|
||||||
|
|
||||||
|
result = await init_tool_server(SimpleNamespace(tool_server="http://mcp.test"))
|
||||||
|
|
||||||
|
assert result is server
|
||||||
|
server.add_tool_server.assert_awaited_once_with("http://mcp.test")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip_global_cleanup
|
@pytest.mark.skip_global_cleanup
|
||||||
class TestMCPToolServerUnit:
|
class TestMCPToolServerUnit:
|
||||||
"""Test MCP tool description processing and filtering.
|
"""Test MCP tool description processing and filtering.
|
||||||
|
|||||||
@@ -2,12 +2,16 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from contextlib import AsyncExitStack
|
from contextlib import AsyncExitStack
|
||||||
from unittest.mock import MagicMock
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
from openai.types.responses import (
|
from openai.types.responses import (
|
||||||
|
ResponseFunctionToolCall,
|
||||||
ResponseOutputItemDoneEvent,
|
ResponseOutputItemDoneEvent,
|
||||||
|
ResponseOutputMessage,
|
||||||
|
ResponseOutputText,
|
||||||
ResponseReasoningItem,
|
ResponseReasoningItem,
|
||||||
ResponseReasoningTextDeltaEvent,
|
ResponseReasoningTextDeltaEvent,
|
||||||
ResponseReasoningTextDoneEvent,
|
ResponseReasoningTextDoneEvent,
|
||||||
@@ -23,6 +27,8 @@ from openai.types.responses.tool import (
|
|||||||
Mcp,
|
Mcp,
|
||||||
Tool,
|
Tool,
|
||||||
)
|
)
|
||||||
|
from openai_harmony import Message as OpenAIHarmonyMessage
|
||||||
|
from openai_harmony import Role
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
from vllm.entrypoints.generate.base.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
@@ -32,7 +38,11 @@ from vllm.entrypoints.generate.base.protocol import (
|
|||||||
RequestResponseMetadata,
|
RequestResponseMetadata,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||||
from vllm.entrypoints.openai.responses.context import ConversationContext, SimpleContext
|
from vllm.entrypoints.openai.responses.context import (
|
||||||
|
ConversationContext,
|
||||||
|
HarmonyContext,
|
||||||
|
SimpleContext,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.responses.protocol import (
|
from vllm.entrypoints.openai.responses.protocol import (
|
||||||
ResponseCreatedEvent,
|
ResponseCreatedEvent,
|
||||||
ResponseRawMessageAndToken,
|
ResponseRawMessageAndToken,
|
||||||
@@ -42,7 +52,6 @@ from vllm.entrypoints.openai.responses.protocol import (
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.responses.serving import (
|
from vllm.entrypoints.openai.responses.serving import (
|
||||||
OpenAIServingResponses,
|
OpenAIServingResponses,
|
||||||
_extract_allowed_tools_from_mcp_requests,
|
|
||||||
extract_tool_types,
|
extract_tool_types,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.responses.streaming_events import (
|
from vllm.entrypoints.openai.responses.streaming_events import (
|
||||||
@@ -52,8 +61,23 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
|||||||
from vllm.inputs import tokens_input
|
from vllm.inputs import tokens_input
|
||||||
from vllm.outputs import CompletionOutput, RequestOutput
|
from vllm.outputs import CompletionOutput, RequestOutput
|
||||||
from vllm.parser.harmony import Segment
|
from vllm.parser.harmony import Segment
|
||||||
|
from vllm.renderers import TokenizeParams
|
||||||
|
from vllm.renderers.online_renderer import (
|
||||||
|
OnlineRenderer,
|
||||||
|
_extract_allowed_tools_from_mcp_requests,
|
||||||
|
)
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.skip_global_cleanup
|
||||||
|
|
||||||
|
|
||||||
|
def _new_online_renderer() -> OnlineRenderer:
|
||||||
|
renderer = OnlineRenderer.__new__(OnlineRenderer)
|
||||||
|
renderer.exclude_tools_when_tool_choice_none = False
|
||||||
|
renderer.parser = None
|
||||||
|
renderer.trust_request_chat_template = True
|
||||||
|
return renderer
|
||||||
|
|
||||||
|
|
||||||
class MockConversationContext(ConversationContext):
|
class MockConversationContext(ConversationContext):
|
||||||
"""Mock conversation context for testing"""
|
"""Mock conversation context for testing"""
|
||||||
@@ -205,6 +229,325 @@ def test_response_created_event_uses_public_json_schema_alias() -> None:
|
|||||||
assert event.response.text.format.model_dump(by_alias=True)["schema"] == schema
|
assert event.response.text.format.model_dump(by_alias=True)["schema"] == schema
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_renders_non_harmony_responses_with_explicit_history():
|
||||||
|
renderer = _new_online_renderer()
|
||||||
|
renderer.use_harmony = False
|
||||||
|
renderer.chat_template = "server-template"
|
||||||
|
renderer.chat_template_content_format = "string"
|
||||||
|
renderer.default_chat_template_kwargs = {"server_default": "kept"}
|
||||||
|
renderer.parser = None
|
||||||
|
renderer.preprocess_chat = AsyncMock(
|
||||||
|
return_value=(
|
||||||
|
[],
|
||||||
|
[tokens_input([11, 12, 13], cache_salt="request-salt")],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
previous_messages = [
|
||||||
|
{"role": "system", "content": "old instructions"},
|
||||||
|
{"role": "user", "content": "first"},
|
||||||
|
]
|
||||||
|
previous_outputs = [
|
||||||
|
ResponseOutputMessage(
|
||||||
|
id="msg_1",
|
||||||
|
content=[
|
||||||
|
ResponseOutputText(
|
||||||
|
annotations=[],
|
||||||
|
text="prior answer",
|
||||||
|
type="output_text",
|
||||||
|
)
|
||||||
|
],
|
||||||
|
role="assistant",
|
||||||
|
status="completed",
|
||||||
|
type="message",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="next",
|
||||||
|
instructions="new instructions",
|
||||||
|
tools=[],
|
||||||
|
cache_salt="request-salt",
|
||||||
|
chat_template_kwargs={"request_default": "kept"},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await renderer.render_responses(
|
||||||
|
request,
|
||||||
|
previous_messages=previous_messages,
|
||||||
|
previous_response_outputs=previous_outputs,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not isinstance(result, ErrorResponse)
|
||||||
|
assert result.messages == [
|
||||||
|
{"role": "system", "content": "new instructions"},
|
||||||
|
{"role": "user", "content": "first"},
|
||||||
|
{"role": "assistant", "content": "prior answer"},
|
||||||
|
{"role": "user", "content": "next"},
|
||||||
|
]
|
||||||
|
assert result.engine_input["prompt_token_ids"] == [11, 12, 13]
|
||||||
|
assert result.engine_input["cache_salt"] == "request-salt"
|
||||||
|
assert previous_messages[0]["content"] == "old instructions"
|
||||||
|
assert previous_outputs[0].content[0].text == "prior answer"
|
||||||
|
|
||||||
|
preprocess_kwargs = renderer.preprocess_chat.call_args.kwargs
|
||||||
|
assert preprocess_kwargs["default_template"] == "server-template"
|
||||||
|
assert preprocess_kwargs["default_template_content_format"] == "string"
|
||||||
|
assert preprocess_kwargs["default_template_kwargs"]["server_default"] == "kept"
|
||||||
|
assert preprocess_kwargs["default_template_kwargs"]["request_default"] == "kept"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_rejects_untrusted_responses_chat_template():
|
||||||
|
renderer = _new_online_renderer()
|
||||||
|
renderer.use_harmony = False
|
||||||
|
renderer.chat_template = "server-template"
|
||||||
|
renderer.chat_template_content_format = "string"
|
||||||
|
renderer.default_chat_template_kwargs = {}
|
||||||
|
renderer.exclude_tools_when_tool_choice_none = False
|
||||||
|
renderer.trust_request_chat_template = False
|
||||||
|
renderer.parser = None
|
||||||
|
renderer.preprocess_chat = AsyncMock(
|
||||||
|
return_value=([], [tokens_input([1])]),
|
||||||
|
)
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="hello",
|
||||||
|
chat_template_kwargs={"chat_template": "{{ messages }}"},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await renderer.render_responses(request)
|
||||||
|
|
||||||
|
assert isinstance(result, ErrorResponse)
|
||||||
|
assert result.error.code == 400
|
||||||
|
assert "untrusted chat template" in result.error.message.lower()
|
||||||
|
renderer.preprocess_chat.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_rejects_harmony_history_for_non_harmony_model():
|
||||||
|
renderer = _new_online_renderer()
|
||||||
|
renderer.use_harmony = False
|
||||||
|
request = ResponsesRequest(input="next")
|
||||||
|
|
||||||
|
result = await renderer.render_responses(
|
||||||
|
request,
|
||||||
|
previous_messages=[
|
||||||
|
OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first")
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(result, ErrorResponse)
|
||||||
|
assert result.error.param == "previous_response_id"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_rejects_chat_history_for_harmony_model():
|
||||||
|
renderer = _new_online_renderer()
|
||||||
|
renderer.use_harmony = True
|
||||||
|
request = ResponsesRequest(input="next")
|
||||||
|
|
||||||
|
result = await renderer.render_responses(
|
||||||
|
request,
|
||||||
|
previous_messages=[{"role": "user", "content": "first"}],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(result, ErrorResponse)
|
||||||
|
assert result.error.param == "previous_response_id"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_adjusts_harmony_tool_choice(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
):
|
||||||
|
class StubParser:
|
||||||
|
def __init__(self, tokenizer, tools, *, model_config):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def adjust_request(self, request):
|
||||||
|
request.cache_salt = "adjusted"
|
||||||
|
return request
|
||||||
|
|
||||||
|
renderer = _new_online_renderer()
|
||||||
|
renderer.use_harmony = True
|
||||||
|
renderer.parser = StubParser
|
||||||
|
renderer.model_config = SimpleNamespace(max_model_len=100)
|
||||||
|
tokenizer = SimpleNamespace(truncation_side="left")
|
||||||
|
renderer.renderer = SimpleNamespace(
|
||||||
|
tokenizer=tokenizer,
|
||||||
|
get_tokenizer=lambda: tokenizer,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.renderers.online_renderer.render_for_completion",
|
||||||
|
lambda messages: [1],
|
||||||
|
)
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="next",
|
||||||
|
tools=[
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"name": "lookup",
|
||||||
|
"parameters": {"type": "object", "properties": {}},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
tool_choice={"type": "function", "name": "lookup"},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await renderer.render_responses(request)
|
||||||
|
|
||||||
|
assert not isinstance(result, ErrorResponse)
|
||||||
|
assert result.engine_input["cache_salt"] == "adjusted"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
):
|
||||||
|
renderer = _new_online_renderer()
|
||||||
|
renderer.use_harmony = True
|
||||||
|
renderer.model_config = SimpleNamespace(max_model_len=5)
|
||||||
|
renderer.renderer = SimpleNamespace(
|
||||||
|
tokenizer=SimpleNamespace(truncation_side="left")
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.renderers.online_renderer.render_for_completion",
|
||||||
|
lambda messages: [1, 2, 3, 4, 5, 6],
|
||||||
|
)
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="hello",
|
||||||
|
max_output_tokens=2,
|
||||||
|
truncation="auto",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await renderer.render_responses(request)
|
||||||
|
|
||||||
|
assert not isinstance(result, ErrorResponse)
|
||||||
|
assert result.engine_input["prompt_token_ids"] == [4, 5, 6]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_renders_harmony_continuation_with_call_linkage(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
):
|
||||||
|
renderer = _new_online_renderer()
|
||||||
|
renderer.use_harmony = True
|
||||||
|
renderer.model_config = SimpleNamespace(max_model_len=100)
|
||||||
|
renderer.renderer = SimpleNamespace(
|
||||||
|
tokenizer=SimpleNamespace(truncation_side="left")
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.renderers.online_renderer.render_for_completion",
|
||||||
|
lambda messages: [1],
|
||||||
|
)
|
||||||
|
previous_messages = [
|
||||||
|
OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first"),
|
||||||
|
]
|
||||||
|
previous_outputs = [
|
||||||
|
ResponseFunctionToolCall(
|
||||||
|
arguments='{"city":"Paris"}',
|
||||||
|
call_id="call_1",
|
||||||
|
name="lookup_weather",
|
||||||
|
type="function_call",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input=[
|
||||||
|
{
|
||||||
|
"type": "function_call_output",
|
||||||
|
"call_id": "call_1",
|
||||||
|
"output": "sunny",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
instructions="replacement instructions are ignored on continuations",
|
||||||
|
reasoning={"effort": "high"},
|
||||||
|
tools=[
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"name": "lookup_weather",
|
||||||
|
"parameters": {"type": "object", "properties": {}},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
cache_salt="request-salt",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await renderer.render_responses(
|
||||||
|
request,
|
||||||
|
previous_messages=previous_messages,
|
||||||
|
previous_response_outputs=previous_outputs,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not isinstance(result, ErrorResponse)
|
||||||
|
assert result.messages[0] is previous_messages[0]
|
||||||
|
tool_output = result.messages[-1]
|
||||||
|
assert tool_output.author.role == "tool"
|
||||||
|
assert tool_output.author.name == "functions.lookup_weather"
|
||||||
|
assert tool_output.channel == "commentary"
|
||||||
|
assert tool_output.recipient == "assistant"
|
||||||
|
assert result.engine_input["cache_salt"] == "request-salt"
|
||||||
|
assert "arrival_time" in result.engine_input
|
||||||
|
assert previous_messages == [
|
||||||
|
OpenAIHarmonyMessage.from_role_and_content(Role.USER, "first"),
|
||||||
|
]
|
||||||
|
assert previous_outputs[0].call_id == "call_1"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_returns_typed_error_for_bad_harmony_call_reference():
|
||||||
|
renderer = _new_online_renderer()
|
||||||
|
renderer.use_harmony = True
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input=[
|
||||||
|
{
|
||||||
|
"type": "function_call_output",
|
||||||
|
"call_id": "missing",
|
||||||
|
"output": "orphaned",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
tools=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await renderer.render_responses(request)
|
||||||
|
|
||||||
|
assert isinstance(result, ErrorResponse)
|
||||||
|
assert result.error.type == "invalid_request_error"
|
||||||
|
assert result.error.param == "input"
|
||||||
|
assert "No call message found for missing" in result.error.message
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_online_renderer_preserves_missing_harmony_builtin_tool_behavior(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
):
|
||||||
|
renderer = _new_online_renderer()
|
||||||
|
renderer.use_harmony = True
|
||||||
|
renderer.model_config = SimpleNamespace(max_model_len=100)
|
||||||
|
renderer.renderer = SimpleNamespace(
|
||||||
|
tokenizer=SimpleNamespace(truncation_side="left")
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.renderers.online_renderer.render_for_completion",
|
||||||
|
lambda messages: [1],
|
||||||
|
)
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="search",
|
||||||
|
tools=[{"type": "web_search_preview"}],
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await renderer.render_responses(request, tool_server=None)
|
||||||
|
|
||||||
|
assert not isinstance(result, ErrorResponse)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"engine_inputs",
|
||||||
|
[[], [tokens_input([1]), tokens_input([2])]],
|
||||||
|
)
|
||||||
|
def test_responses_render_result_rejects_non_single_prompt(engine_inputs):
|
||||||
|
renderer = _new_online_renderer()
|
||||||
|
|
||||||
|
result = renderer._responses_render_result([], engine_inputs)
|
||||||
|
|
||||||
|
assert isinstance(result, ErrorResponse)
|
||||||
|
assert f"got {len(engine_inputs)}" in result.error.message
|
||||||
|
|
||||||
|
|
||||||
class TestInitializeToolSessions:
|
class TestInitializeToolSessions:
|
||||||
"""Test class for _initialize_tool_sessions method"""
|
"""Test class for _initialize_tool_sessions method"""
|
||||||
|
|
||||||
@@ -240,6 +583,139 @@ class TestInitializeToolSessions:
|
|||||||
|
|
||||||
return instance
|
return instance
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_stateful_harmony_delegates_named_tool_choice_to_renderer(
|
||||||
|
self,
|
||||||
|
serving_responses_instance,
|
||||||
|
):
|
||||||
|
serving_responses_instance.use_harmony = True
|
||||||
|
serving_responses_instance.online_renderer.render_responses = AsyncMock(
|
||||||
|
return_value=SimpleNamespace(
|
||||||
|
messages=[],
|
||||||
|
engine_input=tokens_input([1]),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="hello",
|
||||||
|
tool_choice={"type": "function", "name": "lookup"},
|
||||||
|
tools=[
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"name": "lookup",
|
||||||
|
"parameters": {"type": "object", "properties": {}},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await serving_responses_instance._render_resolved_response_inputs(
|
||||||
|
request,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.engine_input["prompt_token_ids"] == [1]
|
||||||
|
serving_responses_instance.online_renderer.render_responses.assert_awaited_once()
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_render_next_turn_reuses_shared_responses_renderer(
|
||||||
|
self, serving_responses_instance
|
||||||
|
):
|
||||||
|
serving_responses_instance.online_renderer.render_responses = AsyncMock(
|
||||||
|
return_value=SimpleNamespace(
|
||||||
|
messages=[],
|
||||||
|
engine_input=tokens_input([8, 9], cache_salt="request-salt"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
serving_responses_instance.online_renderer.preprocess_chat = AsyncMock(
|
||||||
|
return_value=([], [tokens_input([1])])
|
||||||
|
)
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="first",
|
||||||
|
instructions="initial instructions",
|
||||||
|
cache_salt="request-salt",
|
||||||
|
)
|
||||||
|
turn_messages = [
|
||||||
|
{"role": "user", "content": "first"},
|
||||||
|
{"role": "assistant", "content": "tool call"},
|
||||||
|
{"role": "tool", "content": "tool output"},
|
||||||
|
]
|
||||||
|
|
||||||
|
engine_inputs = await serving_responses_instance._render_next_turn(
|
||||||
|
request,
|
||||||
|
turn_messages,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert engine_inputs[0]["prompt_token_ids"] == [8, 9]
|
||||||
|
render_request = (
|
||||||
|
serving_responses_instance.online_renderer.render_responses.call_args.args[
|
||||||
|
0
|
||||||
|
]
|
||||||
|
)
|
||||||
|
assert render_request.input == turn_messages
|
||||||
|
assert render_request.instructions is None
|
||||||
|
assert render_request.cache_salt == "request-salt"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_harmony_tool_followup_preserves_render_params(
|
||||||
|
self, serving_responses_instance, monkeypatch: pytest.MonkeyPatch
|
||||||
|
):
|
||||||
|
class ToolCallingHarmonyContext(HarmonyContext):
|
||||||
|
def __init__(self):
|
||||||
|
self._messages = []
|
||||||
|
self._needs_tool_call = True
|
||||||
|
|
||||||
|
def append_output(self, output) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def need_builtin_tool_call(self) -> bool:
|
||||||
|
return self._needs_tool_call
|
||||||
|
|
||||||
|
async def call_tool(self):
|
||||||
|
self._needs_tool_call = False
|
||||||
|
return []
|
||||||
|
|
||||||
|
def append_tool_output(self, output) -> None:
|
||||||
|
self._messages.extend(output)
|
||||||
|
|
||||||
|
async def generate_output():
|
||||||
|
yield MagicMock()
|
||||||
|
|
||||||
|
context = ToolCallingHarmonyContext()
|
||||||
|
serving_responses_instance.engine_client.generate.side_effect = (
|
||||||
|
lambda *args, **kwargs: generate_output()
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.renderers.online_renderer.render_for_completion",
|
||||||
|
lambda messages: [1, 2, 3, 4],
|
||||||
|
)
|
||||||
|
online_renderer = serving_responses_instance.online_renderer
|
||||||
|
online_renderer.renderer = SimpleNamespace(
|
||||||
|
tokenizer=SimpleNamespace(truncation_side="left")
|
||||||
|
)
|
||||||
|
serving_responses_instance.online_renderer.render_responses_harmony_messages = (
|
||||||
|
OnlineRenderer.render_responses_harmony_messages.__get__(online_renderer)
|
||||||
|
)
|
||||||
|
serving_responses_instance._extract_prompt_len = MagicMock(return_value=2)
|
||||||
|
|
||||||
|
async for _ in serving_responses_instance._generate_with_builtin_tools(
|
||||||
|
request_id="req",
|
||||||
|
engine_input=tokens_input([1], cache_salt="request-salt"),
|
||||||
|
sampling_params=MagicMock(),
|
||||||
|
context=context,
|
||||||
|
tok_params=TokenizeParams(
|
||||||
|
max_total_tokens=3,
|
||||||
|
max_output_tokens=1,
|
||||||
|
truncate_prompt_tokens=-1,
|
||||||
|
),
|
||||||
|
):
|
||||||
|
pass
|
||||||
|
|
||||||
|
assert serving_responses_instance.engine_client.generate.call_count == 2
|
||||||
|
followup_engine_input = (
|
||||||
|
serving_responses_instance.engine_client.generate.call_args_list[1].args[0]
|
||||||
|
)
|
||||||
|
assert followup_engine_input.get("cache_salt") == "request-salt"
|
||||||
|
assert followup_engine_input["prompt_token_ids"] == [3, 4]
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_initialize_tool_sessions(
|
async def test_initialize_tool_sessions(
|
||||||
self, serving_responses_instance, mock_context, mock_exit_stack
|
self, serving_responses_instance, mock_context, mock_exit_stack
|
||||||
|
|||||||
@@ -151,7 +151,8 @@ async def _capture_responses(
|
|||||||
request: ResponsesRequest,
|
request: ResponsesRequest,
|
||||||
) -> CapturedRenderInputs:
|
) -> CapturedRenderInputs:
|
||||||
capture = RenderCapture(serving.online_renderer)
|
capture = RenderCapture(serving.online_renderer)
|
||||||
await serving._make_request(request, prev_response=None)
|
result = await serving.online_renderer.render_responses(request)
|
||||||
|
assert not isinstance(result, ErrorResponse), result
|
||||||
return capture.take()
|
return capture.take()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,15 +3,138 @@
|
|||||||
|
|
||||||
"""Tests for the /render endpoints that expose prompt preprocessing."""
|
"""Tests for the /render endpoints that expose prompt preprocessing."""
|
||||||
|
|
||||||
|
from http import HTTPStatus
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
import pytest_asyncio
|
import pytest_asyncio
|
||||||
|
|
||||||
from tests.utils import RemoteLaunchRenderServer
|
from tests.utils import RemoteLaunchRenderServer
|
||||||
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
|
from vllm.entrypoints.scale_out.render.api_router import router
|
||||||
|
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||||
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
|
|
||||||
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
|
||||||
|
|
||||||
|
|
||||||
|
def _build_responses_serving_render() -> ServingRender:
|
||||||
|
serving = ServingRender.__new__(ServingRender)
|
||||||
|
serving.model_config = SimpleNamespace(
|
||||||
|
max_model_len=100,
|
||||||
|
is_encoder_decoder=False,
|
||||||
|
)
|
||||||
|
serving.default_sampling_params = {}
|
||||||
|
serving.override_max_tokens = None
|
||||||
|
serving.tool_server = MagicMock()
|
||||||
|
serving.online_renderer = MagicMock()
|
||||||
|
serving.online_renderer.create_error_response = (
|
||||||
|
OnlineRenderer.create_error_response.__get__(serving.online_renderer)
|
||||||
|
)
|
||||||
|
serving.online_renderer.validate_chat_template = (
|
||||||
|
OnlineRenderer.validate_chat_template.__get__(serving.online_renderer)
|
||||||
|
)
|
||||||
|
serving.online_renderer.trust_request_chat_template = True
|
||||||
|
serving._check_model = AsyncMock(return_value=None)
|
||||||
|
return serving
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
def test_responses_render_route_is_registered():
|
||||||
|
assert any(route.path == "/v1/responses/render" for route in router.routes)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
def test_responses_render_route_documents_not_implemented():
|
||||||
|
route = next(
|
||||||
|
route for route in router.routes if route.path == "/v1/responses/render"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert HTTPStatus.NOT_IMPLEMENTED.value in route.responses
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
async def test_render_responses_returns_generate_request_without_stored_state():
|
||||||
|
serving = _build_responses_serving_render()
|
||||||
|
serving.online_renderer.render_responses = AsyncMock(
|
||||||
|
return_value=MagicMock(
|
||||||
|
messages=[{"role": "user", "content": "Test prompt"}],
|
||||||
|
engine_input={"prompt_token_ids": [7, 8, 9]},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
request = ResponsesRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
input="Test prompt",
|
||||||
|
max_output_tokens=12,
|
||||||
|
stream=True,
|
||||||
|
cache_salt="request-salt",
|
||||||
|
priority=3,
|
||||||
|
kv_transfer_params={"do_remote_prefill": True},
|
||||||
|
ec_transfer_params={"remote": True},
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await serving.render_responses_request(request)
|
||||||
|
|
||||||
|
assert response.token_ids == [7, 8, 9]
|
||||||
|
assert response.request_id == request.request_id
|
||||||
|
assert response.sampling_params.max_tokens == 12
|
||||||
|
assert response.model == MODEL_NAME
|
||||||
|
assert response.stream is True
|
||||||
|
assert response.cache_salt == "request-salt"
|
||||||
|
assert response.priority == 3
|
||||||
|
assert response.kv_transfer_params == {"do_remote_prefill": True}
|
||||||
|
assert response.ec_transfer_params == {"remote": True}
|
||||||
|
serving.online_renderer.render_responses.assert_awaited_once_with(
|
||||||
|
request,
|
||||||
|
previous_messages=None,
|
||||||
|
previous_response_outputs=None,
|
||||||
|
tool_server=serving.tool_server,
|
||||||
|
skip_mm_cache=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
async def test_render_responses_rejects_previous_response_id():
|
||||||
|
serving = _build_responses_serving_render()
|
||||||
|
serving.online_renderer.render_responses = AsyncMock()
|
||||||
|
request = ResponsesRequest(
|
||||||
|
model=MODEL_NAME,
|
||||||
|
input="Test prompt",
|
||||||
|
previous_response_id="resp_previous",
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await serving.render_responses_request(request)
|
||||||
|
|
||||||
|
assert isinstance(response, ErrorResponse)
|
||||||
|
assert response.error.code == 400
|
||||||
|
assert response.error.param == "previous_response_id"
|
||||||
|
serving.online_renderer.render_responses.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
async def test_render_responses_rejects_empty_token_ids():
|
||||||
|
serving = _build_responses_serving_render()
|
||||||
|
serving.online_renderer.render_responses = AsyncMock(
|
||||||
|
return_value=MagicMock(
|
||||||
|
messages=[],
|
||||||
|
engine_input={"prompt_token_ids": []},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
response = await serving.render_responses_request(
|
||||||
|
ResponsesRequest(model=MODEL_NAME, input="Test prompt")
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(response, ErrorResponse)
|
||||||
|
assert response.error.message == "No token_ids rendered"
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
def server():
|
def server():
|
||||||
args: list[str] = ["--trust-request-chat-template"]
|
args: list[str] = ["--trust-request-chat-template"]
|
||||||
@@ -28,6 +151,71 @@ async def client(server):
|
|||||||
yield http_client
|
yield http_client
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_responses_render_basic(client):
|
||||||
|
response = await client.post(
|
||||||
|
"/v1/responses/render",
|
||||||
|
json={
|
||||||
|
"model": MODEL_NAME,
|
||||||
|
"input": "When should a Responses handler return an empty string?",
|
||||||
|
"max_output_tokens": 7,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
data = response.json()
|
||||||
|
assert data["model"] == MODEL_NAME
|
||||||
|
assert data["request_id"].startswith("resp_")
|
||||||
|
assert data["sampling_params"]["max_tokens"] == 7
|
||||||
|
assert data["token_ids"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_responses_render_includes_prior_input_items(client):
|
||||||
|
current_turn = {
|
||||||
|
"role": "user",
|
||||||
|
"content": "Which color should I remember?",
|
||||||
|
}
|
||||||
|
current_turn_response = await client.post(
|
||||||
|
"/v1/responses/render",
|
||||||
|
json={"model": MODEL_NAME, "input": [current_turn]},
|
||||||
|
)
|
||||||
|
multi_turn_response = await client.post(
|
||||||
|
"/v1/responses/render",
|
||||||
|
json={
|
||||||
|
"model": MODEL_NAME,
|
||||||
|
"input": [
|
||||||
|
{"role": "user", "content": "Remember the color cobalt."},
|
||||||
|
{"role": "assistant", "content": "I will remember cobalt."},
|
||||||
|
current_turn,
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert current_turn_response.status_code == 200
|
||||||
|
assert multi_turn_response.status_code == 200
|
||||||
|
current_turn_token_ids = current_turn_response.json()["token_ids"]
|
||||||
|
multi_turn_token_ids = multi_turn_response.json()["token_ids"]
|
||||||
|
assert len(multi_turn_token_ids) > len(current_turn_token_ids)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_responses_render_rejects_previous_response_id_over_http(client):
|
||||||
|
response = await client.post(
|
||||||
|
"/v1/responses/render",
|
||||||
|
json={
|
||||||
|
"model": MODEL_NAME,
|
||||||
|
"input": "Continue the previous response.",
|
||||||
|
"previous_response_id": "resp_previous",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 400
|
||||||
|
error = response.json()["error"]
|
||||||
|
assert error["type"] == "invalid_request_error"
|
||||||
|
assert error["param"] == "previous_response_id"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_completion_render_basic(client):
|
async def test_completion_render_basic(client):
|
||||||
"""Test basic completion render endpoint."""
|
"""Test basic completion render endpoint."""
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ RENDER_PATHS = {
|
|||||||
"/v1/completions/render",
|
"/v1/completions/render",
|
||||||
"/v1/completions/derender",
|
"/v1/completions/derender",
|
||||||
"/v1/messages/render",
|
"/v1/messages/render",
|
||||||
|
"/v1/responses/render",
|
||||||
}
|
}
|
||||||
GENERATE_PATH = "/inference/v1/generate"
|
GENERATE_PATH = "/inference/v1/generate"
|
||||||
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
|
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
|
|
||||||
@@ -60,6 +60,7 @@ async def init_generate_state(
|
|||||||
args: "Namespace",
|
args: "Namespace",
|
||||||
request_logger: RequestLogger | None,
|
request_logger: RequestLogger | None,
|
||||||
supported_tasks: tuple["SupportedTask", ...],
|
supported_tasks: tuple["SupportedTask", ...],
|
||||||
|
default_chat_template_kwargs: dict[str, Any],
|
||||||
):
|
):
|
||||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||||
from vllm.entrypoints.chat_utils import load_chat_template
|
from vllm.entrypoints.chat_utils import load_chat_template
|
||||||
@@ -79,12 +80,6 @@ async def init_generate_state(
|
|||||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||||
else:
|
else:
|
||||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||||
|
|
||||||
from vllm.entrypoints.mcp.tool_server import (
|
|
||||||
DemoToolServer,
|
|
||||||
MCPToolServer,
|
|
||||||
ToolServer,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||||
OpenAIServingChatBatch,
|
OpenAIServingChatBatch,
|
||||||
)
|
)
|
||||||
@@ -100,33 +95,11 @@ async def init_generate_state(
|
|||||||
getattr(args, "fingerprint_value", None),
|
getattr(args, "fingerprint_value", None),
|
||||||
)
|
)
|
||||||
|
|
||||||
if args.tool_server == "demo":
|
|
||||||
tool_server: ToolServer | None = DemoToolServer()
|
|
||||||
assert isinstance(tool_server, DemoToolServer)
|
|
||||||
await tool_server.init_and_validate()
|
|
||||||
elif args.tool_server:
|
|
||||||
tool_server = MCPToolServer()
|
|
||||||
await tool_server.add_tool_server(args.tool_server)
|
|
||||||
else:
|
|
||||||
tool_server = None
|
|
||||||
resolved_chat_template = load_chat_template(args.chat_template)
|
resolved_chat_template = load_chat_template(args.chat_template)
|
||||||
|
|
||||||
# Fold the dedicated ``--cohere-format`` CLI flag into the renderer's
|
# Render endpoints are always backed by OnlineRenderer so that chat,
|
||||||
# default chat-template kwargs. The cohere renderer reads
|
# completion, and Responses rendering work on both generate-mode and
|
||||||
# ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4
|
# render-only servers. Created in init_app_state.
|
||||||
# rendering; making this a first-class flag keeps the right format
|
|
||||||
# discoverable for ``vllm serve --tokenizer-mode cohere`` users
|
|
||||||
# without forcing them to hand-construct a JSON dict for
|
|
||||||
# ``--default-chat-template-kwargs``. Per-request overrides still
|
|
||||||
# take precedence (see ``merge_kwargs`` in
|
|
||||||
# ``ChatCompletionRequest.build_chat_params``).
|
|
||||||
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
|
|
||||||
if getattr(args, "cohere_format", None):
|
|
||||||
default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
|
|
||||||
|
|
||||||
# Render endpoints are always backed by OnlineRenderer so that
|
|
||||||
# /v1/chat/completions/render and /v1/completions/render work on both
|
|
||||||
# generate-mode and render-only servers. Created in init_app_state.
|
|
||||||
|
|
||||||
state.openai_serving_responses = (
|
state.openai_serving_responses = (
|
||||||
OpenAIServingResponses(
|
OpenAIServingResponses(
|
||||||
@@ -139,7 +112,7 @@ async def init_generate_state(
|
|||||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||||
enable_auto_tools=args.enable_auto_tool_choice,
|
enable_auto_tools=args.enable_auto_tool_choice,
|
||||||
tool_parser=args.tool_call_parser,
|
tool_parser=args.tool_call_parser,
|
||||||
tool_server=tool_server,
|
tool_server=state.tool_server,
|
||||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||||
enable_force_include_usage=args.enable_force_include_usage,
|
enable_force_include_usage=args.enable_force_include_usage,
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ from starlette.datastructures import State
|
|||||||
|
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.chat_utils import load_chat_template
|
from vllm.entrypoints.chat_utils import load_chat_template
|
||||||
|
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
|
||||||
|
from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||||
@@ -63,6 +65,10 @@ async def init_app_state(
|
|||||||
state.vllm_config = vllm_config
|
state.vllm_config = vllm_config
|
||||||
state.args = args
|
state.args = args
|
||||||
resolved_chat_template = load_chat_template(args.chat_template)
|
resolved_chat_template = load_chat_template(args.chat_template)
|
||||||
|
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||||
|
state.tool_server = (
|
||||||
|
await init_tool_server(args) if "generate" in supported_tasks else None
|
||||||
|
)
|
||||||
|
|
||||||
# Merge default_mm_loras into the static lora_modules
|
# Merge default_mm_loras into the static lora_modules
|
||||||
default_mm_loras = (
|
default_mm_loras = (
|
||||||
@@ -90,7 +96,7 @@ async def init_app_state(
|
|||||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||||
tool_parser=args.tool_call_parser,
|
tool_parser=args.tool_call_parser,
|
||||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||||
log_error_stack=args.log_error_stack,
|
log_error_stack=args.log_error_stack,
|
||||||
)
|
)
|
||||||
state.online_renderer.warmup()
|
state.online_renderer.warmup()
|
||||||
@@ -106,7 +112,7 @@ async def init_app_state(
|
|||||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||||
tool_parser=args.tool_call_parser,
|
tool_parser=args.tool_call_parser,
|
||||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||||
log_error_stack=args.log_error_stack,
|
log_error_stack=args.log_error_stack,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -116,7 +122,7 @@ async def init_app_state(
|
|||||||
request_logger=request_logger,
|
request_logger=request_logger,
|
||||||
chat_template=resolved_chat_template,
|
chat_template=resolved_chat_template,
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -124,7 +130,12 @@ async def init_app_state(
|
|||||||
from vllm.entrypoints.generate.api_router import init_generate_state
|
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||||
|
|
||||||
await init_generate_state(
|
await init_generate_state(
|
||||||
engine_client, state, args, request_logger, supported_tasks
|
engine_client,
|
||||||
|
state,
|
||||||
|
args,
|
||||||
|
request_logger,
|
||||||
|
supported_tasks,
|
||||||
|
default_chat_template_kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
from vllm.entrypoints.scale_out.factories import init_scale_out_state
|
from vllm.entrypoints.scale_out.factories import init_scale_out_state
|
||||||
|
|||||||
@@ -370,6 +370,16 @@ class FrontendArgs(BaseFrontendArgs):
|
|||||||
return frontend_kwargs
|
return frontend_kwargs
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_default_chat_template_kwargs(
|
||||||
|
args: argparse.Namespace,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Resolve renderer defaults, including the dedicated Cohere format flag."""
|
||||||
|
kwargs = dict(args.default_chat_template_kwargs or {})
|
||||||
|
if getattr(args, "cohere_format", None):
|
||||||
|
kwargs.setdefault("cohere_format", args.cohere_format)
|
||||||
|
return kwargs
|
||||||
|
|
||||||
|
|
||||||
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||||
"""Create the CLI argument parser used by the OpenAI API server.
|
"""Create the CLI argument parser used by the OpenAI API server.
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ from starlette.datastructures import State
|
|||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
from vllm.entrypoints.chat_utils import load_chat_template
|
from vllm.entrypoints.chat_utils import load_chat_template
|
||||||
|
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
|
||||||
|
from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||||
from vllm.entrypoints.scale_out.factories import init_render_state
|
from vllm.entrypoints.scale_out.factories import init_render_state
|
||||||
@@ -46,6 +48,8 @@ async def init_render_app_state(
|
|||||||
|
|
||||||
renderer = renderer_from_config(vllm_config)
|
renderer = renderer_from_config(vllm_config)
|
||||||
resolved_chat_template = load_chat_template(args.chat_template)
|
resolved_chat_template = load_chat_template(args.chat_template)
|
||||||
|
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||||
|
state.tool_server = await init_tool_server(args)
|
||||||
|
|
||||||
state.online_renderer = OnlineRenderer(
|
state.online_renderer = OnlineRenderer(
|
||||||
model_config=vllm_config.model_config,
|
model_config=vllm_config.model_config,
|
||||||
@@ -58,7 +62,7 @@ async def init_render_app_state(
|
|||||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||||
tool_parser=args.tool_call_parser,
|
tool_parser=args.tool_call_parser,
|
||||||
reasoning_parser=args.reasoning_parser,
|
reasoning_parser=args.reasoning_parser,
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||||
log_error_stack=args.log_error_stack,
|
log_error_stack=args.log_error_stack,
|
||||||
)
|
)
|
||||||
state.online_renderer.warmup()
|
state.online_renderer.warmup()
|
||||||
@@ -74,7 +78,7 @@ async def init_render_app_state(
|
|||||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||||
tool_parser=args.tool_call_parser,
|
tool_parser=args.tool_call_parser,
|
||||||
reasoning_parser=args.reasoning_parser,
|
reasoning_parser=args.reasoning_parser,
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||||
log_error_stack=args.log_error_stack,
|
log_error_stack=args.log_error_stack,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -85,7 +89,7 @@ async def init_render_app_state(
|
|||||||
request_logger=request_logger,
|
request_logger=request_logger,
|
||||||
chat_template=resolved_chat_template,
|
chat_template=resolved_chat_template,
|
||||||
chat_template_content_format=args.chat_template_content_format,
|
chat_template_content_format=args.chat_template_content_format,
|
||||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||||
trust_request_chat_template=args.trust_request_chat_template,
|
trust_request_chat_template=args.trust_request_chat_template,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ from vllm.logger import init_logger
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
from argparse import Namespace
|
||||||
|
|
||||||
from mcp.types import ListToolsResult
|
from mcp.types import ListToolsResult
|
||||||
|
|
||||||
|
|
||||||
@@ -232,3 +234,16 @@ class DemoToolServer(ToolServer):
|
|||||||
if tool_name not in self.tools:
|
if tool_name not in self.tools:
|
||||||
raise KeyError(f"Tool '{tool_name}' is not supported")
|
raise KeyError(f"Tool '{tool_name}' is not supported")
|
||||||
yield self.tools[tool_name]
|
yield self.tools[tool_name]
|
||||||
|
|
||||||
|
|
||||||
|
async def init_tool_server(args: "Namespace") -> ToolServer | None:
|
||||||
|
tool_server = getattr(args, "tool_server", None)
|
||||||
|
if tool_server == "demo":
|
||||||
|
demo_server = DemoToolServer()
|
||||||
|
await demo_server.init_and_validate()
|
||||||
|
return demo_server
|
||||||
|
if tool_server:
|
||||||
|
mcp_server = MCPToolServer()
|
||||||
|
await mcp_server.add_tool_server(tool_server)
|
||||||
|
return mcp_server
|
||||||
|
return None
|
||||||
|
|||||||
@@ -6,13 +6,11 @@ import time
|
|||||||
from collections import deque
|
from collections import deque
|
||||||
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, Sequence
|
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, Sequence
|
||||||
from contextlib import AsyncExitStack
|
from contextlib import AsyncExitStack
|
||||||
from copy import copy
|
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
from typing import Any, Final
|
from typing import Any, Final, cast
|
||||||
|
|
||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from openai.types.responses import (
|
from openai.types.responses import (
|
||||||
ResponseFunctionToolCall,
|
|
||||||
ResponseOutputItem,
|
ResponseOutputItem,
|
||||||
ResponseOutputMessage,
|
ResponseOutputMessage,
|
||||||
ResponseOutputText,
|
ResponseOutputText,
|
||||||
@@ -20,15 +18,12 @@ from openai.types.responses import (
|
|||||||
response_text_delta_event,
|
response_text_delta_event,
|
||||||
)
|
)
|
||||||
from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
|
from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
|
||||||
from openai.types.responses.tool import Mcp, Tool
|
|
||||||
from openai_harmony import Message as OpenAIHarmonyMessage
|
|
||||||
from pydantic import TypeAdapter
|
from pydantic import TypeAdapter
|
||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.config.utils import replace
|
from vllm.config.utils import replace
|
||||||
from vllm.engine.protocol import EngineClient
|
from vllm.engine.protocol import EngineClient
|
||||||
from vllm.entrypoints.chat_utils import (
|
from vllm.entrypoints.chat_utils import (
|
||||||
ChatCompletionMessageParam,
|
|
||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.generate.base.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
@@ -38,24 +33,13 @@ from vllm.entrypoints.generate.base.protocol import (
|
|||||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
|
||||||
build_harmony_preamble,
|
|
||||||
extract_instructions_from_messages,
|
|
||||||
get_user_message,
|
|
||||||
has_custom_tools,
|
|
||||||
render_for_completion,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.responses.context import (
|
from vllm.entrypoints.openai.responses.context import (
|
||||||
ConversationContext,
|
ConversationContext,
|
||||||
HarmonyContext,
|
HarmonyContext,
|
||||||
ParsableContext,
|
ParsableContext,
|
||||||
SimpleContext,
|
SimpleContext,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.responses.harmony import (
|
from vllm.entrypoints.openai.responses.harmony import harmony_to_response_output
|
||||||
construct_harmony_previous_input_messages,
|
|
||||||
harmony_to_response_output,
|
|
||||||
response_input_to_harmony,
|
|
||||||
)
|
|
||||||
from vllm.entrypoints.openai.responses.protocol import (
|
from vllm.entrypoints.openai.responses.protocol import (
|
||||||
InputTokensDetails,
|
InputTokensDetails,
|
||||||
OutputTokensDetails,
|
OutputTokensDetails,
|
||||||
@@ -80,8 +64,6 @@ from vllm.entrypoints.openai.responses.streaming_events import (
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.responses.utils import (
|
from vllm.entrypoints.openai.responses.utils import (
|
||||||
build_response_output_items,
|
build_response_output_items,
|
||||||
construct_input_messages,
|
|
||||||
construct_tool_dicts,
|
|
||||||
extract_function_tool_names,
|
extract_function_tool_names,
|
||||||
extract_tool_types,
|
extract_tool_types,
|
||||||
)
|
)
|
||||||
@@ -89,14 +71,19 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
|||||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||||
from vllm.inputs import EngineInput, tokens_input
|
from vllm.inputs import EngineInput
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.logprobs import Logprob as SampleLogprob
|
from vllm.logprobs import Logprob as SampleLogprob
|
||||||
from vllm.logprobs import SampleLogprobs
|
from vllm.logprobs import SampleLogprobs
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
from vllm.outputs import CompletionOutput
|
from vllm.outputs import CompletionOutput
|
||||||
from vllm.parser import Parser, ParserManager
|
from vllm.parser import Parser, ParserManager
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers import TokenizeParams
|
||||||
|
from vllm.renderers.online_renderer import (
|
||||||
|
OnlineRenderer,
|
||||||
|
ResponsesPreviousMessages,
|
||||||
|
ResponsesRenderResult,
|
||||||
|
)
|
||||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.utils import random_uuid
|
from vllm.utils import random_uuid
|
||||||
@@ -105,45 +92,6 @@ from vllm.utils.collection_utils import as_list
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _extract_allowed_tools_from_mcp_requests(
|
|
||||||
tools: list[Tool],
|
|
||||||
) -> dict[str, list[str] | None]:
|
|
||||||
"""
|
|
||||||
Extract allowed_tools mapping from MCP tool requests.
|
|
||||||
|
|
||||||
Returns a dictionary mapping server_label to allowed_tools list.
|
|
||||||
Handles both list format and McpAllowedToolsMcpToolFilter object format.
|
|
||||||
|
|
||||||
Special handling:
|
|
||||||
- If allowed_tools is None, returns None (allows all tools)
|
|
||||||
- If allowed_tools contains "*", returns None (allows all tools)
|
|
||||||
- Otherwise, returns the list of specific tool names
|
|
||||||
|
|
||||||
This function can be reused for both harmony and non-harmony MCP calls.
|
|
||||||
"""
|
|
||||||
allowed_tools_map: dict[str, list[str] | None] = {}
|
|
||||||
for tool in tools:
|
|
||||||
if not isinstance(tool, Mcp):
|
|
||||||
continue
|
|
||||||
|
|
||||||
# allowed_tools can be a list or an object with tool_names
|
|
||||||
# Extract the actual list of tool names
|
|
||||||
allowed_tools_val = None
|
|
||||||
if tool.allowed_tools is not None:
|
|
||||||
if isinstance(tool.allowed_tools, list):
|
|
||||||
allowed_tools_val = tool.allowed_tools
|
|
||||||
elif hasattr(tool.allowed_tools, "tool_names"):
|
|
||||||
# It's an McpAllowedToolsMcpToolFilter object
|
|
||||||
allowed_tools_val = tool.allowed_tools.tool_names
|
|
||||||
|
|
||||||
# Normalize "*" to None (both mean "allow all tools")
|
|
||||||
if allowed_tools_val is not None and "*" in allowed_tools_val:
|
|
||||||
allowed_tools_val = None
|
|
||||||
|
|
||||||
allowed_tools_map[tool.server_label] = allowed_tools_val
|
|
||||||
return allowed_tools_map
|
|
||||||
|
|
||||||
|
|
||||||
class OpenAIServingResponses(GenerateBaseServing):
|
class OpenAIServingResponses(GenerateBaseServing):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -226,7 +174,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
# HACK(woosuk): This is a hack. We should use a better store.
|
# HACK(woosuk): This is a hack. We should use a better store.
|
||||||
# FIXME: If enable_store=True, this may cause a memory leak since we
|
# FIXME: If enable_store=True, this may cause a memory leak since we
|
||||||
# never remove messages from the store.
|
# never remove messages from the store.
|
||||||
self.msg_store: dict[str, list[ChatCompletionMessageParam]] = {}
|
self.msg_store: dict[str, ResponsesPreviousMessages] = {}
|
||||||
|
|
||||||
# HACK(wuhang): This is a hack. We should use a better store.
|
# HACK(wuhang): This is a hack. We should use a better store.
|
||||||
# FIXME: If enable_store=True, this may cause a memory leak since we
|
# FIXME: If enable_store=True, this may cause a memory leak since we
|
||||||
@@ -322,6 +270,33 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def _render_resolved_response_inputs(
|
||||||
|
self,
|
||||||
|
request: ResponsesRequest,
|
||||||
|
prev_response: ResponsesResponse | None,
|
||||||
|
*,
|
||||||
|
skip_mm_cache: bool = False,
|
||||||
|
) -> ResponsesRenderResult | ErrorResponse:
|
||||||
|
previous_messages: ResponsesPreviousMessages | None = None
|
||||||
|
previous_response_outputs: list[ResponseOutputItem] | None = None
|
||||||
|
if prev_response is not None:
|
||||||
|
stored_messages = (
|
||||||
|
self.msg_store[prev_response.id]
|
||||||
|
if self.use_harmony
|
||||||
|
else self.msg_store.get(prev_response.id)
|
||||||
|
)
|
||||||
|
if stored_messages is not None:
|
||||||
|
previous_messages = stored_messages.copy()
|
||||||
|
previous_response_outputs = list(prev_response.output)
|
||||||
|
|
||||||
|
return await self.online_renderer.render_responses(
|
||||||
|
request,
|
||||||
|
previous_messages=previous_messages,
|
||||||
|
previous_response_outputs=previous_response_outputs,
|
||||||
|
tool_server=self.tool_server,
|
||||||
|
skip_mm_cache=skip_mm_cache,
|
||||||
|
)
|
||||||
|
|
||||||
async def create_responses(
|
async def create_responses(
|
||||||
self,
|
self,
|
||||||
request: ResponsesRequest,
|
request: ResponsesRequest,
|
||||||
@@ -374,12 +349,14 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
lora_request = self._maybe_get_adapters(request)
|
lora_request = self._maybe_get_adapters(request)
|
||||||
model_name = self.models.model_name(lora_request)
|
model_name = self.models.model_name(lora_request)
|
||||||
|
|
||||||
if self.use_harmony:
|
render_result = await self._render_resolved_response_inputs(
|
||||||
messages, engine_inputs = self._make_request_with_harmony(
|
request,
|
||||||
request, prev_response
|
prev_response,
|
||||||
)
|
)
|
||||||
else:
|
if isinstance(render_result, ErrorResponse):
|
||||||
messages, engine_inputs = await self._make_request(request, prev_response)
|
return render_result
|
||||||
|
messages = render_result.messages
|
||||||
|
engine_inputs = [render_result.engine_input]
|
||||||
|
|
||||||
request_metadata = RequestResponseMetadata(request_id=request.request_id)
|
request_metadata = RequestResponseMetadata(request_id=request.request_id)
|
||||||
if raw_request:
|
if raw_request:
|
||||||
@@ -508,6 +485,11 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
engine_input=engine_input,
|
engine_input=engine_input,
|
||||||
sampling_params=sampling_params,
|
sampling_params=sampling_params,
|
||||||
context=context,
|
context=context,
|
||||||
|
tok_params=(
|
||||||
|
request.build_tok_params(self.model_config)
|
||||||
|
if isinstance(context, HarmonyContext)
|
||||||
|
else None
|
||||||
|
),
|
||||||
lora_request=lora_request,
|
lora_request=lora_request,
|
||||||
priority=self._get_priority(request, raw_request),
|
priority=self._get_priority(request, raw_request),
|
||||||
trace_headers=trace_headers,
|
trace_headers=trace_headers,
|
||||||
@@ -599,60 +581,28 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
request_metadata,
|
request_metadata,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _make_request(
|
|
||||||
self,
|
|
||||||
request: ResponsesRequest,
|
|
||||||
prev_response: ResponsesResponse | None,
|
|
||||||
):
|
|
||||||
tool_dicts = construct_tool_dicts(
|
|
||||||
request.tools,
|
|
||||||
request.tool_choice,
|
|
||||||
exclude_tools_when_tool_choice_none=(
|
|
||||||
self.online_renderer.exclude_tools_when_tool_choice_none
|
|
||||||
),
|
|
||||||
)
|
|
||||||
# Construct the input messages.
|
|
||||||
messages = construct_input_messages(
|
|
||||||
request_instructions=request.instructions,
|
|
||||||
request_input=request.input,
|
|
||||||
prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,
|
|
||||||
prev_response_output=prev_response.output if prev_response else None,
|
|
||||||
)
|
|
||||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
|
||||||
_, engine_inputs = await self.online_renderer.preprocess_chat(
|
|
||||||
request,
|
|
||||||
messages,
|
|
||||||
default_template=self.chat_template,
|
|
||||||
default_template_content_format=self.chat_template_content_format,
|
|
||||||
default_template_kwargs=chat_template_kwargs,
|
|
||||||
tool_dicts=tool_dicts,
|
|
||||||
parser=self.parser,
|
|
||||||
)
|
|
||||||
return messages, engine_inputs
|
|
||||||
|
|
||||||
async def _render_next_turn(
|
async def _render_next_turn(
|
||||||
self,
|
self,
|
||||||
request: ResponsesRequest,
|
request: ResponsesRequest,
|
||||||
messages: list[ResponseInputOutputItem],
|
messages: list[ResponseInputOutputItem],
|
||||||
tool_dicts: list[dict[str, Any]] | None,
|
) -> list[EngineInput]:
|
||||||
parser: type[Parser] | None,
|
render_request = request.model_copy(
|
||||||
chat_template: str | None,
|
update={
|
||||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
"input": list(messages),
|
||||||
):
|
"instructions": None,
|
||||||
new_messages = construct_input_messages(
|
"previous_input_messages": None,
|
||||||
request_input=messages,
|
"previous_response_id": None,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
render_result = await self.online_renderer.render_responses(
|
||||||
_, engine_inputs = await self.online_renderer.preprocess_chat(
|
render_request,
|
||||||
request,
|
previous_messages=None,
|
||||||
new_messages,
|
previous_response_outputs=None,
|
||||||
default_template=chat_template,
|
tool_server=self.tool_server,
|
||||||
default_template_content_format=chat_template_content_format,
|
|
||||||
default_template_kwargs=chat_template_kwargs,
|
|
||||||
tool_dicts=tool_dicts,
|
|
||||||
parser=parser,
|
|
||||||
)
|
)
|
||||||
return engine_inputs
|
if isinstance(render_result, ErrorResponse):
|
||||||
|
raise VLLMValidationError(render_result.error.message)
|
||||||
|
return [render_result.engine_input]
|
||||||
|
|
||||||
async def _generate_with_builtin_tools(
|
async def _generate_with_builtin_tools(
|
||||||
self,
|
self,
|
||||||
@@ -660,6 +610,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
engine_input: EngineInput,
|
engine_input: EngineInput,
|
||||||
sampling_params: SamplingParams,
|
sampling_params: SamplingParams,
|
||||||
context: ConversationContext,
|
context: ConversationContext,
|
||||||
|
tok_params: TokenizeParams | None = None,
|
||||||
lora_request: LoRARequest | None = None,
|
lora_request: LoRARequest | None = None,
|
||||||
priority: int = 0,
|
priority: int = 0,
|
||||||
trace_headers: Mapping[str, str] | None = None,
|
trace_headers: Mapping[str, str] | None = None,
|
||||||
@@ -667,6 +618,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||||
):
|
):
|
||||||
max_model_len = self.model_config.max_model_len
|
max_model_len = self.model_config.max_model_len
|
||||||
|
cache_salt = cast(str | None, engine_input.get("cache_salt"))
|
||||||
|
|
||||||
orig_priority = priority
|
orig_priority = priority
|
||||||
sub_request = 0
|
sub_request = 0
|
||||||
@@ -711,18 +663,19 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
# Create inputs for the next turn.
|
# Create inputs for the next turn.
|
||||||
# Render the next prompt token ids and update sampling_params.
|
# Render the next prompt token ids and update sampling_params.
|
||||||
if isinstance(context, HarmonyContext):
|
if isinstance(context, HarmonyContext):
|
||||||
token_ids = context.render_for_completion()
|
engine_input = self.online_renderer.render_responses_harmony_messages(
|
||||||
engine_input = tokens_input(token_ids)
|
context.messages,
|
||||||
|
cache_salt=cache_salt,
|
||||||
|
tok_params=tok_params,
|
||||||
|
)
|
||||||
|
|
||||||
sampling_params.max_tokens = max_model_len - len(token_ids)
|
sampling_params.max_tokens = max_model_len - self._extract_prompt_len(
|
||||||
|
engine_input
|
||||||
|
)
|
||||||
elif isinstance(context, ParsableContext):
|
elif isinstance(context, ParsableContext):
|
||||||
(engine_input,) = await self._render_next_turn(
|
(engine_input,) = await self._render_next_turn(
|
||||||
context.request,
|
context.request,
|
||||||
context.response_messages,
|
context.response_messages,
|
||||||
context.tool_dicts,
|
|
||||||
context.parser_cls,
|
|
||||||
context.chat_template,
|
|
||||||
context.chat_template_content_format,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
sampling_params.max_tokens = get_max_tokens(
|
sampling_params.max_tokens = get_max_tokens(
|
||||||
@@ -740,28 +693,6 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
priority = orig_priority - 1
|
priority = orig_priority - 1
|
||||||
sub_request += 1
|
sub_request += 1
|
||||||
|
|
||||||
def _make_request_with_harmony(
|
|
||||||
self,
|
|
||||||
request: ResponsesRequest,
|
|
||||||
prev_response: ResponsesResponse | None,
|
|
||||||
):
|
|
||||||
if self.parser is not None:
|
|
||||||
# HarmonyParser doesn't need chat_template_kwargs
|
|
||||||
# TODO: Unify adjust_request() call with non-harmony branch
|
|
||||||
self.parser(
|
|
||||||
self.renderer.get_tokenizer(),
|
|
||||||
request.tools,
|
|
||||||
model_config=self.model_config,
|
|
||||||
).adjust_request(request=request)
|
|
||||||
|
|
||||||
arrival_time = time.time()
|
|
||||||
messages = self._construct_input_messages_with_harmony(request, prev_response)
|
|
||||||
prompt_token_ids = render_for_completion(messages)
|
|
||||||
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
|
|
||||||
engine_input["arrival_time"] = arrival_time
|
|
||||||
|
|
||||||
return messages, [engine_input]
|
|
||||||
|
|
||||||
async def _initialize_tool_sessions(
|
async def _initialize_tool_sessions(
|
||||||
self,
|
self,
|
||||||
request: ResponsesRequest,
|
request: ResponsesRequest,
|
||||||
@@ -1104,113 +1035,6 @@ class OpenAIServingResponses(GenerateBaseServing):
|
|||||||
)
|
)
|
||||||
]
|
]
|
||||||
|
|
||||||
def _get_harmony_builtin_tool_descriptions(
|
|
||||||
self, request: ResponsesRequest, tool_types: set[str]
|
|
||||||
) -> dict[str, str | None]:
|
|
||||||
# Extract allowed_tools from MCP tool requests
|
|
||||||
allowed_tools_map = _extract_allowed_tools_from_mcp_requests(request.tools)
|
|
||||||
|
|
||||||
# Get filtered tool descriptions first.
|
|
||||||
# If get_tool_description returns None (due to filtering), the tool is disabled.
|
|
||||||
browser_description = (
|
|
||||||
self.tool_server.get_tool_description(
|
|
||||||
"browser", allowed_tools_map.get("web_search_preview")
|
|
||||||
)
|
|
||||||
if "web_search_preview" in tool_types
|
|
||||||
and self.tool_server is not None
|
|
||||||
and self.tool_server.has_tool("browser")
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
python_description = (
|
|
||||||
self.tool_server.get_tool_description(
|
|
||||||
"python", allowed_tools_map.get("code_interpreter")
|
|
||||||
)
|
|
||||||
if "code_interpreter" in tool_types
|
|
||||||
and self.tool_server is not None
|
|
||||||
and self.tool_server.has_tool("python")
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
container_description = (
|
|
||||||
self.tool_server.get_tool_description(
|
|
||||||
"container", allowed_tools_map.get("container")
|
|
||||||
)
|
|
||||||
if "container" in tool_types
|
|
||||||
and self.tool_server is not None
|
|
||||||
and self.tool_server.has_tool("container")
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
return {
|
|
||||||
"browser_description": browser_description,
|
|
||||||
"python_description": python_description,
|
|
||||||
"container_description": container_description,
|
|
||||||
}
|
|
||||||
|
|
||||||
def _construct_input_messages_with_harmony(
|
|
||||||
self,
|
|
||||||
request: ResponsesRequest,
|
|
||||||
prev_response: ResponsesResponse | None,
|
|
||||||
) -> list[OpenAIHarmonyMessage]:
|
|
||||||
messages: list[OpenAIHarmonyMessage] = []
|
|
||||||
request_input = request.input
|
|
||||||
if prev_response is None:
|
|
||||||
# New conversation.
|
|
||||||
tool_types = extract_tool_types(request.tools)
|
|
||||||
with_custom_tools = has_custom_tools(tool_types)
|
|
||||||
instructions = request.instructions
|
|
||||||
if instructions is None and isinstance(request_input, list):
|
|
||||||
instructions, request_input = extract_instructions_from_messages(
|
|
||||||
request_input
|
|
||||||
)
|
|
||||||
tool_descriptions = self._get_harmony_builtin_tool_descriptions(
|
|
||||||
request, tool_types
|
|
||||||
)
|
|
||||||
tools = request.tools if with_custom_tools else None
|
|
||||||
messages.extend(
|
|
||||||
build_harmony_preamble(
|
|
||||||
instructions=instructions,
|
|
||||||
tools=tools,
|
|
||||||
reasoning_effort=(
|
|
||||||
request.reasoning.effort if request.reasoning else None
|
|
||||||
),
|
|
||||||
with_custom_tools=with_custom_tools,
|
|
||||||
**tool_descriptions,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
messages += construct_harmony_previous_input_messages(request)
|
|
||||||
|
|
||||||
else:
|
|
||||||
# Continue the previous conversation.
|
|
||||||
# FIXME(woosuk): Currently, request params like reasoning and
|
|
||||||
# instructions are ignored.
|
|
||||||
prev_msgs = self.msg_store[prev_response.id]
|
|
||||||
|
|
||||||
messages.extend(prev_msgs)
|
|
||||||
# Append the new input.
|
|
||||||
# Responses API supports simple text inputs without chat format.
|
|
||||||
if isinstance(request_input, str):
|
|
||||||
# Skip empty string input when previous_input_messages supplies
|
|
||||||
# the full conversation history --- an empty trailing user message
|
|
||||||
# confuses the model into thinking nothing was sent.
|
|
||||||
if request_input or not request.previous_input_messages:
|
|
||||||
messages.append(get_user_message(request_input))
|
|
||||||
else:
|
|
||||||
if prev_response is not None:
|
|
||||||
prev_outputs = copy(prev_response.output)
|
|
||||||
else:
|
|
||||||
prev_outputs = []
|
|
||||||
for response_msg in request_input:
|
|
||||||
new_msg = response_input_to_harmony(response_msg, prev_outputs)
|
|
||||||
if new_msg is not None:
|
|
||||||
messages.append(new_msg)
|
|
||||||
|
|
||||||
# User passes in a tool call request and its output. We need
|
|
||||||
# to add the tool call request to prev_outputs so that
|
|
||||||
# response_input_to_harmony can find the tool call request when
|
|
||||||
# parsing the tool call output.
|
|
||||||
if isinstance(response_msg, ResponseFunctionToolCall):
|
|
||||||
prev_outputs.append(response_msg)
|
|
||||||
return messages
|
|
||||||
|
|
||||||
async def _run_background_request_stream(
|
async def _run_background_request_stream(
|
||||||
self,
|
self,
|
||||||
request: ResponsesRequest,
|
request: ResponsesRequest,
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ def init_render_state(
|
|||||||
state.openai_serving_models,
|
state.openai_serving_models,
|
||||||
state.online_renderer,
|
state.online_renderer,
|
||||||
request_logger=request_logger,
|
request_logger=request_logger,
|
||||||
|
tool_server=state.tool_server,
|
||||||
)
|
)
|
||||||
|
|
||||||
state.serving_derender = ServingDerender(
|
state.serving_derender = ServingDerender(
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from fastapi.responses import JSONResponse
|
|||||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||||
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
@@ -95,3 +96,25 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
|
|||||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||||
|
|
||||||
return JSONResponse(content=[item.model_dump() for item in result])
|
return JSONResponse(content=[item.model_dump() for item in result])
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/v1/responses/render",
|
||||||
|
dependencies=[Depends(validate_json_request)],
|
||||||
|
response_model=GenerateRequest,
|
||||||
|
responses={
|
||||||
|
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||||
|
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||||
|
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||||
|
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
async def render_responses(request: ResponsesRequest, raw_request: Request):
|
||||||
|
handler = render(raw_request)
|
||||||
|
if handler is None:
|
||||||
|
raise NotImplementedError("The model does not support Responses Render API")
|
||||||
|
|
||||||
|
result = await handler.render_responses_request(request)
|
||||||
|
if isinstance(result, ErrorResponse):
|
||||||
|
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||||
|
return JSONResponse(content=result.model_dump())
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
@@ -8,6 +10,7 @@ from vllm.entrypoints.openai.models.serving import (
|
|||||||
OpenAIModelRegistry,
|
OpenAIModelRegistry,
|
||||||
OpenAIServingModels,
|
OpenAIServingModels,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
||||||
extract_mm_features,
|
extract_mm_features,
|
||||||
)
|
)
|
||||||
@@ -31,6 +34,9 @@ from vllm.utils import random_uuid
|
|||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||||
|
|
||||||
|
|
||||||
class ServingRender(BaseServing):
|
class ServingRender(BaseServing):
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -39,6 +45,7 @@ class ServingRender(BaseServing):
|
|||||||
online_renderer: "OnlineRenderer",
|
online_renderer: "OnlineRenderer",
|
||||||
*,
|
*,
|
||||||
request_logger: RequestLogger | None = None,
|
request_logger: RequestLogger | None = None,
|
||||||
|
tool_server: "ToolServer | None" = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__(
|
super().__init__(
|
||||||
models=models,
|
models=models,
|
||||||
@@ -47,6 +54,7 @@ class ServingRender(BaseServing):
|
|||||||
)
|
)
|
||||||
|
|
||||||
self.online_renderer = online_renderer
|
self.online_renderer = online_renderer
|
||||||
|
self.tool_server = tool_server
|
||||||
|
|
||||||
self._merge_inline_system = (
|
self._merge_inline_system = (
|
||||||
AnthropicServingMessages._detect_merge_inline_system(
|
AnthropicServingMessages._detect_merge_inline_system(
|
||||||
@@ -229,6 +237,64 @@ class ServingRender(BaseServing):
|
|||||||
|
|
||||||
return generate_requests
|
return generate_requests
|
||||||
|
|
||||||
|
async def render_responses_request(
|
||||||
|
self,
|
||||||
|
request: ResponsesRequest,
|
||||||
|
) -> GenerateRequest | ErrorResponse:
|
||||||
|
error_check_ret = await self._check_model(request)
|
||||||
|
if error_check_ret is not None:
|
||||||
|
return error_check_ret
|
||||||
|
if request.previous_response_id is not None:
|
||||||
|
return self.create_error_response(
|
||||||
|
message=(
|
||||||
|
"previous_response_id is not supported by the stateless "
|
||||||
|
"render endpoint."
|
||||||
|
),
|
||||||
|
err_type="invalid_request_error",
|
||||||
|
param="previous_response_id",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await self.online_renderer.render_responses(
|
||||||
|
request,
|
||||||
|
previous_messages=None,
|
||||||
|
previous_response_outputs=None,
|
||||||
|
tool_server=self.tool_server,
|
||||||
|
skip_mm_cache=True,
|
||||||
|
)
|
||||||
|
if isinstance(result, ErrorResponse):
|
||||||
|
return result
|
||||||
|
|
||||||
|
engine_input = result.engine_input
|
||||||
|
prompt_components = extract_prompt_components(self.model_config, engine_input)
|
||||||
|
token_ids = prompt_components.token_ids
|
||||||
|
if not token_ids:
|
||||||
|
return self.create_error_response("No token_ids rendered")
|
||||||
|
|
||||||
|
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||||
|
max_tokens = get_max_tokens(
|
||||||
|
self.model_config.max_model_len,
|
||||||
|
request.max_output_tokens,
|
||||||
|
input_length,
|
||||||
|
self.default_sampling_params,
|
||||||
|
self.override_max_tokens,
|
||||||
|
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
|
||||||
|
)
|
||||||
|
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||||
|
|
||||||
|
return GenerateRequest(
|
||||||
|
request_id=request.request_id,
|
||||||
|
token_ids=list(token_ids),
|
||||||
|
features=self._extract_mm_features(engine_input),
|
||||||
|
sampling_params=params,
|
||||||
|
model=request.model,
|
||||||
|
stream=bool(request.stream),
|
||||||
|
cache_salt=request.cache_salt,
|
||||||
|
priority=request.priority,
|
||||||
|
kv_transfer_params=request.kv_transfer_params,
|
||||||
|
ec_transfer_params=request.ec_transfer_params,
|
||||||
|
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||||
|
)
|
||||||
|
|
||||||
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
||||||
"""Fields the EC consumer still requires after embeddings transfer."""
|
"""Fields the EC consumer still requires after embeddings transfer."""
|
||||||
if self._placeholder_metadata_parser_failed:
|
if self._placeholder_metadata_parser_failed:
|
||||||
|
|||||||
@@ -1,13 +1,19 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import time
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
from typing import Any
|
from typing import TYPE_CHECKING, Any, TypeAlias
|
||||||
|
|
||||||
|
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
|
||||||
|
from openai.types.responses.tool import Mcp, Tool
|
||||||
from openai_harmony import Message as OpenAIMessage
|
from openai_harmony import Message as OpenAIMessage
|
||||||
|
from openai_harmony import ToolNamespaceConfig
|
||||||
|
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
from vllm.entrypoints.chat_utils import (
|
from vllm.entrypoints.chat_utils import (
|
||||||
|
ChatCompletionMessageParam,
|
||||||
ChatTemplateContentFormatOption,
|
ChatTemplateContentFormatOption,
|
||||||
ConversationMessage,
|
ConversationMessage,
|
||||||
)
|
)
|
||||||
@@ -19,12 +25,24 @@ from vllm.entrypoints.openai.completion.protocol import (
|
|||||||
CompletionRequest,
|
CompletionRequest,
|
||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||||
|
BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
|
||||||
build_harmony_preamble,
|
build_harmony_preamble,
|
||||||
extract_instructions_from_messages,
|
extract_instructions_from_messages,
|
||||||
|
get_user_message,
|
||||||
|
has_custom_tools,
|
||||||
parse_chat_inputs_to_harmony_messages,
|
parse_chat_inputs_to_harmony_messages,
|
||||||
render_for_completion,
|
render_for_completion,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.responses.harmony import (
|
||||||
|
construct_harmony_previous_input_messages,
|
||||||
|
response_input_to_harmony,
|
||||||
|
)
|
||||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
|
from vllm.entrypoints.openai.responses.utils import (
|
||||||
|
construct_input_messages,
|
||||||
|
construct_tool_dicts,
|
||||||
|
extract_tool_types,
|
||||||
|
)
|
||||||
from vllm.entrypoints.serve import create_error_response
|
from vllm.entrypoints.serve import create_error_response
|
||||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
@@ -32,11 +50,12 @@ from vllm.inputs import (
|
|||||||
EngineInput,
|
EngineInput,
|
||||||
PromptType,
|
PromptType,
|
||||||
SingletonPrompt,
|
SingletonPrompt,
|
||||||
|
TokensPrompt,
|
||||||
tokens_input,
|
tokens_input,
|
||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.parser import Parser, ParserManager
|
from vllm.parser import Parser, ParserManager
|
||||||
from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs
|
from vllm.renderers import BaseRenderer, ChatParams, TokenizeParams, merge_kwargs
|
||||||
from vllm.renderers.inputs.preprocess import (
|
from vllm.renderers.inputs.preprocess import (
|
||||||
parse_model_prompt,
|
parse_model_prompt,
|
||||||
prompt_to_seq,
|
prompt_to_seq,
|
||||||
@@ -46,6 +65,39 @@ from vllm.utils.mistral import mt as _mt
|
|||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||||
|
|
||||||
|
|
||||||
|
ResponsesPreviousMessages: TypeAlias = (
|
||||||
|
list[ChatCompletionMessageParam] | list[OpenAIMessage]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ResponsesRenderResult:
|
||||||
|
messages: ResponsesPreviousMessages
|
||||||
|
engine_input: EngineInput
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_allowed_tools_from_mcp_requests(
|
||||||
|
tools: list[Tool],
|
||||||
|
) -> dict[str, list[str] | None]:
|
||||||
|
allowed_tools_map: dict[str, list[str] | None] = {}
|
||||||
|
for tool in tools:
|
||||||
|
if not isinstance(tool, Mcp):
|
||||||
|
continue
|
||||||
|
|
||||||
|
allowed_tools = None
|
||||||
|
if isinstance(tool.allowed_tools, list):
|
||||||
|
allowed_tools = tool.allowed_tools
|
||||||
|
elif tool.allowed_tools is not None:
|
||||||
|
allowed_tools = tool.allowed_tools.tool_names
|
||||||
|
if allowed_tools is not None and "*" in allowed_tools:
|
||||||
|
allowed_tools = None
|
||||||
|
allowed_tools_map[tool.server_label] = allowed_tools
|
||||||
|
return allowed_tools_map
|
||||||
|
|
||||||
|
|
||||||
def _reused_prompt_token_ids(request: Any) -> list[int] | None:
|
def _reused_prompt_token_ids(request: Any) -> list[int] | None:
|
||||||
"""Pop prompt token ids forwarded for decode-side reuse, if any.
|
"""Pop prompt token ids forwarded for decode-side reuse, if any.
|
||||||
@@ -217,6 +269,219 @@ class OnlineRenderer:
|
|||||||
|
|
||||||
return conversation, engine_inputs
|
return conversation, engine_inputs
|
||||||
|
|
||||||
|
async def render_responses(
|
||||||
|
self,
|
||||||
|
request: ResponsesRequest,
|
||||||
|
*,
|
||||||
|
previous_messages: ResponsesPreviousMessages | None = None,
|
||||||
|
previous_response_outputs: list[ResponseOutputItem] | None = None,
|
||||||
|
tool_server: "ToolServer | None" = None,
|
||||||
|
skip_mm_cache: bool = False,
|
||||||
|
) -> ResponsesRenderResult | ErrorResponse:
|
||||||
|
"""Render a Responses request using only explicitly supplied history."""
|
||||||
|
template_error = self.validate_chat_template(
|
||||||
|
request_chat_template=None,
|
||||||
|
chat_template_kwargs=request.chat_template_kwargs,
|
||||||
|
trust_request_chat_template=self.trust_request_chat_template,
|
||||||
|
)
|
||||||
|
if template_error is not None:
|
||||||
|
return template_error
|
||||||
|
|
||||||
|
if self.use_harmony:
|
||||||
|
return self._render_responses_with_harmony(
|
||||||
|
request,
|
||||||
|
previous_messages=previous_messages,
|
||||||
|
previous_response_outputs=previous_response_outputs,
|
||||||
|
tool_server=tool_server,
|
||||||
|
)
|
||||||
|
|
||||||
|
if previous_messages is not None and any(
|
||||||
|
isinstance(message, OpenAIMessage) for message in previous_messages
|
||||||
|
):
|
||||||
|
return self.create_error_response(
|
||||||
|
"Non-Harmony Responses history must use chat messages.",
|
||||||
|
err_type="invalid_request_error",
|
||||||
|
param="previous_response_id",
|
||||||
|
)
|
||||||
|
|
||||||
|
tool_dicts = construct_tool_dicts(
|
||||||
|
request.tools,
|
||||||
|
request.tool_choice,
|
||||||
|
exclude_tools_when_tool_choice_none=(
|
||||||
|
self.exclude_tools_when_tool_choice_none
|
||||||
|
),
|
||||||
|
)
|
||||||
|
messages = construct_input_messages(
|
||||||
|
request_instructions=request.instructions,
|
||||||
|
request_input=request.input,
|
||||||
|
prev_msg=list(previous_messages) if previous_messages is not None else None,
|
||||||
|
prev_response_output=(
|
||||||
|
list(previous_response_outputs)
|
||||||
|
if previous_response_outputs is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
chat_template_kwargs = (
|
||||||
|
request.build_chat_params(
|
||||||
|
self.chat_template,
|
||||||
|
self.chat_template_content_format,
|
||||||
|
)
|
||||||
|
.with_defaults(self.default_chat_template_kwargs)
|
||||||
|
.chat_template_kwargs
|
||||||
|
)
|
||||||
|
_, engine_inputs = await self.preprocess_chat(
|
||||||
|
request,
|
||||||
|
messages,
|
||||||
|
default_template=self.chat_template,
|
||||||
|
default_template_content_format=self.chat_template_content_format,
|
||||||
|
default_template_kwargs=chat_template_kwargs,
|
||||||
|
tool_dicts=tool_dicts,
|
||||||
|
parser=self.parser,
|
||||||
|
skip_mm_cache=skip_mm_cache,
|
||||||
|
)
|
||||||
|
return self._responses_render_result(messages, engine_inputs)
|
||||||
|
|
||||||
|
def _render_responses_with_harmony(
|
||||||
|
self,
|
||||||
|
request: ResponsesRequest,
|
||||||
|
*,
|
||||||
|
previous_messages: ResponsesPreviousMessages | None,
|
||||||
|
previous_response_outputs: list[ResponseOutputItem] | None,
|
||||||
|
tool_server: "ToolServer | None",
|
||||||
|
) -> ResponsesRenderResult | ErrorResponse:
|
||||||
|
if self.parser is not None:
|
||||||
|
# HarmonyParser doesn't need chat_template_kwargs
|
||||||
|
# TODO: Unify adjust_request() call with non-harmony branch
|
||||||
|
self.parser(
|
||||||
|
self.renderer.get_tokenizer(),
|
||||||
|
request.tools,
|
||||||
|
model_config=self.model_config,
|
||||||
|
).adjust_request(request=request)
|
||||||
|
|
||||||
|
if previous_messages is not None and any(
|
||||||
|
not isinstance(message, OpenAIMessage) for message in previous_messages
|
||||||
|
):
|
||||||
|
return self.create_error_response(
|
||||||
|
"Harmony Responses history must use Harmony messages.",
|
||||||
|
err_type="invalid_request_error",
|
||||||
|
param="previous_response_id",
|
||||||
|
)
|
||||||
|
|
||||||
|
messages: list[OpenAIMessage] = []
|
||||||
|
request_input = request.input
|
||||||
|
if previous_messages is None:
|
||||||
|
tool_types = extract_tool_types(request.tools)
|
||||||
|
with_custom_tools = has_custom_tools(tool_types)
|
||||||
|
instructions = request.instructions
|
||||||
|
if instructions is None and isinstance(request_input, list):
|
||||||
|
instructions, request_input = extract_instructions_from_messages(
|
||||||
|
request_input
|
||||||
|
)
|
||||||
|
descriptions = self._get_harmony_builtin_tool_descriptions(
|
||||||
|
request.tools,
|
||||||
|
tool_types,
|
||||||
|
tool_server,
|
||||||
|
)
|
||||||
|
messages.extend(
|
||||||
|
build_harmony_preamble(
|
||||||
|
instructions=instructions,
|
||||||
|
tools=request.tools if with_custom_tools else None,
|
||||||
|
reasoning_effort=(
|
||||||
|
request.reasoning.effort if request.reasoning else None
|
||||||
|
),
|
||||||
|
with_custom_tools=with_custom_tools,
|
||||||
|
**descriptions,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
messages.extend(construct_harmony_previous_input_messages(request))
|
||||||
|
else:
|
||||||
|
messages.extend(previous_messages)
|
||||||
|
|
||||||
|
previous_outputs = list(previous_response_outputs or ())
|
||||||
|
try:
|
||||||
|
if isinstance(request_input, str):
|
||||||
|
if request_input or not request.previous_input_messages:
|
||||||
|
messages.append(get_user_message(request_input))
|
||||||
|
else:
|
||||||
|
for response_message in request_input:
|
||||||
|
new_message = response_input_to_harmony(
|
||||||
|
response_message,
|
||||||
|
previous_outputs,
|
||||||
|
)
|
||||||
|
if new_message is not None:
|
||||||
|
messages.append(new_message)
|
||||||
|
if isinstance(response_message, ResponseFunctionToolCall):
|
||||||
|
previous_outputs.append(response_message)
|
||||||
|
except ValueError as exc:
|
||||||
|
return self.create_error_response(
|
||||||
|
str(exc),
|
||||||
|
err_type="invalid_request_error",
|
||||||
|
param="input",
|
||||||
|
)
|
||||||
|
|
||||||
|
return ResponsesRenderResult(
|
||||||
|
messages=messages,
|
||||||
|
engine_input=self.render_responses_harmony_messages(
|
||||||
|
messages,
|
||||||
|
cache_salt=request.cache_salt,
|
||||||
|
tok_params=request.build_tok_params(self.model_config),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _get_harmony_builtin_tool_descriptions(
|
||||||
|
self,
|
||||||
|
tools: list[Tool],
|
||||||
|
tool_types: set[str],
|
||||||
|
tool_server: "ToolServer | None",
|
||||||
|
) -> dict[str, ToolNamespaceConfig | None]:
|
||||||
|
allowed_tools = _extract_allowed_tools_from_mcp_requests(tools)
|
||||||
|
descriptions: dict[str, ToolNamespaceConfig | None] = {}
|
||||||
|
for server_name, request_name in BUILTIN_TOOL_TO_MCP_SERVER_LABEL.items():
|
||||||
|
description = (
|
||||||
|
tool_server.get_tool_description(
|
||||||
|
server_name,
|
||||||
|
allowed_tools.get(request_name),
|
||||||
|
)
|
||||||
|
if request_name in tool_types
|
||||||
|
and tool_server is not None
|
||||||
|
and tool_server.has_tool(server_name)
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
descriptions[f"{server_name}_description"] = description
|
||||||
|
return descriptions
|
||||||
|
|
||||||
|
def render_responses_harmony_messages(
|
||||||
|
self,
|
||||||
|
messages: list[OpenAIMessage],
|
||||||
|
*,
|
||||||
|
cache_salt: str | None,
|
||||||
|
tok_params: TokenizeParams | None = None,
|
||||||
|
) -> EngineInput:
|
||||||
|
arrival_time = time.time()
|
||||||
|
prompt = TokensPrompt(prompt_token_ids=render_for_completion(messages))
|
||||||
|
if tok_params is not None:
|
||||||
|
tok_params.apply_post_tokenization(
|
||||||
|
self.renderer.tokenizer,
|
||||||
|
prompt,
|
||||||
|
)
|
||||||
|
engine_input = tokens_input(prompt["prompt_token_ids"], cache_salt=cache_salt)
|
||||||
|
engine_input["arrival_time"] = arrival_time
|
||||||
|
return engine_input
|
||||||
|
|
||||||
|
def _responses_render_result(
|
||||||
|
self,
|
||||||
|
messages: list[ChatCompletionMessageParam],
|
||||||
|
engine_inputs: list[EngineInput],
|
||||||
|
) -> ResponsesRenderResult | ErrorResponse:
|
||||||
|
if len(engine_inputs) != 1:
|
||||||
|
return self.create_error_response(
|
||||||
|
f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
|
||||||
|
)
|
||||||
|
return ResponsesRenderResult(
|
||||||
|
messages=messages,
|
||||||
|
engine_input=engine_inputs[0],
|
||||||
|
)
|
||||||
|
|
||||||
def _make_request_with_harmony(
|
def _make_request_with_harmony(
|
||||||
self,
|
self,
|
||||||
request: ChatCompletionRequest,
|
request: ChatCompletionRequest,
|
||||||
|
|||||||
Reference in New Issue
Block a user