[Frontend] Add stateless /v1/responses/render endpoint (#50195)
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
5e6f6a8ed4
commit
6a2a2bb02b
@@ -1,13 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any, TypeAlias
|
||||
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
|
||||
from openai.types.responses.tool import Mcp, Tool
|
||||
from openai_harmony import Message as OpenAIMessage
|
||||
from openai_harmony import ToolNamespaceConfig
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
)
|
||||
@@ -19,12 +25,24 @@ from vllm.entrypoints.openai.completion.protocol import (
|
||||
CompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
|
||||
build_harmony_preamble,
|
||||
extract_instructions_from_messages,
|
||||
get_user_message,
|
||||
has_custom_tools,
|
||||
parse_chat_inputs_to_harmony_messages,
|
||||
render_for_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.harmony import (
|
||||
construct_harmony_previous_input_messages,
|
||||
response_input_to_harmony,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.openai.responses.utils import (
|
||||
construct_input_messages,
|
||||
construct_tool_dicts,
|
||||
extract_tool_types,
|
||||
)
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
@@ -32,11 +50,12 @@ from vllm.inputs import (
|
||||
EngineInput,
|
||||
PromptType,
|
||||
SingletonPrompt,
|
||||
TokensPrompt,
|
||||
tokens_input,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers import BaseRenderer, ChatParams, merge_kwargs
|
||||
from vllm.renderers import BaseRenderer, ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
@@ -46,6 +65,39 @@ from vllm.utils.mistral import mt as _mt
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
|
||||
|
||||
ResponsesPreviousMessages: TypeAlias = (
|
||||
list[ChatCompletionMessageParam] | list[OpenAIMessage]
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResponsesRenderResult:
|
||||
messages: ResponsesPreviousMessages
|
||||
engine_input: EngineInput
|
||||
|
||||
|
||||
def _extract_allowed_tools_from_mcp_requests(
|
||||
tools: list[Tool],
|
||||
) -> dict[str, list[str] | None]:
|
||||
allowed_tools_map: dict[str, list[str] | None] = {}
|
||||
for tool in tools:
|
||||
if not isinstance(tool, Mcp):
|
||||
continue
|
||||
|
||||
allowed_tools = None
|
||||
if isinstance(tool.allowed_tools, list):
|
||||
allowed_tools = tool.allowed_tools
|
||||
elif tool.allowed_tools is not None:
|
||||
allowed_tools = tool.allowed_tools.tool_names
|
||||
if allowed_tools is not None and "*" in allowed_tools:
|
||||
allowed_tools = None
|
||||
allowed_tools_map[tool.server_label] = allowed_tools
|
||||
return allowed_tools_map
|
||||
|
||||
|
||||
def _reused_prompt_token_ids(request: Any) -> list[int] | None:
|
||||
"""Pop prompt token ids forwarded for decode-side reuse, if any.
|
||||
@@ -217,6 +269,219 @@ class OnlineRenderer:
|
||||
|
||||
return conversation, engine_inputs
|
||||
|
||||
async def render_responses(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
*,
|
||||
previous_messages: ResponsesPreviousMessages | None = None,
|
||||
previous_response_outputs: list[ResponseOutputItem] | None = None,
|
||||
tool_server: "ToolServer | None" = None,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> ResponsesRenderResult | ErrorResponse:
|
||||
"""Render a Responses request using only explicitly supplied history."""
|
||||
template_error = self.validate_chat_template(
|
||||
request_chat_template=None,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
if template_error is not None:
|
||||
return template_error
|
||||
|
||||
if self.use_harmony:
|
||||
return self._render_responses_with_harmony(
|
||||
request,
|
||||
previous_messages=previous_messages,
|
||||
previous_response_outputs=previous_response_outputs,
|
||||
tool_server=tool_server,
|
||||
)
|
||||
|
||||
if previous_messages is not None and any(
|
||||
isinstance(message, OpenAIMessage) for message in previous_messages
|
||||
):
|
||||
return self.create_error_response(
|
||||
"Non-Harmony Responses history must use chat messages.",
|
||||
err_type="invalid_request_error",
|
||||
param="previous_response_id",
|
||||
)
|
||||
|
||||
tool_dicts = construct_tool_dicts(
|
||||
request.tools,
|
||||
request.tool_choice,
|
||||
exclude_tools_when_tool_choice_none=(
|
||||
self.exclude_tools_when_tool_choice_none
|
||||
),
|
||||
)
|
||||
messages = construct_input_messages(
|
||||
request_instructions=request.instructions,
|
||||
request_input=request.input,
|
||||
prev_msg=list(previous_messages) if previous_messages is not None else None,
|
||||
prev_response_output=(
|
||||
list(previous_response_outputs)
|
||||
if previous_response_outputs is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
chat_template_kwargs = (
|
||||
request.build_chat_params(
|
||||
self.chat_template,
|
||||
self.chat_template_content_format,
|
||||
)
|
||||
.with_defaults(self.default_chat_template_kwargs)
|
||||
.chat_template_kwargs
|
||||
)
|
||||
_, engine_inputs = await self.preprocess_chat(
|
||||
request,
|
||||
messages,
|
||||
default_template=self.chat_template,
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
parser=self.parser,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
return self._responses_render_result(messages, engine_inputs)
|
||||
|
||||
def _render_responses_with_harmony(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
*,
|
||||
previous_messages: ResponsesPreviousMessages | None,
|
||||
previous_response_outputs: list[ResponseOutputItem] | None,
|
||||
tool_server: "ToolServer | None",
|
||||
) -> ResponsesRenderResult | ErrorResponse:
|
||||
if self.parser is not None:
|
||||
# HarmonyParser doesn't need chat_template_kwargs
|
||||
# TODO: Unify adjust_request() call with non-harmony branch
|
||||
self.parser(
|
||||
self.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
).adjust_request(request=request)
|
||||
|
||||
if previous_messages is not None and any(
|
||||
not isinstance(message, OpenAIMessage) for message in previous_messages
|
||||
):
|
||||
return self.create_error_response(
|
||||
"Harmony Responses history must use Harmony messages.",
|
||||
err_type="invalid_request_error",
|
||||
param="previous_response_id",
|
||||
)
|
||||
|
||||
messages: list[OpenAIMessage] = []
|
||||
request_input = request.input
|
||||
if previous_messages is None:
|
||||
tool_types = extract_tool_types(request.tools)
|
||||
with_custom_tools = has_custom_tools(tool_types)
|
||||
instructions = request.instructions
|
||||
if instructions is None and isinstance(request_input, list):
|
||||
instructions, request_input = extract_instructions_from_messages(
|
||||
request_input
|
||||
)
|
||||
descriptions = self._get_harmony_builtin_tool_descriptions(
|
||||
request.tools,
|
||||
tool_types,
|
||||
tool_server,
|
||||
)
|
||||
messages.extend(
|
||||
build_harmony_preamble(
|
||||
instructions=instructions,
|
||||
tools=request.tools if with_custom_tools else None,
|
||||
reasoning_effort=(
|
||||
request.reasoning.effort if request.reasoning else None
|
||||
),
|
||||
with_custom_tools=with_custom_tools,
|
||||
**descriptions,
|
||||
)
|
||||
)
|
||||
messages.extend(construct_harmony_previous_input_messages(request))
|
||||
else:
|
||||
messages.extend(previous_messages)
|
||||
|
||||
previous_outputs = list(previous_response_outputs or ())
|
||||
try:
|
||||
if isinstance(request_input, str):
|
||||
if request_input or not request.previous_input_messages:
|
||||
messages.append(get_user_message(request_input))
|
||||
else:
|
||||
for response_message in request_input:
|
||||
new_message = response_input_to_harmony(
|
||||
response_message,
|
||||
previous_outputs,
|
||||
)
|
||||
if new_message is not None:
|
||||
messages.append(new_message)
|
||||
if isinstance(response_message, ResponseFunctionToolCall):
|
||||
previous_outputs.append(response_message)
|
||||
except ValueError as exc:
|
||||
return self.create_error_response(
|
||||
str(exc),
|
||||
err_type="invalid_request_error",
|
||||
param="input",
|
||||
)
|
||||
|
||||
return ResponsesRenderResult(
|
||||
messages=messages,
|
||||
engine_input=self.render_responses_harmony_messages(
|
||||
messages,
|
||||
cache_salt=request.cache_salt,
|
||||
tok_params=request.build_tok_params(self.model_config),
|
||||
),
|
||||
)
|
||||
|
||||
def _get_harmony_builtin_tool_descriptions(
|
||||
self,
|
||||
tools: list[Tool],
|
||||
tool_types: set[str],
|
||||
tool_server: "ToolServer | None",
|
||||
) -> dict[str, ToolNamespaceConfig | None]:
|
||||
allowed_tools = _extract_allowed_tools_from_mcp_requests(tools)
|
||||
descriptions: dict[str, ToolNamespaceConfig | None] = {}
|
||||
for server_name, request_name in BUILTIN_TOOL_TO_MCP_SERVER_LABEL.items():
|
||||
description = (
|
||||
tool_server.get_tool_description(
|
||||
server_name,
|
||||
allowed_tools.get(request_name),
|
||||
)
|
||||
if request_name in tool_types
|
||||
and tool_server is not None
|
||||
and tool_server.has_tool(server_name)
|
||||
else None
|
||||
)
|
||||
descriptions[f"{server_name}_description"] = description
|
||||
return descriptions
|
||||
|
||||
def render_responses_harmony_messages(
|
||||
self,
|
||||
messages: list[OpenAIMessage],
|
||||
*,
|
||||
cache_salt: str | None,
|
||||
tok_params: TokenizeParams | None = None,
|
||||
) -> EngineInput:
|
||||
arrival_time = time.time()
|
||||
prompt = TokensPrompt(prompt_token_ids=render_for_completion(messages))
|
||||
if tok_params is not None:
|
||||
tok_params.apply_post_tokenization(
|
||||
self.renderer.tokenizer,
|
||||
prompt,
|
||||
)
|
||||
engine_input = tokens_input(prompt["prompt_token_ids"], cache_salt=cache_salt)
|
||||
engine_input["arrival_time"] = arrival_time
|
||||
return engine_input
|
||||
|
||||
def _responses_render_result(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
engine_inputs: list[EngineInput],
|
||||
) -> ResponsesRenderResult | ErrorResponse:
|
||||
if len(engine_inputs) != 1:
|
||||
return self.create_error_response(
|
||||
f"Expected exactly 1 engine prompt, got {len(engine_inputs)}"
|
||||
)
|
||||
return ResponsesRenderResult(
|
||||
messages=messages,
|
||||
engine_input=engine_inputs[0],
|
||||
)
|
||||
|
||||
def _make_request_with_harmony(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
|
||||
Reference in New Issue
Block a user