[Frontend] Add stateless /v1/responses/render endpoint (#50195)

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Francisco Javier Arceo
2026-09-07 12:50:42 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 5e6f6a8ed4
commit 6a2a2bb02b
19 changed files with 1228 additions and 298 deletions
+6 -33
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from fastapi import FastAPI
@@ -60,6 +60,7 @@ async def init_generate_state(
args: "Namespace",
request_logger: RequestLogger | None,
supported_tasks: tuple["SupportedTask", ...],
default_chat_template_kwargs: dict[str, Any],
):
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.chat_utils import load_chat_template
@@ -79,12 +80,6 @@ async def init_generate_state(
CohereServingChatV2 = None # type: ignore[assignment,misc]
else:
CohereServingChatV2 = None # type: ignore[assignment,misc]
from vllm.entrypoints.mcp.tool_server import (
DemoToolServer,
MCPToolServer,
ToolServer,
)
from vllm.entrypoints.openai.chat_completion.batch_serving import (
OpenAIServingChatBatch,
)
@@ -100,33 +95,11 @@ async def init_generate_state(
getattr(args, "fingerprint_value", None),
)
if args.tool_server == "demo":
tool_server: ToolServer | None = DemoToolServer()
assert isinstance(tool_server, DemoToolServer)
await tool_server.init_and_validate()
elif args.tool_server:
tool_server = MCPToolServer()
await tool_server.add_tool_server(args.tool_server)
else:
tool_server = None
resolved_chat_template = load_chat_template(args.chat_template)
# Fold the dedicated ``--cohere-format`` CLI flag into the renderer's
# default chat-template kwargs. The cohere renderer reads
# ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4
# rendering; making this a first-class flag keeps the right format
# discoverable for ``vllm serve --tokenizer-mode cohere`` users
# without forcing them to hand-construct a JSON dict for
# ``--default-chat-template-kwargs``. Per-request overrides still
# take precedence (see ``merge_kwargs`` in
# ``ChatCompletionRequest.build_chat_params``).
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
if getattr(args, "cohere_format", None):
default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
# Render endpoints are always backed by OnlineRenderer so that
# /v1/chat/completions/render and /v1/completions/render work on both
# generate-mode and render-only servers. Created in init_app_state.
# Render endpoints are always backed by OnlineRenderer so that chat,
# completion, and Responses rendering work on both generate-mode and
# render-only servers. Created in init_app_state.
state.openai_serving_responses = (
OpenAIServingResponses(
@@ -139,7 +112,7 @@ async def init_generate_state(
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
tool_server=tool_server,
tool_server=state.tool_server,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
@@ -8,6 +8,8 @@ from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
@@ -63,6 +65,10 @@ async def init_app_state(
state.vllm_config = vllm_config
state.args = args
resolved_chat_template = load_chat_template(args.chat_template)
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
state.tool_server = (
await init_tool_server(args) if "generate" in supported_tasks else None
)
# Merge default_mm_loras into the static lora_modules
default_mm_loras = (
@@ -90,7 +96,7 @@ async def init_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.online_renderer.warmup()
@@ -106,7 +112,7 @@ async def init_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
@@ -116,7 +122,7 @@ async def init_app_state(
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
@@ -124,7 +130,12 @@ async def init_app_state(
from vllm.entrypoints.generate.api_router import init_generate_state
await init_generate_state(
engine_client, state, args, request_logger, supported_tasks
engine_client,
state,
args,
request_logger,
supported_tasks,
default_chat_template_kwargs,
)
from vllm.entrypoints.scale_out.factories import init_scale_out_state
+10
View File
@@ -370,6 +370,16 @@ class FrontendArgs(BaseFrontendArgs):
return frontend_kwargs
def resolve_default_chat_template_kwargs(
args: argparse.Namespace,
) -> dict[str, Any]:
"""Resolve renderer defaults, including the dedicated Cohere format flag."""
kwargs = dict(args.default_chat_template_kwargs or {})
if getattr(args, "cohere_format", None):
kwargs.setdefault("cohere_format", args.cohere_format)
return kwargs
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
"""Create the CLI argument parser used by the OpenAI API server.
@@ -6,6 +6,8 @@ from starlette.datastructures import State
from vllm.config import VllmConfig
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
from vllm.entrypoints.scale_out.factories import init_render_state
@@ -46,6 +48,8 @@ async def init_render_app_state(
renderer = renderer_from_config(vllm_config)
resolved_chat_template = load_chat_template(args.chat_template)
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
state.tool_server = await init_tool_server(args)
state.online_renderer = OnlineRenderer(
model_config=vllm_config.model_config,
@@ -58,7 +62,7 @@ async def init_render_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.online_renderer.warmup()
@@ -74,7 +78,7 @@ async def init_render_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
@@ -85,7 +89,7 @@ async def init_render_app_state(
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
+15
View File
@@ -12,6 +12,8 @@ from vllm.logger import init_logger
logger = init_logger(__name__)
if TYPE_CHECKING:
from argparse import Namespace
from mcp.types import ListToolsResult
@@ -232,3 +234,16 @@ class DemoToolServer(ToolServer):
if tool_name not in self.tools:
raise KeyError(f"Tool '{tool_name}' is not supported")
yield self.tools[tool_name]
async def init_tool_server(args: "Namespace") -> ToolServer | None:
tool_server = getattr(args, "tool_server", None)
if tool_server == "demo":
demo_server = DemoToolServer()
await demo_server.init_and_validate()
return demo_server
if tool_server:
mcp_server = MCPToolServer()
await mcp_server.add_tool_server(tool_server)
return mcp_server
return None
+76 -252
View File
@@ -6,13 +6,11 @@ import time
from collections import deque
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, Sequence
from contextlib import AsyncExitStack
from copy import copy
from http import HTTPStatus
from typing import Any, Final
from typing import Any, Final, cast
from fastapi import Request
from openai.types.responses import (
ResponseFunctionToolCall,
ResponseOutputItem,
ResponseOutputMessage,
ResponseOutputText,
@@ -20,15 +18,12 @@ from openai.types.responses import (
response_text_delta_event,
)
from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
from openai.types.responses.tool import Mcp, Tool
from openai_harmony import Message as OpenAIHarmonyMessage
from pydantic import TypeAdapter
from vllm import envs
from vllm.config.utils import replace
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
)
from vllm.entrypoints.generate.base.protocol import (
@@ -38,24 +33,13 @@ from vllm.entrypoints.generate.base.protocol import (
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
from vllm.entrypoints.mcp.tool_server import ToolServer
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.openai.parser.harmony_utils import (
build_harmony_preamble,
extract_instructions_from_messages,
get_user_message,
has_custom_tools,
render_for_completion,
)
from vllm.entrypoints.openai.responses.context import (
ConversationContext,
HarmonyContext,
ParsableContext,
SimpleContext,
)
from vllm.entrypoints.openai.responses.harmony import (
construct_harmony_previous_input_messages,
harmony_to_response_output,
response_input_to_harmony,
)
from vllm.entrypoints.openai.responses.harmony import harmony_to_response_output
from vllm.entrypoints.openai.responses.protocol import (
InputTokensDetails,
OutputTokensDetails,
@@ -80,8 +64,6 @@ from vllm.entrypoints.openai.responses.streaming_events import (
)
from vllm.entrypoints.openai.responses.utils import (
build_response_output_items,
construct_input_messages,
construct_tool_dicts,
extract_function_tool_names,
extract_tool_types,
)
@@ -89,14 +71,19 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import GenerationError, VLLMValidationError
from vllm.inputs import EngineInput, tokens_input
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.logprobs import Logprob as SampleLogprob
from vllm.logprobs import SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.outputs import CompletionOutput
from vllm.parser import Parser, ParserManager
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.renderers import TokenizeParams
from vllm.renderers.online_renderer import (
OnlineRenderer,
ResponsesPreviousMessages,
ResponsesRenderResult,
)
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.utils import random_uuid
@@ -105,45 +92,6 @@ from vllm.utils.collection_utils import as_list
logger = init_logger(__name__)
def _extract_allowed_tools_from_mcp_requests(
tools: list[Tool],
) -> dict[str, list[str] | None]:
"""
Extract allowed_tools mapping from MCP tool requests.
Returns a dictionary mapping server_label to allowed_tools list.
Handles both list format and McpAllowedToolsMcpToolFilter object format.
Special handling:
- If allowed_tools is None, returns None (allows all tools)
- If allowed_tools contains "*", returns None (allows all tools)
- Otherwise, returns the list of specific tool names
This function can be reused for both harmony and non-harmony MCP calls.
"""
allowed_tools_map: dict[str, list[str] | None] = {}
for tool in tools:
if not isinstance(tool, Mcp):
continue
# allowed_tools can be a list or an object with tool_names
# Extract the actual list of tool names
allowed_tools_val = None
if tool.allowed_tools is not None:
if isinstance(tool.allowed_tools, list):
allowed_tools_val = tool.allowed_tools
elif hasattr(tool.allowed_tools, "tool_names"):
# It's an McpAllowedToolsMcpToolFilter object
allowed_tools_val = tool.allowed_tools.tool_names
# Normalize "*" to None (both mean "allow all tools")
if allowed_tools_val is not None and "*" in allowed_tools_val:
allowed_tools_val = None
allowed_tools_map[tool.server_label] = allowed_tools_val
return allowed_tools_map
class OpenAIServingResponses(GenerateBaseServing):
def __init__(
self,
@@ -226,7 +174,7 @@ class OpenAIServingResponses(GenerateBaseServing):
# HACK(woosuk): This is a hack. We should use a better store.
# FIXME: If enable_store=True, this may cause a memory leak since we
# never remove messages from the store.
self.msg_store: dict[str, list[ChatCompletionMessageParam]] = {}
self.msg_store: dict[str, ResponsesPreviousMessages] = {}
# HACK(wuhang): This is a hack. We should use a better store.
# FIXME: If enable_store=True, this may cause a memory leak since we
@@ -322,6 +270,33 @@ class OpenAIServingResponses(GenerateBaseServing):
)
return None
async def _render_resolved_response_inputs(
self,
request: ResponsesRequest,
prev_response: ResponsesResponse | None,
*,
skip_mm_cache: bool = False,
) -> ResponsesRenderResult | ErrorResponse:
previous_messages: ResponsesPreviousMessages | None = None
previous_response_outputs: list[ResponseOutputItem] | None = None
if prev_response is not None:
stored_messages = (
self.msg_store[prev_response.id]
if self.use_harmony
else self.msg_store.get(prev_response.id)
)
if stored_messages is not None:
previous_messages = stored_messages.copy()
previous_response_outputs = list(prev_response.output)
return await self.online_renderer.render_responses(
request,
previous_messages=previous_messages,
previous_response_outputs=previous_response_outputs,
tool_server=self.tool_server,
skip_mm_cache=skip_mm_cache,
)
async def create_responses(
self,
request: ResponsesRequest,
@@ -374,12 +349,14 @@ class OpenAIServingResponses(GenerateBaseServing):
lora_request = self._maybe_get_adapters(request)
model_name = self.models.model_name(lora_request)
if self.use_harmony:
messages, engine_inputs = self._make_request_with_harmony(
request, prev_response
)
else:
messages, engine_inputs = await self._make_request(request, prev_response)
render_result = await self._render_resolved_response_inputs(
request,
prev_response,
)
if isinstance(render_result, ErrorResponse):
return render_result
messages = render_result.messages
engine_inputs = [render_result.engine_input]
request_metadata = RequestResponseMetadata(request_id=request.request_id)
if raw_request:
@@ -508,6 +485,11 @@ class OpenAIServingResponses(GenerateBaseServing):
engine_input=engine_input,
sampling_params=sampling_params,
context=context,
tok_params=(
request.build_tok_params(self.model_config)
if isinstance(context, HarmonyContext)
else None
),
lora_request=lora_request,
priority=self._get_priority(request, raw_request),
trace_headers=trace_headers,
@@ -599,60 +581,28 @@ class OpenAIServingResponses(GenerateBaseServing):
request_metadata,
)
async def _make_request(
self,
request: ResponsesRequest,
prev_response: ResponsesResponse | None,
):
tool_dicts = construct_tool_dicts(
request.tools,
request.tool_choice,
exclude_tools_when_tool_choice_none=(
self.online_renderer.exclude_tools_when_tool_choice_none
),
)
# Construct the input messages.
messages = construct_input_messages(
request_instructions=request.instructions,
request_input=request.input,
prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,
prev_response_output=prev_response.output if prev_response else None,
)
chat_template_kwargs = self._effective_chat_template_kwargs(request)
_, engine_inputs = await self.online_renderer.preprocess_chat(
request,
messages,
default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=chat_template_kwargs,
tool_dicts=tool_dicts,
parser=self.parser,
)
return messages, engine_inputs
async def _render_next_turn(
self,
request: ResponsesRequest,
messages: list[ResponseInputOutputItem],
tool_dicts: list[dict[str, Any]] | None,
parser: type[Parser] | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
):
new_messages = construct_input_messages(
request_input=messages,
) -> list[EngineInput]:
render_request = request.model_copy(
update={
"input": list(messages),
"instructions": None,
"previous_input_messages": None,
"previous_response_id": None,
}
)
chat_template_kwargs = self._effective_chat_template_kwargs(request)
_, engine_inputs = await self.online_renderer.preprocess_chat(
request,
new_messages,
default_template=chat_template,
default_template_content_format=chat_template_content_format,
default_template_kwargs=chat_template_kwargs,
tool_dicts=tool_dicts,
parser=parser,
render_result = await self.online_renderer.render_responses(
render_request,
previous_messages=None,
previous_response_outputs=None,
tool_server=self.tool_server,
)
return engine_inputs
if isinstance(render_result, ErrorResponse):
raise VLLMValidationError(render_result.error.message)
return [render_result.engine_input]
async def _generate_with_builtin_tools(
self,
@@ -660,6 +610,7 @@ class OpenAIServingResponses(GenerateBaseServing):
engine_input: EngineInput,
sampling_params: SamplingParams,
context: ConversationContext,
tok_params: TokenizeParams | None = None,
lora_request: LoRARequest | None = None,
priority: int = 0,
trace_headers: Mapping[str, str] | None = None,
@@ -667,6 +618,7 @@ class OpenAIServingResponses(GenerateBaseServing):
reasoning_parser_kwargs: dict[str, Any] | None = None,
):
max_model_len = self.model_config.max_model_len
cache_salt = cast(str | None, engine_input.get("cache_salt"))
orig_priority = priority
sub_request = 0
@@ -711,18 +663,19 @@ class OpenAIServingResponses(GenerateBaseServing):
# Create inputs for the next turn.
# Render the next prompt token ids and update sampling_params.
if isinstance(context, HarmonyContext):
token_ids = context.render_for_completion()
engine_input = tokens_input(token_ids)
engine_input = self.online_renderer.render_responses_harmony_messages(
context.messages,
cache_salt=cache_salt,
tok_params=tok_params,
)
sampling_params.max_tokens = max_model_len - len(token_ids)
sampling_params.max_tokens = max_model_len - self._extract_prompt_len(
engine_input
)
elif isinstance(context, ParsableContext):
(engine_input,) = await self._render_next_turn(
context.request,
context.response_messages,
context.tool_dicts,
context.parser_cls,
context.chat_template,
context.chat_template_content_format,
)
sampling_params.max_tokens = get_max_tokens(
@@ -740,28 +693,6 @@ class OpenAIServingResponses(GenerateBaseServing):
priority = orig_priority - 1
sub_request += 1
def _make_request_with_harmony(
self,
request: ResponsesRequest,
prev_response: ResponsesResponse | None,
):
if self.parser is not None:
# HarmonyParser doesn't need chat_template_kwargs
# TODO: Unify adjust_request() call with non-harmony branch
self.parser(
self.renderer.get_tokenizer(),
request.tools,
model_config=self.model_config,
).adjust_request(request=request)
arrival_time = time.time()
messages = self._construct_input_messages_with_harmony(request, prev_response)
prompt_token_ids = render_for_completion(messages)
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
engine_input["arrival_time"] = arrival_time
return messages, [engine_input]
async def _initialize_tool_sessions(
self,
request: ResponsesRequest,
@@ -1104,113 +1035,6 @@ class OpenAIServingResponses(GenerateBaseServing):
)
]
def _get_harmony_builtin_tool_descriptions(
self, request: ResponsesRequest, tool_types: set[str]
) -> dict[str, str | None]:
# Extract allowed_tools from MCP tool requests
allowed_tools_map = _extract_allowed_tools_from_mcp_requests(request.tools)
# Get filtered tool descriptions first.
# If get_tool_description returns None (due to filtering), the tool is disabled.
browser_description = (
self.tool_server.get_tool_description(
"browser", allowed_tools_map.get("web_search_preview")
)
if "web_search_preview" in tool_types
and self.tool_server is not None
and self.tool_server.has_tool("browser")
else None
)
python_description = (
self.tool_server.get_tool_description(
"python", allowed_tools_map.get("code_interpreter")
)
if "code_interpreter" in tool_types
and self.tool_server is not None
and self.tool_server.has_tool("python")
else None
)
container_description = (
self.tool_server.get_tool_description(
"container", allowed_tools_map.get("container")
)
if "container" in tool_types
and self.tool_server is not None
and self.tool_server.has_tool("container")
else None
)
return {
"browser_description": browser_description,
"python_description": python_description,
"container_description": container_description,
}
def _construct_input_messages_with_harmony(
self,
request: ResponsesRequest,
prev_response: ResponsesResponse | None,
) -> list[OpenAIHarmonyMessage]:
messages: list[OpenAIHarmonyMessage] = []
request_input = request.input
if prev_response is None:
# New conversation.
tool_types = extract_tool_types(request.tools)
with_custom_tools = has_custom_tools(tool_types)
instructions = request.instructions
if instructions is None and isinstance(request_input, list):
instructions, request_input = extract_instructions_from_messages(
request_input
)
tool_descriptions = self._get_harmony_builtin_tool_descriptions(
request, tool_types
)
tools = request.tools if with_custom_tools else None
messages.extend(
build_harmony_preamble(
instructions=instructions,
tools=tools,
reasoning_effort=(
request.reasoning.effort if request.reasoning else None
),
with_custom_tools=with_custom_tools,
**tool_descriptions,
)
)
messages += construct_harmony_previous_input_messages(request)
else:
# Continue the previous conversation.
# FIXME(woosuk): Currently, request params like reasoning and
# instructions are ignored.
prev_msgs = self.msg_store[prev_response.id]
messages.extend(prev_msgs)
# Append the new input.
# Responses API supports simple text inputs without chat format.
if isinstance(request_input, str):
# Skip empty string input when previous_input_messages supplies
# the full conversation history --- an empty trailing user message
# confuses the model into thinking nothing was sent.
if request_input or not request.previous_input_messages:
messages.append(get_user_message(request_input))
else:
if prev_response is not None:
prev_outputs = copy(prev_response.output)
else:
prev_outputs = []
for response_msg in request_input:
new_msg = response_input_to_harmony(response_msg, prev_outputs)
if new_msg is not None:
messages.append(new_msg)
# User passes in a tool call request and its output. We need
# to add the tool call request to prev_outputs so that
# response_input_to_harmony can find the tool call request when
# parsing the tool call output.
if isinstance(response_msg, ResponseFunctionToolCall):
prev_outputs.append(response_msg)
return messages
async def _run_background_request_stream(
self,
request: ResponsesRequest,
+1
View File
@@ -31,6 +31,7 @@ def init_render_state(
state.openai_serving_models,
state.online_renderer,
request_logger=request_logger,
tool_server=state.tool_server,
)
state.serving_derender = ServingDerender(
@@ -8,6 +8,7 @@ from fastapi.responses import JSONResponse
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
@@ -95,3 +96,25 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=[item.model_dump() for item in result])
@router.post(
"/v1/responses/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def render_responses(request: ResponsesRequest, raw_request: Request):
handler = render(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Responses Render API")
result = await handler.render_responses_request(request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
@@ -8,6 +10,7 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
extract_mm_features,
)
@@ -31,6 +34,9 @@ from vllm.utils import random_uuid
logger = init_logger(__name__)
if TYPE_CHECKING:
from vllm.entrypoints.mcp.tool_server import ToolServer
class ServingRender(BaseServing):
def __init__(
@@ -39,6 +45,7 @@ class ServingRender(BaseServing):
online_renderer: "OnlineRenderer",
*,
request_logger: RequestLogger | None = None,
tool_server: "ToolServer | None" = None,
) -> None:
super().__init__(
models=models,
@@ -47,6 +54,7 @@ class ServingRender(BaseServing):
)
self.online_renderer = online_renderer
self.tool_server = tool_server
self._merge_inline_system = (
AnthropicServingMessages._detect_merge_inline_system(
@@ -229,6 +237,64 @@ class ServingRender(BaseServing):
return generate_requests
async def render_responses_request(
self,
request: ResponsesRequest,
) -> GenerateRequest | ErrorResponse:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if request.previous_response_id is not None:
return self.create_error_response(
message=(
"previous_response_id is not supported by the stateless "
"render endpoint."
),
err_type="invalid_request_error",
param="previous_response_id",
)
result = await self.online_renderer.render_responses(
request,
previous_messages=None,
previous_response_outputs=None,
tool_server=self.tool_server,
skip_mm_cache=True,
)
if isinstance(result, ErrorResponse):
return result
engine_input = result.engine_input
prompt_components = extract_prompt_components(self.model_config, engine_input)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_output_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
return GenerateRequest(
request_id=request.request_id,
token_ids=list(token_ids),
features=self._extract_mm_features(engine_input),
sampling_params=params,
model=request.model,
stream=bool(request.stream),
cache_salt=request.cache_salt,
priority=request.priority,
kv_transfer_params=request.kv_transfer_params,
ec_transfer_params=request.ec_transfer_params,
token_offsets=engine_input.get("prompt_token_offsets"),
)
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
"""Fields the EC consumer still requires after embeddings transfer."""
if self._placeholder_metadata_parser_failed: