[Frontend] Add stateless /v1/responses/render endpoint (#50195)

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Francisco Javier Arceo
2026-09-07 12:50:42 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 5e6f6a8ed4
commit 6a2a2bb02b
19 changed files with 1228 additions and 298 deletions
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
@@ -8,6 +10,7 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
extract_mm_features,
)
@@ -31,6 +34,9 @@ from vllm.utils import random_uuid
logger = init_logger(__name__)
if TYPE_CHECKING:
from vllm.entrypoints.mcp.tool_server import ToolServer
class ServingRender(BaseServing):
def __init__(
@@ -39,6 +45,7 @@ class ServingRender(BaseServing):
online_renderer: "OnlineRenderer",
*,
request_logger: RequestLogger | None = None,
tool_server: "ToolServer | None" = None,
) -> None:
super().__init__(
models=models,
@@ -47,6 +54,7 @@ class ServingRender(BaseServing):
)
self.online_renderer = online_renderer
self.tool_server = tool_server
self._merge_inline_system = (
AnthropicServingMessages._detect_merge_inline_system(
@@ -229,6 +237,64 @@ class ServingRender(BaseServing):
return generate_requests
async def render_responses_request(
self,
request: ResponsesRequest,
) -> GenerateRequest | ErrorResponse:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if request.previous_response_id is not None:
return self.create_error_response(
message=(
"previous_response_id is not supported by the stateless "
"render endpoint."
),
err_type="invalid_request_error",
param="previous_response_id",
)
result = await self.online_renderer.render_responses(
request,
previous_messages=None,
previous_response_outputs=None,
tool_server=self.tool_server,
skip_mm_cache=True,
)
if isinstance(result, ErrorResponse):
return result
engine_input = result.engine_input
prompt_components = extract_prompt_components(self.model_config, engine_input)
token_ids = prompt_components.token_ids
if not token_ids:
return self.create_error_response("No token_ids rendered")
input_length = extract_prompt_len(self.model_config, engine_input)
max_tokens = get_max_tokens(
self.model_config.max_model_len,
request.max_output_tokens,
input_length,
self.default_sampling_params,
self.override_max_tokens,
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
)
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
return GenerateRequest(
request_id=request.request_id,
token_ids=list(token_ids),
features=self._extract_mm_features(engine_input),
sampling_params=params,
model=request.model,
stream=bool(request.stream),
cache_salt=request.cache_salt,
priority=request.priority,
kv_transfer_params=request.kv_transfer_params,
ec_transfer_params=request.ec_transfer_params,
token_offsets=engine_input.get("prompt_token_offsets"),
)
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
"""Fields the EC consumer still requires after embeddings transfer."""
if self._placeholder_metadata_parser_failed: