[Frontend] Add stateless /v1/responses/render endpoint (#50195)

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Francisco Javier Arceo
2026-09-07 12:50:42 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 5e6f6a8ed4
commit 6a2a2bb02b
19 changed files with 1228 additions and 298 deletions
+6 -33
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from fastapi import FastAPI
@@ -60,6 +60,7 @@ async def init_generate_state(
args: "Namespace",
request_logger: RequestLogger | None,
supported_tasks: tuple["SupportedTask", ...],
default_chat_template_kwargs: dict[str, Any],
):
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.chat_utils import load_chat_template
@@ -79,12 +80,6 @@ async def init_generate_state(
CohereServingChatV2 = None # type: ignore[assignment,misc]
else:
CohereServingChatV2 = None # type: ignore[assignment,misc]
from vllm.entrypoints.mcp.tool_server import (
DemoToolServer,
MCPToolServer,
ToolServer,
)
from vllm.entrypoints.openai.chat_completion.batch_serving import (
OpenAIServingChatBatch,
)
@@ -100,33 +95,11 @@ async def init_generate_state(
getattr(args, "fingerprint_value", None),
)
if args.tool_server == "demo":
tool_server: ToolServer | None = DemoToolServer()
assert isinstance(tool_server, DemoToolServer)
await tool_server.init_and_validate()
elif args.tool_server:
tool_server = MCPToolServer()
await tool_server.add_tool_server(args.tool_server)
else:
tool_server = None
resolved_chat_template = load_chat_template(args.chat_template)
# Fold the dedicated ``--cohere-format`` CLI flag into the renderer's
# default chat-template kwargs. The cohere renderer reads
# ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4
# rendering; making this a first-class flag keeps the right format
# discoverable for ``vllm serve --tokenizer-mode cohere`` users
# without forcing them to hand-construct a JSON dict for
# ``--default-chat-template-kwargs``. Per-request overrides still
# take precedence (see ``merge_kwargs`` in
# ``ChatCompletionRequest.build_chat_params``).
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
if getattr(args, "cohere_format", None):
default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
# Render endpoints are always backed by OnlineRenderer so that
# /v1/chat/completions/render and /v1/completions/render work on both
# generate-mode and render-only servers. Created in init_app_state.
# Render endpoints are always backed by OnlineRenderer so that chat,
# completion, and Responses rendering work on both generate-mode and
# render-only servers. Created in init_app_state.
state.openai_serving_responses = (
OpenAIServingResponses(
@@ -139,7 +112,7 @@ async def init_generate_state(
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
tool_server=tool_server,
tool_server=state.tool_server,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,