[Frontend] Add stateless /v1/responses/render endpoint (#50195)

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Francisco Javier Arceo
2026-09-07 12:50:42 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 5e6f6a8ed4
commit 6a2a2bb02b
19 changed files with 1228 additions and 298 deletions
@@ -8,6 +8,8 @@ from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
@@ -63,6 +65,10 @@ async def init_app_state(
state.vllm_config = vllm_config
state.args = args
resolved_chat_template = load_chat_template(args.chat_template)
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
state.tool_server = (
await init_tool_server(args) if "generate" in supported_tasks else None
)
# Merge default_mm_loras into the static lora_modules
default_mm_loras = (
@@ -90,7 +96,7 @@ async def init_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.online_renderer.warmup()
@@ -106,7 +112,7 @@ async def init_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
@@ -116,7 +122,7 @@ async def init_app_state(
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
@@ -124,7 +130,12 @@ async def init_app_state(
from vllm.entrypoints.generate.api_router import init_generate_state
await init_generate_state(
engine_client, state, args, request_logger, supported_tasks
engine_client,
state,
args,
request_logger,
supported_tasks,
default_chat_template_kwargs,
)
from vllm.entrypoints.scale_out.factories import init_scale_out_state
+10
View File
@@ -370,6 +370,16 @@ class FrontendArgs(BaseFrontendArgs):
return frontend_kwargs
def resolve_default_chat_template_kwargs(
args: argparse.Namespace,
) -> dict[str, Any]:
"""Resolve renderer defaults, including the dedicated Cohere format flag."""
kwargs = dict(args.default_chat_template_kwargs or {})
if getattr(args, "cohere_format", None):
kwargs.setdefault("cohere_format", args.cohere_format)
return kwargs
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
"""Create the CLI argument parser used by the OpenAI API server.
@@ -6,6 +6,8 @@ from starlette.datastructures import State
from vllm.config import VllmConfig
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
from vllm.entrypoints.mcp.tool_server import init_tool_server
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
from vllm.entrypoints.scale_out.factories import init_render_state
@@ -46,6 +48,8 @@ async def init_render_app_state(
renderer = renderer_from_config(vllm_config)
resolved_chat_template = load_chat_template(args.chat_template)
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
state.tool_server = await init_tool_server(args)
state.online_renderer = OnlineRenderer(
model_config=vllm_config.model_config,
@@ -58,7 +62,7 @@ async def init_render_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.online_renderer.warmup()
@@ -74,7 +78,7 @@ async def init_render_app_state(
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
@@ -85,7 +89,7 @@ async def init_render_app_state(
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)