[Frontend] Add stateless /v1/responses/render endpoint (#50195)
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
5e6f6a8ed4
commit
6a2a2bb02b
@@ -8,6 +8,8 @@ from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
|
||||
from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
@@ -63,6 +65,10 @@ async def init_app_state(
|
||||
state.vllm_config = vllm_config
|
||||
state.args = args
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||
state.tool_server = (
|
||||
await init_tool_server(args) if "generate" in supported_tasks else None
|
||||
)
|
||||
|
||||
# Merge default_mm_loras into the static lora_modules
|
||||
default_mm_loras = (
|
||||
@@ -90,7 +96,7 @@ async def init_app_state(
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
@@ -106,7 +112,7 @@ async def init_app_state(
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
@@ -116,7 +122,7 @@ async def init_app_state(
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
@@ -124,7 +130,12 @@ async def init_app_state(
|
||||
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||
|
||||
await init_generate_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
engine_client,
|
||||
state,
|
||||
args,
|
||||
request_logger,
|
||||
supported_tasks,
|
||||
default_chat_template_kwargs,
|
||||
)
|
||||
|
||||
from vllm.entrypoints.scale_out.factories import init_scale_out_state
|
||||
|
||||
@@ -370,6 +370,16 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
return frontend_kwargs
|
||||
|
||||
|
||||
def resolve_default_chat_template_kwargs(
|
||||
args: argparse.Namespace,
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve renderer defaults, including the dedicated Cohere format flag."""
|
||||
kwargs = dict(args.default_chat_template_kwargs or {})
|
||||
if getattr(args, "cohere_format", None):
|
||||
kwargs.setdefault("cohere_format", args.cohere_format)
|
||||
return kwargs
|
||||
|
||||
|
||||
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
"""Create the CLI argument parser used by the OpenAI API server.
|
||||
|
||||
|
||||
@@ -6,6 +6,8 @@ from starlette.datastructures import State
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
|
||||
from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.entrypoints.scale_out.factories import init_render_state
|
||||
@@ -46,6 +48,8 @@ async def init_render_app_state(
|
||||
|
||||
renderer = renderer_from_config(vllm_config)
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||
state.tool_server = await init_tool_server(args)
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
@@ -58,7 +62,7 @@ async def init_render_app_state(
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
@@ -74,7 +78,7 @@ async def init_render_app_state(
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
@@ -85,7 +89,7 @@ async def init_render_app_state(
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user