[Frontend] Add stateless /v1/responses/render endpoint (#50195)
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
5e6f6a8ed4
commit
6a2a2bb02b
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
@@ -60,6 +60,7 @@ async def init_generate_state(
|
||||
args: "Namespace",
|
||||
request_logger: RequestLogger | None,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
):
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
@@ -79,12 +80,6 @@ async def init_generate_state(
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
else:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
|
||||
from vllm.entrypoints.mcp.tool_server import (
|
||||
DemoToolServer,
|
||||
MCPToolServer,
|
||||
ToolServer,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
OpenAIServingChatBatch,
|
||||
)
|
||||
@@ -100,33 +95,11 @@ async def init_generate_state(
|
||||
getattr(args, "fingerprint_value", None),
|
||||
)
|
||||
|
||||
if args.tool_server == "demo":
|
||||
tool_server: ToolServer | None = DemoToolServer()
|
||||
assert isinstance(tool_server, DemoToolServer)
|
||||
await tool_server.init_and_validate()
|
||||
elif args.tool_server:
|
||||
tool_server = MCPToolServer()
|
||||
await tool_server.add_tool_server(args.tool_server)
|
||||
else:
|
||||
tool_server = None
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
# Fold the dedicated ``--cohere-format`` CLI flag into the renderer's
|
||||
# default chat-template kwargs. The cohere renderer reads
|
||||
# ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4
|
||||
# rendering; making this a first-class flag keeps the right format
|
||||
# discoverable for ``vllm serve --tokenizer-mode cohere`` users
|
||||
# without forcing them to hand-construct a JSON dict for
|
||||
# ``--default-chat-template-kwargs``. Per-request overrides still
|
||||
# take precedence (see ``merge_kwargs`` in
|
||||
# ``ChatCompletionRequest.build_chat_params``).
|
||||
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
|
||||
if getattr(args, "cohere_format", None):
|
||||
default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
|
||||
|
||||
# Render endpoints are always backed by OnlineRenderer so that
|
||||
# /v1/chat/completions/render and /v1/completions/render work on both
|
||||
# generate-mode and render-only servers. Created in init_app_state.
|
||||
# Render endpoints are always backed by OnlineRenderer so that chat,
|
||||
# completion, and Responses rendering work on both generate-mode and
|
||||
# render-only servers. Created in init_app_state.
|
||||
|
||||
state.openai_serving_responses = (
|
||||
OpenAIServingResponses(
|
||||
@@ -139,7 +112,7 @@ async def init_generate_state(
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_server=tool_server,
|
||||
tool_server=state.tool_server,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
|
||||
Reference in New Issue
Block a user