[Frontend] Add stateless /v1/responses/render endpoint (#50195)
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
5e6f6a8ed4
commit
6a2a2bb02b
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
@@ -60,6 +60,7 @@ async def init_generate_state(
|
||||
args: "Namespace",
|
||||
request_logger: RequestLogger | None,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
):
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
@@ -79,12 +80,6 @@ async def init_generate_state(
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
else:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
|
||||
from vllm.entrypoints.mcp.tool_server import (
|
||||
DemoToolServer,
|
||||
MCPToolServer,
|
||||
ToolServer,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
OpenAIServingChatBatch,
|
||||
)
|
||||
@@ -100,33 +95,11 @@ async def init_generate_state(
|
||||
getattr(args, "fingerprint_value", None),
|
||||
)
|
||||
|
||||
if args.tool_server == "demo":
|
||||
tool_server: ToolServer | None = DemoToolServer()
|
||||
assert isinstance(tool_server, DemoToolServer)
|
||||
await tool_server.init_and_validate()
|
||||
elif args.tool_server:
|
||||
tool_server = MCPToolServer()
|
||||
await tool_server.add_tool_server(args.tool_server)
|
||||
else:
|
||||
tool_server = None
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
# Fold the dedicated ``--cohere-format`` CLI flag into the renderer's
|
||||
# default chat-template kwargs. The cohere renderer reads
|
||||
# ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4
|
||||
# rendering; making this a first-class flag keeps the right format
|
||||
# discoverable for ``vllm serve --tokenizer-mode cohere`` users
|
||||
# without forcing them to hand-construct a JSON dict for
|
||||
# ``--default-chat-template-kwargs``. Per-request overrides still
|
||||
# take precedence (see ``merge_kwargs`` in
|
||||
# ``ChatCompletionRequest.build_chat_params``).
|
||||
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
|
||||
if getattr(args, "cohere_format", None):
|
||||
default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
|
||||
|
||||
# Render endpoints are always backed by OnlineRenderer so that
|
||||
# /v1/chat/completions/render and /v1/completions/render work on both
|
||||
# generate-mode and render-only servers. Created in init_app_state.
|
||||
# Render endpoints are always backed by OnlineRenderer so that chat,
|
||||
# completion, and Responses rendering work on both generate-mode and
|
||||
# render-only servers. Created in init_app_state.
|
||||
|
||||
state.openai_serving_responses = (
|
||||
OpenAIServingResponses(
|
||||
@@ -139,7 +112,7 @@ async def init_generate_state(
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_server=tool_server,
|
||||
tool_server=state.tool_server,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
|
||||
@@ -8,6 +8,8 @@ from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
|
||||
from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
@@ -63,6 +65,10 @@ async def init_app_state(
|
||||
state.vllm_config = vllm_config
|
||||
state.args = args
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||
state.tool_server = (
|
||||
await init_tool_server(args) if "generate" in supported_tasks else None
|
||||
)
|
||||
|
||||
# Merge default_mm_loras into the static lora_modules
|
||||
default_mm_loras = (
|
||||
@@ -90,7 +96,7 @@ async def init_app_state(
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
@@ -106,7 +112,7 @@ async def init_app_state(
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
@@ -116,7 +122,7 @@ async def init_app_state(
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
@@ -124,7 +130,12 @@ async def init_app_state(
|
||||
from vllm.entrypoints.generate.api_router import init_generate_state
|
||||
|
||||
await init_generate_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
engine_client,
|
||||
state,
|
||||
args,
|
||||
request_logger,
|
||||
supported_tasks,
|
||||
default_chat_template_kwargs,
|
||||
)
|
||||
|
||||
from vllm.entrypoints.scale_out.factories import init_scale_out_state
|
||||
|
||||
@@ -370,6 +370,16 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
return frontend_kwargs
|
||||
|
||||
|
||||
def resolve_default_chat_template_kwargs(
|
||||
args: argparse.Namespace,
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve renderer defaults, including the dedicated Cohere format flag."""
|
||||
kwargs = dict(args.default_chat_template_kwargs or {})
|
||||
if getattr(args, "cohere_format", None):
|
||||
kwargs.setdefault("cohere_format", args.cohere_format)
|
||||
return kwargs
|
||||
|
||||
|
||||
def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
"""Create the CLI argument parser used by the OpenAI API server.
|
||||
|
||||
|
||||
@@ -6,6 +6,8 @@ from starlette.datastructures import State
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.launchers.cli_args import resolve_default_chat_template_kwargs
|
||||
from vllm.entrypoints.mcp.tool_server import init_tool_server
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
|
||||
from vllm.entrypoints.scale_out.factories import init_render_state
|
||||
@@ -46,6 +48,8 @@ async def init_render_app_state(
|
||||
|
||||
renderer = renderer_from_config(vllm_config)
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
default_chat_template_kwargs = resolve_default_chat_template_kwargs(args)
|
||||
state.tool_server = await init_tool_server(args)
|
||||
|
||||
state.online_renderer = OnlineRenderer(
|
||||
model_config=vllm_config.model_config,
|
||||
@@ -58,7 +62,7 @@ async def init_render_app_state(
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
state.online_renderer.warmup()
|
||||
@@ -74,7 +78,7 @@ async def init_render_app_state(
|
||||
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.reasoning_parser,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
log_error_stack=args.log_error_stack,
|
||||
)
|
||||
|
||||
@@ -85,7 +89,7 @@ async def init_render_app_state(
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@ from vllm.logger import init_logger
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
from mcp.types import ListToolsResult
|
||||
|
||||
|
||||
@@ -232,3 +234,16 @@ class DemoToolServer(ToolServer):
|
||||
if tool_name not in self.tools:
|
||||
raise KeyError(f"Tool '{tool_name}' is not supported")
|
||||
yield self.tools[tool_name]
|
||||
|
||||
|
||||
async def init_tool_server(args: "Namespace") -> ToolServer | None:
|
||||
tool_server = getattr(args, "tool_server", None)
|
||||
if tool_server == "demo":
|
||||
demo_server = DemoToolServer()
|
||||
await demo_server.init_and_validate()
|
||||
return demo_server
|
||||
if tool_server:
|
||||
mcp_server = MCPToolServer()
|
||||
await mcp_server.add_tool_server(tool_server)
|
||||
return mcp_server
|
||||
return None
|
||||
|
||||
@@ -6,13 +6,11 @@ import time
|
||||
from collections import deque
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping, Sequence
|
||||
from contextlib import AsyncExitStack
|
||||
from copy import copy
|
||||
from http import HTTPStatus
|
||||
from typing import Any, Final
|
||||
from typing import Any, Final, cast
|
||||
|
||||
from fastapi import Request
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
ResponseOutputItem,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
@@ -20,15 +18,12 @@ from openai.types.responses import (
|
||||
response_text_delta_event,
|
||||
)
|
||||
from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
|
||||
from openai.types.responses.tool import Mcp, Tool
|
||||
from openai_harmony import Message as OpenAIHarmonyMessage
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config.utils import replace
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
@@ -38,24 +33,13 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
from vllm.entrypoints.generate.base.serving import GenerateBaseServing
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
build_harmony_preamble,
|
||||
extract_instructions_from_messages,
|
||||
get_user_message,
|
||||
has_custom_tools,
|
||||
render_for_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.context import (
|
||||
ConversationContext,
|
||||
HarmonyContext,
|
||||
ParsableContext,
|
||||
SimpleContext,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.harmony import (
|
||||
construct_harmony_previous_input_messages,
|
||||
harmony_to_response_output,
|
||||
response_input_to_harmony,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.harmony import harmony_to_response_output
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
InputTokensDetails,
|
||||
OutputTokensDetails,
|
||||
@@ -80,8 +64,6 @@ from vllm.entrypoints.openai.responses.streaming_events import (
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.utils import (
|
||||
build_response_output_items,
|
||||
construct_input_messages,
|
||||
construct_tool_dicts,
|
||||
extract_function_tool_names,
|
||||
extract_tool_types,
|
||||
)
|
||||
@@ -89,14 +71,19 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import GenerationError, VLLMValidationError
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob as SampleLogprob
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import CompletionOutput
|
||||
from vllm.parser import Parser, ParserManager
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.online_renderer import (
|
||||
OnlineRenderer,
|
||||
ResponsesPreviousMessages,
|
||||
ResponsesRenderResult,
|
||||
)
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils import random_uuid
|
||||
@@ -105,45 +92,6 @@ from vllm.utils.collection_utils import as_list
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _extract_allowed_tools_from_mcp_requests(
|
||||
tools: list[Tool],
|
||||
) -> dict[str, list[str] | None]:
|
||||
"""
|
||||
Extract allowed_tools mapping from MCP tool requests.
|
||||
|
||||
Returns a dictionary mapping server_label to allowed_tools list.
|
||||
Handles both list format and McpAllowedToolsMcpToolFilter object format.
|
||||
|
||||
Special handling:
|
||||
- If allowed_tools is None, returns None (allows all tools)
|
||||
- If allowed_tools contains "*", returns None (allows all tools)
|
||||
- Otherwise, returns the list of specific tool names
|
||||
|
||||
This function can be reused for both harmony and non-harmony MCP calls.
|
||||
"""
|
||||
allowed_tools_map: dict[str, list[str] | None] = {}
|
||||
for tool in tools:
|
||||
if not isinstance(tool, Mcp):
|
||||
continue
|
||||
|
||||
# allowed_tools can be a list or an object with tool_names
|
||||
# Extract the actual list of tool names
|
||||
allowed_tools_val = None
|
||||
if tool.allowed_tools is not None:
|
||||
if isinstance(tool.allowed_tools, list):
|
||||
allowed_tools_val = tool.allowed_tools
|
||||
elif hasattr(tool.allowed_tools, "tool_names"):
|
||||
# It's an McpAllowedToolsMcpToolFilter object
|
||||
allowed_tools_val = tool.allowed_tools.tool_names
|
||||
|
||||
# Normalize "*" to None (both mean "allow all tools")
|
||||
if allowed_tools_val is not None and "*" in allowed_tools_val:
|
||||
allowed_tools_val = None
|
||||
|
||||
allowed_tools_map[tool.server_label] = allowed_tools_val
|
||||
return allowed_tools_map
|
||||
|
||||
|
||||
class OpenAIServingResponses(GenerateBaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -226,7 +174,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
# HACK(woosuk): This is a hack. We should use a better store.
|
||||
# FIXME: If enable_store=True, this may cause a memory leak since we
|
||||
# never remove messages from the store.
|
||||
self.msg_store: dict[str, list[ChatCompletionMessageParam]] = {}
|
||||
self.msg_store: dict[str, ResponsesPreviousMessages] = {}
|
||||
|
||||
# HACK(wuhang): This is a hack. We should use a better store.
|
||||
# FIXME: If enable_store=True, this may cause a memory leak since we
|
||||
@@ -322,6 +270,33 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
)
|
||||
return None
|
||||
|
||||
async def _render_resolved_response_inputs(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
prev_response: ResponsesResponse | None,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> ResponsesRenderResult | ErrorResponse:
|
||||
previous_messages: ResponsesPreviousMessages | None = None
|
||||
previous_response_outputs: list[ResponseOutputItem] | None = None
|
||||
if prev_response is not None:
|
||||
stored_messages = (
|
||||
self.msg_store[prev_response.id]
|
||||
if self.use_harmony
|
||||
else self.msg_store.get(prev_response.id)
|
||||
)
|
||||
if stored_messages is not None:
|
||||
previous_messages = stored_messages.copy()
|
||||
previous_response_outputs = list(prev_response.output)
|
||||
|
||||
return await self.online_renderer.render_responses(
|
||||
request,
|
||||
previous_messages=previous_messages,
|
||||
previous_response_outputs=previous_response_outputs,
|
||||
tool_server=self.tool_server,
|
||||
skip_mm_cache=skip_mm_cache,
|
||||
)
|
||||
|
||||
async def create_responses(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
@@ -374,12 +349,14 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
model_name = self.models.model_name(lora_request)
|
||||
|
||||
if self.use_harmony:
|
||||
messages, engine_inputs = self._make_request_with_harmony(
|
||||
request, prev_response
|
||||
)
|
||||
else:
|
||||
messages, engine_inputs = await self._make_request(request, prev_response)
|
||||
render_result = await self._render_resolved_response_inputs(
|
||||
request,
|
||||
prev_response,
|
||||
)
|
||||
if isinstance(render_result, ErrorResponse):
|
||||
return render_result
|
||||
messages = render_result.messages
|
||||
engine_inputs = [render_result.engine_input]
|
||||
|
||||
request_metadata = RequestResponseMetadata(request_id=request.request_id)
|
||||
if raw_request:
|
||||
@@ -508,6 +485,11 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
engine_input=engine_input,
|
||||
sampling_params=sampling_params,
|
||||
context=context,
|
||||
tok_params=(
|
||||
request.build_tok_params(self.model_config)
|
||||
if isinstance(context, HarmonyContext)
|
||||
else None
|
||||
),
|
||||
lora_request=lora_request,
|
||||
priority=self._get_priority(request, raw_request),
|
||||
trace_headers=trace_headers,
|
||||
@@ -599,60 +581,28 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
request_metadata,
|
||||
)
|
||||
|
||||
async def _make_request(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
prev_response: ResponsesResponse | None,
|
||||
):
|
||||
tool_dicts = construct_tool_dicts(
|
||||
request.tools,
|
||||
request.tool_choice,
|
||||
exclude_tools_when_tool_choice_none=(
|
||||
self.online_renderer.exclude_tools_when_tool_choice_none
|
||||
),
|
||||
)
|
||||
# Construct the input messages.
|
||||
messages = construct_input_messages(
|
||||
request_instructions=request.instructions,
|
||||
request_input=request.input,
|
||||
prev_msg=self.msg_store.get(prev_response.id) if prev_response else None,
|
||||
prev_response_output=prev_response.output if prev_response else None,
|
||||
)
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
||||
_, engine_inputs = await self.online_renderer.preprocess_chat(
|
||||
request,
|
||||
messages,
|
||||
default_template=self.chat_template,
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
parser=self.parser,
|
||||
)
|
||||
return messages, engine_inputs
|
||||
|
||||
async def _render_next_turn(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
messages: list[ResponseInputOutputItem],
|
||||
tool_dicts: list[dict[str, Any]] | None,
|
||||
parser: type[Parser] | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
):
|
||||
new_messages = construct_input_messages(
|
||||
request_input=messages,
|
||||
) -> list[EngineInput]:
|
||||
render_request = request.model_copy(
|
||||
update={
|
||||
"input": list(messages),
|
||||
"instructions": None,
|
||||
"previous_input_messages": None,
|
||||
"previous_response_id": None,
|
||||
}
|
||||
)
|
||||
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
||||
_, engine_inputs = await self.online_renderer.preprocess_chat(
|
||||
request,
|
||||
new_messages,
|
||||
default_template=chat_template,
|
||||
default_template_content_format=chat_template_content_format,
|
||||
default_template_kwargs=chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
parser=parser,
|
||||
render_result = await self.online_renderer.render_responses(
|
||||
render_request,
|
||||
previous_messages=None,
|
||||
previous_response_outputs=None,
|
||||
tool_server=self.tool_server,
|
||||
)
|
||||
return engine_inputs
|
||||
if isinstance(render_result, ErrorResponse):
|
||||
raise VLLMValidationError(render_result.error.message)
|
||||
return [render_result.engine_input]
|
||||
|
||||
async def _generate_with_builtin_tools(
|
||||
self,
|
||||
@@ -660,6 +610,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
engine_input: EngineInput,
|
||||
sampling_params: SamplingParams,
|
||||
context: ConversationContext,
|
||||
tok_params: TokenizeParams | None = None,
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
@@ -667,6 +618,7 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
reasoning_parser_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
max_model_len = self.model_config.max_model_len
|
||||
cache_salt = cast(str | None, engine_input.get("cache_salt"))
|
||||
|
||||
orig_priority = priority
|
||||
sub_request = 0
|
||||
@@ -711,18 +663,19 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
# Create inputs for the next turn.
|
||||
# Render the next prompt token ids and update sampling_params.
|
||||
if isinstance(context, HarmonyContext):
|
||||
token_ids = context.render_for_completion()
|
||||
engine_input = tokens_input(token_ids)
|
||||
engine_input = self.online_renderer.render_responses_harmony_messages(
|
||||
context.messages,
|
||||
cache_salt=cache_salt,
|
||||
tok_params=tok_params,
|
||||
)
|
||||
|
||||
sampling_params.max_tokens = max_model_len - len(token_ids)
|
||||
sampling_params.max_tokens = max_model_len - self._extract_prompt_len(
|
||||
engine_input
|
||||
)
|
||||
elif isinstance(context, ParsableContext):
|
||||
(engine_input,) = await self._render_next_turn(
|
||||
context.request,
|
||||
context.response_messages,
|
||||
context.tool_dicts,
|
||||
context.parser_cls,
|
||||
context.chat_template,
|
||||
context.chat_template_content_format,
|
||||
)
|
||||
|
||||
sampling_params.max_tokens = get_max_tokens(
|
||||
@@ -740,28 +693,6 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
priority = orig_priority - 1
|
||||
sub_request += 1
|
||||
|
||||
def _make_request_with_harmony(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
prev_response: ResponsesResponse | None,
|
||||
):
|
||||
if self.parser is not None:
|
||||
# HarmonyParser doesn't need chat_template_kwargs
|
||||
# TODO: Unify adjust_request() call with non-harmony branch
|
||||
self.parser(
|
||||
self.renderer.get_tokenizer(),
|
||||
request.tools,
|
||||
model_config=self.model_config,
|
||||
).adjust_request(request=request)
|
||||
|
||||
arrival_time = time.time()
|
||||
messages = self._construct_input_messages_with_harmony(request, prev_response)
|
||||
prompt_token_ids = render_for_completion(messages)
|
||||
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
|
||||
engine_input["arrival_time"] = arrival_time
|
||||
|
||||
return messages, [engine_input]
|
||||
|
||||
async def _initialize_tool_sessions(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
@@ -1104,113 +1035,6 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
)
|
||||
]
|
||||
|
||||
def _get_harmony_builtin_tool_descriptions(
|
||||
self, request: ResponsesRequest, tool_types: set[str]
|
||||
) -> dict[str, str | None]:
|
||||
# Extract allowed_tools from MCP tool requests
|
||||
allowed_tools_map = _extract_allowed_tools_from_mcp_requests(request.tools)
|
||||
|
||||
# Get filtered tool descriptions first.
|
||||
# If get_tool_description returns None (due to filtering), the tool is disabled.
|
||||
browser_description = (
|
||||
self.tool_server.get_tool_description(
|
||||
"browser", allowed_tools_map.get("web_search_preview")
|
||||
)
|
||||
if "web_search_preview" in tool_types
|
||||
and self.tool_server is not None
|
||||
and self.tool_server.has_tool("browser")
|
||||
else None
|
||||
)
|
||||
python_description = (
|
||||
self.tool_server.get_tool_description(
|
||||
"python", allowed_tools_map.get("code_interpreter")
|
||||
)
|
||||
if "code_interpreter" in tool_types
|
||||
and self.tool_server is not None
|
||||
and self.tool_server.has_tool("python")
|
||||
else None
|
||||
)
|
||||
container_description = (
|
||||
self.tool_server.get_tool_description(
|
||||
"container", allowed_tools_map.get("container")
|
||||
)
|
||||
if "container" in tool_types
|
||||
and self.tool_server is not None
|
||||
and self.tool_server.has_tool("container")
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"browser_description": browser_description,
|
||||
"python_description": python_description,
|
||||
"container_description": container_description,
|
||||
}
|
||||
|
||||
def _construct_input_messages_with_harmony(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
prev_response: ResponsesResponse | None,
|
||||
) -> list[OpenAIHarmonyMessage]:
|
||||
messages: list[OpenAIHarmonyMessage] = []
|
||||
request_input = request.input
|
||||
if prev_response is None:
|
||||
# New conversation.
|
||||
tool_types = extract_tool_types(request.tools)
|
||||
with_custom_tools = has_custom_tools(tool_types)
|
||||
instructions = request.instructions
|
||||
if instructions is None and isinstance(request_input, list):
|
||||
instructions, request_input = extract_instructions_from_messages(
|
||||
request_input
|
||||
)
|
||||
tool_descriptions = self._get_harmony_builtin_tool_descriptions(
|
||||
request, tool_types
|
||||
)
|
||||
tools = request.tools if with_custom_tools else None
|
||||
messages.extend(
|
||||
build_harmony_preamble(
|
||||
instructions=instructions,
|
||||
tools=tools,
|
||||
reasoning_effort=(
|
||||
request.reasoning.effort if request.reasoning else None
|
||||
),
|
||||
with_custom_tools=with_custom_tools,
|
||||
**tool_descriptions,
|
||||
)
|
||||
)
|
||||
messages += construct_harmony_previous_input_messages(request)
|
||||
|
||||
else:
|
||||
# Continue the previous conversation.
|
||||
# FIXME(woosuk): Currently, request params like reasoning and
|
||||
# instructions are ignored.
|
||||
prev_msgs = self.msg_store[prev_response.id]
|
||||
|
||||
messages.extend(prev_msgs)
|
||||
# Append the new input.
|
||||
# Responses API supports simple text inputs without chat format.
|
||||
if isinstance(request_input, str):
|
||||
# Skip empty string input when previous_input_messages supplies
|
||||
# the full conversation history --- an empty trailing user message
|
||||
# confuses the model into thinking nothing was sent.
|
||||
if request_input or not request.previous_input_messages:
|
||||
messages.append(get_user_message(request_input))
|
||||
else:
|
||||
if prev_response is not None:
|
||||
prev_outputs = copy(prev_response.output)
|
||||
else:
|
||||
prev_outputs = []
|
||||
for response_msg in request_input:
|
||||
new_msg = response_input_to_harmony(response_msg, prev_outputs)
|
||||
if new_msg is not None:
|
||||
messages.append(new_msg)
|
||||
|
||||
# User passes in a tool call request and its output. We need
|
||||
# to add the tool call request to prev_outputs so that
|
||||
# response_input_to_harmony can find the tool call request when
|
||||
# parsing the tool call output.
|
||||
if isinstance(response_msg, ResponseFunctionToolCall):
|
||||
prev_outputs.append(response_msg)
|
||||
return messages
|
||||
|
||||
async def _run_background_request_stream(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
|
||||
@@ -31,6 +31,7 @@ def init_render_state(
|
||||
state.openai_serving_models,
|
||||
state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
tool_server=state.tool_server,
|
||||
)
|
||||
|
||||
state.serving_derender = ServingDerender(
|
||||
|
||||
@@ -8,6 +8,7 @@ from fastapi.responses import JSONResponse
|
||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
@@ -95,3 +96,25 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=[item.model_dump() for item in result])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/responses/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def render_responses(request: ResponsesRequest, raw_request: Request):
|
||||
handler = render(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Responses Render API")
|
||||
|
||||
result = await handler.render_responses_request(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.entrypoints.anthropic.protocol import AnthropicMessagesRequest
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
@@ -8,6 +10,7 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.mm_features import (
|
||||
extract_mm_features,
|
||||
)
|
||||
@@ -31,6 +34,9 @@ from vllm.utils import random_uuid
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.mcp.tool_server import ToolServer
|
||||
|
||||
|
||||
class ServingRender(BaseServing):
|
||||
def __init__(
|
||||
@@ -39,6 +45,7 @@ class ServingRender(BaseServing):
|
||||
online_renderer: "OnlineRenderer",
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
tool_server: "ToolServer | None" = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
@@ -47,6 +54,7 @@ class ServingRender(BaseServing):
|
||||
)
|
||||
|
||||
self.online_renderer = online_renderer
|
||||
self.tool_server = tool_server
|
||||
|
||||
self._merge_inline_system = (
|
||||
AnthropicServingMessages._detect_merge_inline_system(
|
||||
@@ -229,6 +237,64 @@ class ServingRender(BaseServing):
|
||||
|
||||
return generate_requests
|
||||
|
||||
async def render_responses_request(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
) -> GenerateRequest | ErrorResponse:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
if request.previous_response_id is not None:
|
||||
return self.create_error_response(
|
||||
message=(
|
||||
"previous_response_id is not supported by the stateless "
|
||||
"render endpoint."
|
||||
),
|
||||
err_type="invalid_request_error",
|
||||
param="previous_response_id",
|
||||
)
|
||||
|
||||
result = await self.online_renderer.render_responses(
|
||||
request,
|
||||
previous_messages=None,
|
||||
previous_response_outputs=None,
|
||||
tool_server=self.tool_server,
|
||||
skip_mm_cache=True,
|
||||
)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
engine_input = result.engine_input
|
||||
prompt_components = extract_prompt_components(self.model_config, engine_input)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_input)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_output_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
truncate_prompt_tokens=(-1 if request.truncation != "disabled" else None),
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request.request_id,
|
||||
token_ids=list(token_ids),
|
||||
features=self._extract_mm_features(engine_input),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
stream=bool(request.stream),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
kv_transfer_params=request.kv_transfer_params,
|
||||
ec_transfer_params=request.ec_transfer_params,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
||||
"""Fields the EC consumer still requires after embeddings transfer."""
|
||||
if self._placeholder_metadata_parser_failed:
|
||||
|
||||
Reference in New Issue
Block a user