[Frontend] Split ServingRender into renderer and entrypoint. (#44285)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-06-23 11:19:09 +00:00
committed by GitHub
parent d32575a2d2
commit 31ca9504b1
26 changed files with 1155 additions and 939 deletions
+4 -6
View File
@@ -10,7 +10,7 @@ import logging
import time
import uuid
from collections.abc import AsyncGenerator
from typing import TYPE_CHECKING, Any
from typing import Any
import jinja2
from fastapi import Request
@@ -48,9 +48,7 @@ from vllm.entrypoints.openai.engine.protocol import (
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.api_utils import sanitize_message
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
if TYPE_CHECKING:
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.renderers.online_renderer import OnlineRenderer
logger = logging.getLogger(__name__)
@@ -111,7 +109,7 @@ class AnthropicServingMessages(OpenAIServingChat):
models: OpenAIServingModels,
response_role: str,
*,
openai_serving_render: "OpenAIServingRender",
online_renderer: "OnlineRenderer",
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
@@ -127,7 +125,7 @@ class AnthropicServingMessages(OpenAIServingChat):
engine_client=engine_client,
models=models,
response_role=response_role,
openai_serving_render=openai_serving_render,
online_renderer=online_renderer,
request_logger=request_logger,
chat_template=chat_template,
chat_template_content_format=chat_template_content_format,
+6 -6
View File
@@ -87,7 +87,7 @@ async def init_generate_state(
tool_server = None
resolved_chat_template = load_chat_template(args.chat_template)
# Render endpoints are always backed by OpenAIServingRender so that
# Render endpoints are always backed by OnlineRenderer so that
# /v1/chat/completions/render and /v1/completions/render work on both
# generate-mode and render-only servers. Created in init_app_state.
@@ -95,7 +95,7 @@ async def init_generate_state(
OpenAIServingResponses(
engine_client,
state.openai_serving_models,
state.openai_serving_render,
state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
@@ -116,7 +116,7 @@ async def init_generate_state(
engine_client=engine_client,
models=state.openai_serving_models,
response_role=args.response_role,
openai_serving_render=state.openai_serving_render,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
@@ -146,7 +146,7 @@ async def init_generate_state(
OpenAIServingCompletion(
engine_client,
state.openai_serving_models,
openai_serving_render=state.openai_serving_render,
online_renderer=state.online_renderer,
request_logger=request_logger,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
@@ -160,7 +160,7 @@ async def init_generate_state(
engine_client,
state.openai_serving_models,
args.response_role,
openai_serving_render=state.openai_serving_render,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
@@ -179,7 +179,7 @@ async def init_generate_state(
ServingTokens(
engine_client,
state.openai_serving_models,
state.openai_serving_render,
state.online_renderer,
request_logger=request_logger,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
+50 -8
View File
@@ -32,7 +32,7 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import (
@@ -55,6 +55,8 @@ from vllm.entrypoints.serve.utils.server_utils import (
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParserManager
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tasks import POOLING_TASKS, SupportedTask
from vllm.tool_parsers import ToolParserManager
from vllm.tracing import instrument
@@ -360,10 +362,24 @@ async def init_app_state(
)
await state.openai_serving_models.init_static_loras()
state.openai_serving_render = OpenAIServingRender(
state.online_renderer = OnlineRenderer(
model_config=engine_client.model_config,
renderer=engine_client.renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.online_derenderer = OnlineDerenderer(
model_config=engine_client.model_config,
renderer=engine_client.renderer,
model_registry=state.openai_serving_models.registry,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
@@ -378,13 +394,19 @@ async def init_app_state(
state.serving_tokenization = ServingTokenization(
state.openai_serving_models,
state.openai_serving_render,
state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
state.serving_render = ServingRender(
state.openai_serving_models,
state.online_renderer,
state.online_derenderer,
request_logger=request_logger,
)
if "generate" in supported_tasks:
from vllm.entrypoints.generate.api_router import init_generate_state
@@ -423,8 +445,8 @@ async def init_render_app_state(
"""
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
served_model_names = args.served_model_name or [args.model]
model_registry = OpenAIModelRegistry(
@@ -443,10 +465,24 @@ async def init_render_app_state(
renderer = renderer_from_config(vllm_config)
resolved_chat_template = load_chat_template(args.chat_template)
state.openai_serving_render = OpenAIServingRender(
state.online_renderer = OnlineRenderer(
model_config=vllm_config.model_config,
renderer=renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
enable_auto_tools=args.enable_auto_tool_choice,
exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
tool_parser=args.tool_call_parser,
reasoning_parser=args.reasoning_parser,
default_chat_template_kwargs=args.default_chat_template_kwargs,
log_error_stack=args.log_error_stack,
)
state.online_derenderer = OnlineDerenderer(
model_config=vllm_config.model_config,
renderer=renderer,
model_registry=model_registry,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
@@ -462,13 +498,19 @@ async def init_render_app_state(
state.openai_serving_models = model_registry
state.serving_tokenization = ServingTokenization(
model_registry,
state.openai_serving_render,
state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
)
state.serving_render = ServingRender(
model_registry,
state.online_renderer,
state.online_derenderer,
request_logger=request_logger,
)
state.vllm_config = vllm_config
# Disable stats logging — there is no engine to poll.
@@ -47,7 +47,7 @@ class OpenAIServingChatBatch(OpenAIServingChat):
"""Validate the model and preprocess a batched chat completion request.
Performs engine-aware checks then delegates per-conversation
preprocessing to OpenAIServingRender, validating the chat template
preprocessing to OnlineRenderer, validating the chat template
once for the whole batch.
Returns:
@@ -62,19 +62,19 @@ class OpenAIServingChatBatch(OpenAIServingChat):
if self.engine_client.errored:
raise self.engine_client.dead_error
render = self.openai_serving_render
renderer = self.online_renderer
if not render.use_harmony:
if not renderer.use_harmony:
# Common case: validate the chat template once for the whole batch.
error_check_ret = render.validate_chat_template(
error_check_ret = renderer.validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=render.trust_request_chat_template,
trust_request_chat_template=renderer.trust_request_chat_template,
)
if error_check_ret is not None:
return error_check_ret
parser = render.parser
parser = renderer.parser
tool_dicts: list[dict] | None = None
all_conversations: list[list[ConversationMessage]] = []
@@ -82,17 +82,17 @@ class OpenAIServingChatBatch(OpenAIServingChat):
for messages in request.messages:
single_request = request.to_chat_completion_request(messages)
if render.use_harmony:
conversation, engine_prompts = render._make_request_with_harmony(
if renderer.use_harmony:
conversation, engine_prompts = renderer._make_request_with_harmony(
single_request, should_include_tools=tool_dicts is not None
)
else:
conversation, engine_prompts = await render.preprocess_chat(
conversation, engine_prompts = await renderer.preprocess_chat(
single_request,
messages,
default_template=render.chat_template,
default_template_content_format=render.chat_template_content_format,
default_template_kwargs=render.default_chat_template_kwargs,
default_template=renderer.chat_template,
default_template_content_format=renderer.chat_template_content_format,
default_template_kwargs=renderer.default_chat_template_kwargs,
tool_dicts=tool_dicts,
parser=parser,
)
@@ -7,7 +7,7 @@ import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, Final, cast
from typing import Any, Final, cast
import numpy as np
import pybase64 as base64
@@ -61,14 +61,12 @@ from vllm.outputs import RequestOutput
from vllm.parser import ParserManager
from vllm.parser.abstract_parser import Parser
from vllm.renderers import ChatParams
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.collection_utils import as_list
from vllm.utils.mistral import is_mistral_tool_parser
if TYPE_CHECKING:
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
logger = init_logger(__name__)
@@ -112,7 +110,7 @@ class OpenAIServingChat(OpenAIServing):
models: OpenAIServingModels,
response_role: str,
*,
openai_serving_render: "OpenAIServingRender",
online_renderer: "OnlineRenderer",
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
@@ -135,7 +133,7 @@ class OpenAIServingChat(OpenAIServing):
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.openai_serving_render = openai_serving_render
self.online_renderer = online_renderer
self.response_role = response_role
self.chat_template = chat_template
self.chat_template_content_format: Final = chat_template_content_format
@@ -212,7 +210,7 @@ class OpenAIServingChat(OpenAIServing):
"""
Validate the model and preprocess a chat completion request.
Delegates preprocessing logic to OpenAIServingRender, adding the
Delegates preprocessing logic to OnlineRenderer, adding the
engine-aware checks (LoRA model validation, engine health).
Returns:
@@ -230,7 +228,7 @@ class OpenAIServingChat(OpenAIServing):
if self.engine_client.errored:
raise self.engine_client.dead_error
return await self.openai_serving_render.render_chat(request)
return await self.online_renderer.render_chat(request)
async def create_chat_completion(
self,
@@ -6,7 +6,7 @@ import io
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import TYPE_CHECKING, cast
from typing import cast
import numpy as np
import pybase64 as base64
@@ -41,14 +41,12 @@ from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.outputs import RequestOutput
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.async_utils import merge_async_iterators
from vllm.utils.collection_utils import as_list
if TYPE_CHECKING:
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
logger = init_logger(__name__)
@@ -58,7 +56,7 @@ class OpenAIServingCompletion(OpenAIServing):
engine_client: EngineClient,
models: OpenAIServingModels,
*,
openai_serving_render: "OpenAIServingRender",
online_renderer: "OnlineRenderer",
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
enable_prompt_tokens_details: bool = False,
@@ -71,7 +69,7 @@ class OpenAIServingCompletion(OpenAIServing):
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.openai_serving_render = openai_serving_render
self.online_renderer = online_renderer
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_force_include_usage = enable_force_include_usage
@@ -90,7 +88,7 @@ class OpenAIServingCompletion(OpenAIServing):
"""
Validate the model and preprocess a completion request.
Delegates preprocessing logic to OpenAIServingRender, adding the
Delegates preprocessing logic to OnlineRenderer, adding the
engine-aware checks (LoRA model validation, engine health).
Returns:
@@ -106,7 +104,7 @@ class OpenAIServingCompletion(OpenAIServing):
if self.engine_client.errored:
raise self.engine_client.dead_error
return await self.openai_serving_render.render_completion(request)
return await self.online_renderer.render_completion(request)
async def create_completion(
self,
+2 -6
View File
@@ -13,12 +13,8 @@ from starlette.datastructures import Headers
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionRequest,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
GenerationError,
+5 -5
View File
@@ -92,7 +92,6 @@ from vllm.entrypoints.openai.responses.utils import (
extract_function_tool_names,
extract_tool_types,
)
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import VLLMValidationError
@@ -103,6 +102,7 @@ from vllm.logprobs import SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.outputs import CompletionOutput
from vllm.parser import Parser, ParserManager
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.utils import random_uuid
@@ -155,7 +155,7 @@ class OpenAIServingResponses(OpenAIServing):
self,
engine_client: EngineClient,
models: OpenAIServingModels,
openai_serving_render: OpenAIServingRender,
online_renderer: OnlineRenderer,
*,
request_logger: RequestLogger | None,
chat_template: str | None,
@@ -177,7 +177,7 @@ class OpenAIServingResponses(OpenAIServing):
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.openai_serving_render = openai_serving_render
self.online_renderer = online_renderer
self.chat_template = chat_template
self.chat_template_content_format: Final = chat_template_content_format
self.chat_template_kwargs = default_chat_template_kwargs or {}
@@ -633,7 +633,7 @@ class OpenAIServingResponses(OpenAIServing):
prev_response_output=prev_response.output if prev_response else None,
)
chat_template_kwargs = self._effective_chat_template_kwargs(request)
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
_, engine_inputs = await self.online_renderer.preprocess_chat(
request,
messages,
default_template=self.chat_template,
@@ -657,7 +657,7 @@ class OpenAIServingResponses(OpenAIServing):
request_input=messages,
)
chat_template_kwargs = self._effective_chat_template_kwargs(request)
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
_, engine_inputs = await self.online_renderer.preprocess_chat(
request,
new_messages,
default_template=chat_template,
+1 -3
View File
@@ -10,9 +10,7 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Respons
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
)
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.disagg.protocol import (
GenerateRequest,
GenerateResponse,
+4 -4
View File
@@ -36,7 +36,6 @@ from vllm.entrypoints.serve.disagg.protocol import (
GenerateResponseStreamChoice,
GenerateStreamResponse,
)
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput, mm_input
@@ -48,6 +47,7 @@ from vllm.multimodal.inputs import (
PlaceholderRange,
)
from vllm.outputs import RequestOutput
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.utils.collection_utils import as_list
@@ -61,7 +61,7 @@ class ServingTokens(OpenAIServing):
self,
engine_client: EngineClient,
models: OpenAIServingModels,
openai_serving_render: OpenAIServingRender,
online_renderer: OnlineRenderer,
*,
request_logger: RequestLogger | None,
force_no_detokenize: bool = False,
@@ -75,7 +75,7 @@ class ServingTokens(OpenAIServing):
request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.openai_serving_render = openai_serving_render
self.online_renderer = online_renderer
self.enable_prompt_tokens_details = enable_prompt_tokens_details
self.enable_log_outputs = enable_log_outputs
self.force_no_detokenize = force_no_detokenize
@@ -168,7 +168,7 @@ class ServingTokens(OpenAIServing):
cache_salt=request.cache_salt,
)
else:
(engine_input,) = await self.openai_serving_render.preprocess_completion(
(engine_input,) = await self.online_renderer.preprocess_completion(
request,
prompt_input=request.token_ids,
prompt_embeds=None,
+14 -3
View File
@@ -15,7 +15,12 @@ from vllm.entrypoints.openai.completion.protocol import (
CompletionResponse,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
from vllm.entrypoints.serve.disagg.protocol import (
DerenderChatRequest,
DerenderCompletionRequest,
GenerateRequest,
GenerateResponse,
)
from vllm.entrypoints.serve.tokenize.protocol import (
DetokenizeRequest,
TokenizeChatRequest,
@@ -45,11 +50,17 @@ class RendererChatRequest(RendererRequest, Protocol):
CompletionLikeRequest: TypeAlias = (
CompletionRequest | TokenizeCompletionRequest | DetokenizeRequest
CompletionRequest
| TokenizeCompletionRequest
| DetokenizeRequest
| DerenderCompletionRequest
)
ChatLikeRequest: TypeAlias = (
ChatCompletionRequest | BatchChatCompletionRequest | TokenizeChatRequest
ChatCompletionRequest
| BatchChatCompletionRequest
| TokenizeChatRequest
| DerenderChatRequest
)
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
+3 -3
View File
@@ -19,7 +19,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
DerenderCompletionRequest,
GenerateRequest,
)
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.render.serving import ServingRender
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
@@ -28,8 +28,8 @@ logger = init_logger(__name__)
router = APIRouter()
def render(request: Request) -> OpenAIServingRender | None:
return getattr(request.app.state, "openai_serving_render", None)
def render(request: Request) -> ServingRender | None:
return getattr(request.app.state, "serving_render", None)
@router.post(
+129 -747
View File
@@ -1,44 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Sequence
from http import HTTPStatus
from typing import Any, cast
from typing import cast
from openai_harmony import Message as OpenAIMessage
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import (
ChatTemplateContentFormatOption,
ConversationMessage,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionLogProbs,
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionResponseChoice,
ChatMessage,
)
from vllm.entrypoints.openai.completion.protocol import (
CompletionLogProbs,
CompletionRequest,
CompletionResponse,
CompletionResponseChoice,
)
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
ToolCall,
UsageInfo,
)
from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
from vllm.entrypoints.openai.parser.harmony_utils import (
build_harmony_preamble,
extract_instructions_from_messages,
parse_chat_inputs_to_harmony_messages,
render_for_completion,
from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
from vllm.entrypoints.serve.disagg.protocol import (
DerenderChatRequest,
@@ -47,207 +27,49 @@ from vllm.entrypoints.serve.disagg.protocol import (
MultiModalFeatures,
PlaceholderRangeInfo,
)
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens
from vllm.entrypoints.serve.utils.error_response import create_error_response
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import (
EngineInput,
MultiModalHashes,
MultiModalInput,
MultiModalPlaceholders,
PromptType,
SingletonPrompt,
tokens_input,
)
from vllm.logger import init_logger
from vllm.parser import Parser, ParserManager
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs.preprocess import (
extract_prompt_components,
extract_prompt_len,
parse_model_prompt,
prompt_to_seq,
)
from vllm.tokenizers import TokenizerLike
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.utils import random_uuid
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
from vllm.utils.mistral import mt as _mt
logger = init_logger(__name__)
def _parse_token_id_placeholder(token: str) -> int | None:
"""Extract token ID from a 'token_id:N' placeholder string."""
if not token.startswith("token_id:"):
return None
try:
return int(token[len("token_id:") :])
except ValueError:
return None
def _correct_decoded_token(
token_id: int, context_token_ids: list[int], tokenizer: TokenizerLike
) -> str:
"""Use preceding tokens as context to fix U+FFFD from byte-fallback.
Mirrors LogprobsProcessor._correct_decoded_token in v1/engine/logprobs.py.
"""
max_ctx = min(len(context_token_ids), 4)
for num_ctx in range(1, max_ctx + 1):
context = context_token_ids[-num_ctx:]
full_decoded = tokenizer.decode(context + [token_id])
if full_decoded.endswith("�"):
continue
clean_end = len(context)
for j in range(len(context) - 1, -1, -1):
if tokenizer.decode([context[j]]).endswith("�"):
clean_end = j
else:
break
clean_prefix = tokenizer.decode(context[:clean_end]) if clean_end > 0 else ""
if full_decoded.startswith(clean_prefix):
return full_decoded[len(clean_prefix) :]
common_len = 0
for a, b in zip(clean_prefix, full_decoded):
if a != b:
break
common_len += 1
return full_decoded[common_len:]
return ""
def _resolve_logprobs(
logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
) -> ChatCompletionLogProbs:
"""Resolve token_id:N placeholders in a ChatCompletionLogProbs object."""
if logprobs.content is None:
return logprobs
context_token_ids: list[int] = []
resolved_content = []
for entry in logprobs.content:
token_str, token_bytes = resolve_token_id_placeholder(entry.token, tokenizer)
sampled_id = _parse_token_id_placeholder(entry.token)
if token_str.endswith("�") and sampled_id is not None:
token_str = _correct_decoded_token(sampled_id, context_token_ids, tokenizer)
token_bytes = list(token_str.encode("utf-8"))
resolved_top = []
for top in entry.top_logprobs:
top_str, top_bytes = resolve_token_id_placeholder(top.token, tokenizer)
top_id = _parse_token_id_placeholder(top.token)
if top_str.endswith("�") and top_id is not None:
top_str = _correct_decoded_token(top_id, context_token_ids, tokenizer)
top_bytes = list(top_str.encode("utf-8"))
resolved_top.append(
top.model_copy(update={"token": top_str, "bytes": top_bytes})
)
resolved_content.append(
entry.model_copy(
update={
"token": token_str,
"bytes": token_bytes,
"top_logprobs": resolved_top,
}
)
)
if sampled_id is not None:
context_token_ids.append(sampled_id)
return ChatCompletionLogProbs(content=resolved_content)
def _convert_chat_logprobs_to_completion_logprobs(
logprobs: ChatCompletionLogProbs,
) -> CompletionLogProbs:
"""Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
(parallel flat lists) as required by the /v1/completions response schema."""
if logprobs.content is None:
return CompletionLogProbs()
tokens: list[str] = []
token_logprobs: list[float | None] = []
top_logprobs_list: list[dict[str, float] | None] = []
text_offset: list[int] = []
offset = 0
for entry in logprobs.content:
text_offset.append(offset)
tokens.append(entry.token)
token_logprobs.append(entry.logprob)
top_logprobs_list.append(
{t.token: t.logprob for t in entry.top_logprobs}
if entry.top_logprobs
else None
)
offset += len(entry.token)
return CompletionLogProbs(
text_offset=text_offset,
token_logprobs=token_logprobs,
tokens=tokens,
top_logprobs=top_logprobs_list,
)
class OpenAIServingRender:
class ServingRender(BaseServing):
def __init__(
self,
model_config: ModelConfig,
renderer: BaseRenderer,
model_registry: OpenAIModelRegistry,
models: OpenAIServingModels | OpenAIModelRegistry,
online_renderer: "OnlineRenderer",
online_derenderer: "OnlineDerenderer",
*,
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
trust_request_chat_template: bool = False,
enable_auto_tools: bool = False,
exclude_tools_when_tool_choice_none: bool = False,
tool_parser: str | None = None,
reasoning_parser: str | None = None,
default_chat_template_kwargs: dict[str, Any] | None = None,
log_error_stack: bool = False,
request_logger: RequestLogger | None = None,
) -> None:
self.model_config = model_config
self.renderer = renderer
self.model_registry = model_registry
self.request_logger = request_logger
self.chat_template = chat_template
self.chat_template_content_format: ChatTemplateContentFormatOption = (
chat_template_content_format
super().__init__(
models=models,
model_config=online_renderer.model_config,
request_logger=request_logger,
)
self.trust_request_chat_template = trust_request_chat_template
self.enable_auto_tools = enable_auto_tools
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
self.parser: type[Parser] | None = ParserManager.get_parser(
tool_parser_name=tool_parser,
reasoning_parser_name=reasoning_parser,
enable_auto_tools=enable_auto_tools,
model_name=model_config.model,
is_harmony=self.use_harmony,
)
self.default_chat_template_kwargs: dict[str, Any] = (
default_chat_template_kwargs or {}
)
self.log_error_stack = log_error_stack
self.supports_browsing = False
self.supports_code_interpreter = False
self.default_sampling_params = model_config.get_diff_sampling_param()
mc = model_config
self.online_renderer = online_renderer
self.online_derenderer = online_derenderer
self.default_sampling_params = (
online_renderer.model_config.get_diff_sampling_param()
)
mc = online_renderer.model_config
self.override_max_tokens = (
self.default_sampling_params.get("max_tokens")
if mc.generation_config not in ("auto", "vllm")
@@ -273,7 +95,7 @@ class OpenAIServingRender:
"Beam search is not supported by the render endpoint"
)
result = await self.render_chat(request, skip_mm_cache=True)
result = await self.online_renderer.render_chat(request, skip_mm_cache=True)
if isinstance(result, ErrorResponse):
return result
@@ -319,90 +141,6 @@ class OpenAIServingRender:
priority=request.priority,
)
async def render_chat(
self,
request: ChatCompletionRequest,
*,
skip_mm_cache: bool = False,
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
"""Core preprocessing logic for chat requests (no model/engine check).
Called directly by render_chat_request and delegated to by
OpenAIServingChat.render_chat_request after its engine-aware checks.
"""
tokenizer = self.renderer.tokenizer
tool_parser = self.parser.tool_parser_cls if self.parser is not None else None
if is_mistral_tokenizer(tokenizer):
# because of issues with pydantic we need to potentially
# re-serialize the tool_calls field of the request
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
_mt.truncate_tool_call_ids(request) # type: ignore[arg-type]
_mt.validate_request_params(request)
# Check if tool parsing is unavailable (common condition)
tool_parsing_unavailable = (
tool_parser is None
and not is_mistral_tokenizer(tokenizer)
and not self.use_harmony
)
# Validate tool_choice when tool parsing is required but unavailable
if tool_parsing_unavailable and request.tool_choice not in (
None,
"none",
):
if request.tool_choice == "auto" and not self.enable_auto_tools:
# for hf tokenizers, "auto" tools requires
# --enable-auto-tool-choice and --tool-call-parser
return self.create_error_response(
'"auto" tool choice requires '
"--enable-auto-tool-choice and --tool-call-parser to be set"
)
elif request.tool_choice != "auto":
# "required" or named tool requires tool parser
return self.create_error_response(
f'tool_choice="{request.tool_choice}" requires '
"--tool-call-parser to be set"
)
if request.tools is None or (
request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
):
tool_dicts = None
else:
tool_dicts = [tool.model_dump() for tool in request.tools]
if not self.use_harmony:
# Common case.
error_check_ret = self.validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if error_check_ret is not None:
return error_check_ret
conversation, engine_inputs = await self.preprocess_chat(
request,
request.messages,
default_template=self.chat_template,
default_template_content_format=self.chat_template_content_format,
default_template_kwargs=self.default_chat_template_kwargs,
tool_dicts=tool_dicts,
parser=self.parser,
skip_mm_cache=skip_mm_cache,
)
else:
# For GPT-OSS.
should_include_tools = tool_dicts is not None
conversation, engine_inputs = self._make_request_with_harmony(
request, should_include_tools
)
return conversation, engine_inputs
async def render_completion_request(
self,
request: CompletionRequest,
@@ -415,7 +153,9 @@ class OpenAIServingRender:
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
result = await self.render_completion(request, skip_mm_cache=True)
result = await self.online_renderer.render_completion(
request, skip_mm_cache=True
)
if isinstance(result, ErrorResponse):
return result
generate_requests: list[GenerateRequest] = []
@@ -459,37 +199,116 @@ class OpenAIServingRender:
return generate_requests
async def render_completion(
async def derender_chat_response(
self,
request: CompletionRequest,
*,
skip_mm_cache: bool = False,
) -> list[EngineInput] | ErrorResponse:
"""Core preprocessing logic for completion requests (no model/engine check).
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
Called directly by render_completion_request and delegated to by
OpenAIServingCompletion.render_completion_request after its engine-aware checks.
Non-streaming only: expects the complete GenerateResponse with all
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
When ``request.chat_request`` is provided, the parser splits the
output into (reasoning, content, tool_calls). Otherwise falls
back to plain detokenization.
"""
# Return error for unsupported features.
if request.suffix is not None:
return self.create_error_response("suffix is not currently supported")
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if request.echo and request.prompt_embeds is not None:
return self.create_error_response("Echo is unsupported with prompt embeds.")
if request.prompt_logprobs is not None and request.prompt_embeds is not None:
return self.create_error_response(
"prompt_logprobs is not compatible with prompt embeds."
try:
choices = await self.online_derenderer.derender_chat(
request.generate_response, request.chat_request
)
except ValueError as exc:
return self.create_error_response(str(exc))
engine_inputs = await self.preprocess_completion(
request,
prompt_input=request.prompt,
prompt_embeds=request.prompt_embeds,
skip_mm_cache=skip_mm_cache,
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
gen = request.generate_response
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
return engine_inputs
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
request.model,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Non-streaming only. Mirrors the multi-prompt completions case: one
GenerateResponse per prompt, parallel to the list[GenerateRequest]
from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
(
choices,
total_prompt_tokens,
total_completion_tokens,
) = await self.online_derenderer.derender_completion(
request.generate_responses, request.prompt_tokens
)
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
request.model,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
)
@staticmethod
def _extract_mm_features(
@@ -529,440 +348,3 @@ class OpenAIServingRender:
mm_placeholders=mm_placeholders,
kwargs_data=kwargs_data,
)
def _make_request_with_harmony(
self,
request: ChatCompletionRequest,
should_include_tools: bool = True,
):
"""Build Harmony (GPT-OSS) messages and engine prompt from a chat request."""
messages: list[OpenAIMessage] = []
# because of issues with pydantic we need to potentially
# re-serialize the tool_calls field of the request
# for more info: see comment in `maybe_serialize_tool_calls`
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
chat_messages = list(request.messages)
instructions, chat_messages = extract_instructions_from_messages(chat_messages)
# Add system message.
# NOTE: In Chat Completion API, browsing is enabled by default
# if the model supports it. TODO: Support browsing.
assert not self.supports_browsing
assert not self.supports_code_interpreter
if (reasoning_effort := request.reasoning_effort) == "none":
raise ValueError(f"Harmony does not support {reasoning_effort=}")
tools = request.tools if should_include_tools else None
messages.extend(
build_harmony_preamble(
instructions=instructions,
tools=tools, # type: ignore[arg-type]
reasoning_effort=reasoning_effort,
with_custom_tools=should_include_tools,
)
)
# Add remaining conversation messages.
messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages))
# Render prompt token ids.
prompt_token_ids = render_for_completion(messages)
engine_input = tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
return messages, [engine_input]
async def derender_chat_response(
self,
request: DerenderChatRequest,
) -> ChatCompletionResponse | ErrorResponse:
"""Postprocess a GenerateResponse into a ChatCompletionResponse.
Non-streaming only: expects the complete GenerateResponse with all
token IDs present. Uses ``parser.parse()`` for one-shot extraction.
When ``request.chat_request`` is provided, the parser splits the
output into (reasoning, content, tool_calls). Otherwise falls
back to plain detokenization.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
tokenizer = self.renderer.get_tokenizer()
gen = request.generate_response
chat_request = request.chat_request
choices: list[ChatCompletionResponseChoice] = []
try:
for choice in gen.choices:
if not choice.token_ids:
raise ValueError(
f"choice {choice.index} has empty or null token_ids"
)
resolved_logprobs = (
_resolve_logprobs(choice.logprobs, tokenizer)
if choice.logprobs is not None
else None
)
if self.parser is not None and chat_request is not None:
# Parser path: decode with special tokens preserved
# so the parser can see markers like </think>,
# <tool_call>, or Harmony channel tokens.
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=False
)
chat_template_kwargs: dict[str, Any] = {}
if not self.use_harmony:
chat_template_kwargs = (
chat_request.build_chat_params(
self.chat_template,
self.chat_template_content_format,
)
.with_defaults(self.default_chat_template_kwargs)
.chat_template_kwargs
)
parser = self.parser(
tokenizer,
chat_request.tools,
chat_template_kwargs=chat_template_kwargs,
)
reasoning, content, tool_calls = parser.parse(
decoded_text,
chat_request,
enable_auto_tools=self.enable_auto_tools,
model_output_token_ids=choice.token_ids,
)
if not getattr(chat_request, "include_reasoning", True):
reasoning = None
tc_items = (
[
ToolCall(
id=random_uuid(),
function=tc,
)
for tc in tool_calls
]
if tool_calls
else []
)
message = ChatMessage(
role="assistant",
reasoning=reasoning,
content=content,
tool_calls=tc_items,
)
else:
# No parser: plain detokenization.
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
)
message = ChatMessage(role="assistant", content=decoded_text)
choices.append(
ChatCompletionResponseChoice(
index=choice.index,
message=message,
logprobs=resolved_logprobs,
finish_reason=choice.finish_reason,
)
)
except ValueError as exc:
return self.create_error_response(str(exc))
prompt_tokens = (
request.prompt_tokens if request.prompt_tokens is not None else 0
)
completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
usage = UsageInfo(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=prompt_tokens + completion_tokens,
)
logger.debug(
"derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
gen.request_id,
request.model,
len(choices),
completion_tokens,
)
return ChatCompletionResponse(
id=gen.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
prompt_logprobs=gen.prompt_logprobs,
kv_transfer_params=gen.kv_transfer_params,
)
async def derender_completion_response(
self,
request: DerenderCompletionRequest,
) -> CompletionResponse | ErrorResponse:
"""Postprocess a list of GenerateResponses into a CompletionResponse.
Non-streaming only. Mirrors the multi-prompt completions case: one
GenerateResponse per prompt, parallel to the list[GenerateRequest]
from /v1/completions/render.
"""
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
n = len(request.generate_responses)
prompt_tokens_list: list[int] = (
request.prompt_tokens if request.prompt_tokens is not None else [0] * n
)
tokenizer = self.renderer.get_tokenizer()
choices: list[CompletionResponseChoice] = []
total_prompt_tokens = 0
total_completion_tokens = 0
index = 0
for gen, pt in zip(request.generate_responses, prompt_tokens_list):
for choice in gen.choices:
if not choice.token_ids:
return self.create_error_response(
f"choice {choice.index} in response {gen.request_id} "
"has empty or null token_ids"
)
decoded_text = tokenizer.decode(
choice.token_ids, skip_special_tokens=True
)
completion_logprobs = None
if choice.logprobs is not None:
resolved = _resolve_logprobs(choice.logprobs, tokenizer)
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
resolved
)
choices.append(
CompletionResponseChoice(
index=index,
text=decoded_text,
finish_reason=choice.finish_reason,
logprobs=completion_logprobs,
)
)
total_completion_tokens += len(choice.token_ids)
index += 1
total_prompt_tokens += pt
if not request.generate_responses:
return self.create_error_response("generate_responses must not be empty")
first = request.generate_responses[0]
kv_params = first.kv_transfer_params
if any(
r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
):
logger.warning(
"derender_completion: kv_transfer_params differ across responses; "
"setting to None on the aggregated response"
)
kv_params = None
usage = UsageInfo(
prompt_tokens=total_prompt_tokens,
completion_tokens=total_completion_tokens,
total_tokens=total_prompt_tokens + total_completion_tokens,
)
logger.debug(
"derender_completion request_id=%s model=%s choices=%d"
" completion_tokens=%d",
first.request_id,
request.model,
len(choices),
total_completion_tokens,
)
return CompletionResponse(
id=first.request_id,
model=request.model,
created=int(time.time()),
choices=choices,
usage=usage,
kv_transfer_params=kv_params,
)
def create_error_response(
self,
message: str | Exception,
err_type: str = "BadRequestError",
status_code: HTTPStatus = HTTPStatus.BAD_REQUEST,
param: str | None = None,
) -> ErrorResponse:
return create_error_response(message, err_type, status_code, param)
async def _check_model(
self,
request: Any,
) -> ErrorResponse | None:
return await self.model_registry.check_model(request.model)
def validate_chat_template(
self,
request_chat_template: str | None,
chat_template_kwargs: dict[str, Any] | None,
trust_request_chat_template: bool,
) -> ErrorResponse | None:
"""Copied from OpenAIServing._validate_chat_template."""
if not trust_request_chat_template and (
request_chat_template is not None
or (
chat_template_kwargs
and chat_template_kwargs.get("chat_template") is not None
)
):
return self.create_error_response(
"Chat template is passed with request, but "
"--trust-request-chat-template is not set. "
"Refused request with untrusted chat template."
)
return None
async def preprocess_completion(
self,
request: Any,
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
prompt_embeds: bytes | list[bytes] | None,
*,
skip_mm_cache: bool = False,
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_completion."""
prompts = list[SingletonPrompt | bytes]()
if prompt_embeds is not None: # embeds take higher priority
prompts.extend(prompt_to_seq(prompt_embeds))
if prompt_input is not None:
prompts.extend(prompt_to_seq(prompt_input))
return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache)
async def preprocess_cmpl(
self,
request: Any,
prompts: Sequence[PromptType | bytes],
*,
skip_mm_cache: bool = False,
) -> list[EngineInput]:
"""Copied from OpenAIServing._preprocess_cmpl."""
renderer = self.renderer
model_config = self.model_config
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(model_config, prompt)
)
for prompt in prompts
]
tok_params = request.build_tok_params(model_config)
return await renderer.render_cmpl_async(
parsed_prompts,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
skip_mm_cache=skip_mm_cache,
)
async def preprocess_chat(
self,
request: Any,
messages: list[Any],
default_template: str | None,
default_template_content_format: ChatTemplateContentFormatOption,
default_template_kwargs: dict[str, Any] | None,
tool_dicts: list[dict[str, Any]] | None = None,
parser: type[Parser] | None = None,
*,
skip_mm_cache: bool = False,
) -> tuple[list[ConversationMessage], list[EngineInput]]:
"""Copied from OpenAIServing._preprocess_chat."""
renderer = self.renderer
mm_config = self.model_config.multimodal_config
default_template_kwargs = merge_kwargs(
default_template_kwargs,
dict(
tools=tool_dicts,
tokenize=(
is_mistral_tokenizer(renderer.tokenizer)
or self.model_config.enable_prompt_embeds
),
),
)
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
default_template, default_template_content_format
).with_defaults(
default_template_kwargs,
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
)
(conversation,), (engine_input,) = await renderer.render_chat_async(
[messages],
chat_params,
tok_params,
prompt_extras={
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := getattr(request, k, None)) is not None
},
skip_mm_cache=skip_mm_cache,
)
# tool parsing is done only if a tool_parser has been set and if
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
# is set, we want to prevent parsing a tool_call hallucinated by the LLM
#
# Exception: Mistral grammar-capable tokenizers always call
# adjust_request — even for tool_choice="none" — so that the grammar
# factory can prevent special-token leakage.
if parser is not None:
tokenizer = renderer.get_tokenizer()
tool_parser = parser.tool_parser_cls
tool_choice = getattr(request, "tool_choice", "none")
is_mistral_grammar_eligible = (
tool_parser is not None
and is_mistral_tool_parser(tool_parser)
and is_mistral_tokenizer(tokenizer)
and tokenizer.supports_grammar
)
should_adjust_request = (
parser.reasoning_parser_cls is not None
or tool_choice != "none"
or is_mistral_grammar_eligible
)
if should_adjust_request:
if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
msg = (
"Tool usage is only supported "
"for Chat Completions API or Responses API requests, "
f"but got {type(request).__name__}"
)
raise NotImplementedError(msg)
request = parser(
tokenizer,
request.tools,
model_config=self.model_config,
chat_template_kwargs=chat_params.chat_template_kwargs,
).adjust_request(
request=request,
)
return conversation, [engine_input]
+9 -9
View File
@@ -12,7 +12,6 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIServingModels,
)
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.serve.tokenize.protocol import (
DetokenizeRequest,
DetokenizeResponse,
@@ -24,6 +23,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import TokensPrompt, tokens_input
from vllm.logger import init_logger
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tokenizers import TokenizerLike
logger = init_logger(__name__)
@@ -33,22 +33,22 @@ class ServingTokenization(BaseServing):
def __init__(
self,
models: OpenAIServingModels | OpenAIModelRegistry,
openai_serving_render: OpenAIServingRender,
online_renderer: OnlineRenderer,
*,
request_logger: RequestLogger | None,
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
default_chat_template_kwargs: dict[str, Any] | None = None,
trust_request_chat_template: bool = False,
request_logger: RequestLogger | None = None,
) -> None:
super().__init__(
models=models,
model_config=openai_serving_render.model_config,
model_config=online_renderer.model_config,
request_logger=request_logger,
)
self.renderer = openai_serving_render.renderer
self.openai_serving_render = openai_serving_render
self.renderer = online_renderer.renderer
self.online_renderer = online_renderer
self.chat_template = chat_template
self.chat_template_content_format: Final = chat_template_content_format
self.default_chat_template_kwargs = default_chat_template_kwargs or {}
@@ -73,7 +73,7 @@ class ServingTokenization(BaseServing):
if request.tools is None
else [tool.model_dump() for tool in request.tools]
)
error_check_ret = self.openai_serving_render.validate_chat_template(
error_check_ret = self.online_renderer.validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
@@ -81,7 +81,7 @@ class ServingTokenization(BaseServing):
if error_check_ret is not None:
return error_check_ret
_, engine_inputs = await self.openai_serving_render.preprocess_chat(
_, engine_inputs = await self.online_renderer.preprocess_chat(
request,
request.messages,
default_template=self.chat_template,
@@ -91,7 +91,7 @@ class ServingTokenization(BaseServing):
skip_mm_cache=True,
)
else:
engine_inputs = await self.openai_serving_render.preprocess_completion(
engine_inputs = await self.online_renderer.preprocess_completion(
request,
prompt_input=request.prompt,
prompt_embeds=None,