[Frontend] Introduce Renderer for processing chat messages (using ModelConfig) (#30200)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-01-22 12:44:22 +00:00
committed by GitHub
parent 421012b63a
commit d117a4d1a9
48 changed files with 2141 additions and 1585 deletions
+2 -4
View File
@@ -34,6 +34,7 @@ import vllm.envs as envs
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.launcher import serve_http
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.mcp.tool_server import DemoToolServer, MCPToolServer, ToolServer
@@ -62,7 +63,6 @@ from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
from vllm.entrypoints.utils import (
cli_env_setup,
log_non_default_args,
process_chat_template,
process_lora_modules,
sanitize_message,
)
@@ -662,9 +662,7 @@ async def init_app_state(
supported_tasks = await engine_client.get_supported_tasks()
logger.info("Supported tasks: %s", supported_tasks)
resolved_chat_template = await process_chat_template(
args.chat_template, engine_client, vllm_config.model_config
)
resolved_chat_template = load_chat_template(args.chat_template)
if args.tool_server == "demo":
tool_server: ToolServer | None = DemoToolServer()
@@ -186,8 +186,7 @@ class OpenAIServingChat(OpenAIServing):
start_time = time.perf_counter()
try:
# Get the tokenizer from the engine
tokenizer = await self.engine_client.get_tokenizer()
renderer = self.engine_client.renderer
# Create a minimal dummy request
dummy_request = ChatCompletionRequest(
@@ -203,7 +202,7 @@ class OpenAIServingChat(OpenAIServing):
# 3. Tokenizer initialization for chat
await self._preprocess_chat(
dummy_request,
tokenizer,
renderer,
dummy_request.messages,
chat_template=self.chat_template,
chat_template_content_format=self.chat_template_content_format,
@@ -247,7 +246,8 @@ class OpenAIServingChat(OpenAIServing):
raise self.engine_client.dead_error
try:
tokenizer = await self.engine_client.get_tokenizer()
renderer = self.engine_client.renderer
tokenizer = renderer.tokenizer
tool_parser = self.tool_parser
@@ -308,7 +308,7 @@ class OpenAIServingChat(OpenAIServing):
conversation, engine_prompts = await self._preprocess_chat(
request,
tokenizer,
renderer,
request.messages,
chat_template=request.chat_template or self.chat_template,
chat_template_content_format=self.chat_template_content_format,
@@ -365,8 +365,6 @@ class OpenAIServingChat(OpenAIServing):
)
model_name = self.models.model_name(lora_request)
tokenizer = await self.engine_client.get_tokenizer()
except (ValueError, TypeError, RuntimeError) as e:
logger.exception("Error preparing request components")
return self.create_error_response(e)
@@ -463,6 +461,8 @@ class OpenAIServingChat(OpenAIServing):
(result_generator,) = generators
# Streaming response
tokenizer = self.renderer.tokenizer
if request.stream:
return self.chat_completion_stream_generator(
request,
@@ -1784,7 +1784,7 @@ class OpenAIServingChat(OpenAIServing):
else:
if tokenizer is None:
raise ValueError(
"Tokenizer not available when `skip_tokenizer_init=True`"
"Unable to get tokenizer because `skip_tokenizer_init=True`"
)
token = tokenizer.decode(token_id)
+3 -11
View File
@@ -117,12 +117,7 @@ class OpenAIServingCompletion(OpenAIServing):
)
try:
if self.model_config.skip_tokenizer_init:
tokenizer = None
else:
tokenizer = await self.engine_client.get_tokenizer()
renderer = self._get_renderer(tokenizer)
renderer = self._get_completion_renderer()
engine_prompts = await renderer.render_prompt_and_embeds(
prompt_or_prompts=request.prompt,
prompt_embeds=request.prompt_embeds,
@@ -163,11 +158,6 @@ class OpenAIServingCompletion(OpenAIServing):
try:
lora_request = self._maybe_get_adapters(request)
if self.model_config.skip_tokenizer_init:
tokenizer = None
else:
tokenizer = await self.engine_client.get_tokenizer()
except (ValueError, TypeError, RuntimeError) as e:
logger.exception("Error preparing request components")
return self.create_error_response(e)
@@ -280,6 +270,8 @@ class OpenAIServingCompletion(OpenAIServing):
stream = request.stream and not request.use_beam_search
# Streaming response
tokenizer = self.renderer.tokenizer
if stream:
return self.completion_stream_generator(
request,
+54 -109
View File
@@ -6,10 +6,9 @@ import sys
import time
import traceback
from collections.abc import AsyncGenerator, Callable, Iterable, Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from http import HTTPStatus
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, cast
import numpy as np
from fastapi import Request
@@ -26,10 +25,6 @@ from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
ConversationMessage,
apply_hf_chat_template,
apply_mistral_chat_template,
parse_chat_messages_futures,
resolve_chat_template_content_format,
)
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -113,10 +108,9 @@ from vllm.multimodal import MultiModalDataDict
from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
from vllm.pooling_params import PoolingParams
from vllm.reasoning import ReasoningParser, ReasoningParserManager
from vllm.renderers import RendererLike
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tool_parsers import ToolParser, ToolParserManager
from vllm.tracing import (
contains_trace_headers,
@@ -127,10 +121,8 @@ from vllm.utils import random_uuid
from vllm.utils.async_utils import (
AsyncMicrobatchTokenizer,
collect_from_async_generator,
make_async,
merge_async_iterators,
)
from vllm.utils.collection_utils import is_list_of
from vllm.v1.engine import EngineCoreRequest
@@ -215,7 +207,6 @@ class ResponseGenerationMixin:
@dataclass(kw_only=True)
class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[RequestT]):
# Shared across all requests
request: RequestT
raw_request: Request | None = None
model_name: str
@@ -223,9 +214,6 @@ class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[Requ
created_time: int = field(default_factory=lambda: int(time.time()))
lora_request: LoRARequest | None = None
# Shared across most requests
tokenizer: TokenizerLike | None = None
@dataclass(kw_only=True)
class ClassificationServeContext(ServeContext[ClassificationRequest]):
@@ -261,16 +249,13 @@ class OpenAIServing:
self.request_logger = request_logger
self.return_tokens_as_token_ids = return_tokens_as_token_ids
self._tokenizer_executor = ThreadPoolExecutor(max_workers=1)
self._apply_mistral_chat_template_async = make_async(
apply_mistral_chat_template, executor=self._tokenizer_executor
)
self._async_tokenizer_pool: dict[TokenizerLike, AsyncMicrobatchTokenizer] = {}
self.log_error_stack = log_error_stack
self.input_processor = self.models.input_processor
self.io_processor = self.models.io_processor
self.renderer = self.models.renderer
self.model_config = self.models.model_config
self.max_model_len = self.model_config.max_model_len
@@ -557,14 +542,14 @@ class OpenAIServing:
prompt_logprobs=None,
)
def _get_renderer(self, tokenizer: TokenizerLike | None) -> BaseRenderer:
def _get_completion_renderer(self) -> BaseRenderer:
"""
Get a Renderer instance with the provided tokenizer.
Uses shared async tokenizer pool for efficiency.
"""
return CompletionRenderer(
model_config=self.model_config,
tokenizer=tokenizer,
tokenizer=self.renderer.tokenizer,
async_tokenizer_pool=self._async_tokenizer_pool,
)
@@ -1183,7 +1168,7 @@ class OpenAIServing:
async def _preprocess_chat(
self,
request: ChatLikeRequest | ResponsesRequest,
tokenizer: TokenizerLike | None,
renderer: RendererLike,
messages: list[ChatCompletionMessageParam],
chat_template: str | None,
chat_template_content_format: ChatTemplateContentFormatOption,
@@ -1196,59 +1181,58 @@ class OpenAIServing:
tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
add_special_tokens: bool = False,
) -> tuple[list[ConversationMessage], list[TokensPrompt]]:
model_config = self.model_config
resolved_content_format = resolve_chat_template_content_format(
chat_template,
tool_dicts,
chat_template_content_format,
tokenizer,
model_config=model_config,
)
conversation, mm_data_future, mm_uuids = parse_chat_messages_futures(
messages,
model_config,
content_format=resolved_content_format,
)
_chat_template_kwargs: dict[str, Any] = dict(
chat_template=chat_template,
add_generation_prompt=add_generation_prompt,
continue_final_message=continue_final_message,
tools=tool_dicts,
documents=documents,
)
_chat_template_kwargs |= self._prepare_extra_chat_template_kwargs(
chat_template_kwargs = {
"chat_template": chat_template,
"add_generation_prompt": add_generation_prompt,
"continue_final_message": continue_final_message,
"tools": tool_dicts,
"documents": documents,
**(chat_template_kwargs or {}),
}
chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
chat_template_kwargs,
default_chat_template_kwargs,
)
request_prompt: str | list[int]
# Use the async tokenizer in `OpenAIServing` if possible.
# Later we can move it into the renderer so that we can return both
# text and token IDs in the same prompt from `render_messages_async`
# which is used for logging and `enable_response_messages`.
from vllm.tokenizers.mistral import MistralTokenizer
if tokenizer is None:
request_prompt = "placeholder"
elif isinstance(tokenizer, MistralTokenizer):
request_prompt = await self._apply_mistral_chat_template_async(
tokenizer,
messages=messages,
**_chat_template_kwargs,
)
elif isinstance(tokenizer, DeepseekV32Tokenizer):
request_prompt = tokenizer.apply_chat_template(
conversation=conversation,
messages=messages,
model_config=model_config,
**_chat_template_kwargs,
conversation, engine_prompt = await renderer.render_messages_async(
messages,
chat_template_content_format=chat_template_content_format,
tokenize=(
chat_template_kwargs.pop("tokenize", False)
or isinstance(renderer.tokenizer, MistralTokenizer)
),
**chat_template_kwargs,
)
if "prompt_token_ids" not in engine_prompt:
extra_data = engine_prompt
engine_prompt = await self._tokenize_prompt_input_async(
request,
renderer.get_tokenizer(),
engine_prompt["prompt"],
add_special_tokens=add_special_tokens,
)
# Fill in other keys like MM data
engine_prompt.update(extra_data) # type: ignore
else:
request_prompt = apply_hf_chat_template(
tokenizer=tokenizer,
conversation=conversation,
model_config=model_config,
**_chat_template_kwargs,
self._validate_input(
request=request,
input_ids=engine_prompt["prompt_token_ids"], # type: ignore
input_text="",
)
mm_data = await mm_data_future
engine_prompt = cast(TokensPrompt, engine_prompt)
if request.mm_processor_kwargs is not None:
engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
if (cache_salt := getattr(request, "cache_salt", None)) is not None:
engine_prompt["cache_salt"] = cache_salt
# tool parsing is done only if a tool_parser has been set and if
# tool_choice is not "none" (if tool_choice is "none" but a tool_parser
@@ -1264,49 +1248,10 @@ class OpenAIServing:
"or Responses API requests."
)
raise NotImplementedError(msg)
tokenizer = renderer.get_tokenizer()
request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore
if tokenizer is None:
assert isinstance(request_prompt, str), (
"Prompt has to be a string",
"when the tokenizer is not initialised",
)
prompt_inputs = TokensPrompt(prompt=request_prompt, prompt_token_ids=[1])
elif isinstance(request_prompt, str):
prompt_inputs = await self._tokenize_prompt_input_async(
request,
tokenizer,
request_prompt,
add_special_tokens=add_special_tokens,
)
else:
# For MistralTokenizer
assert is_list_of(request_prompt, int), (
"Prompt has to be either a string or a list of token ids"
)
input_text = tokenizer.decode(request_prompt)
prompt_inputs = self._validate_input(
request=request,
input_ids=request_prompt,
input_text=input_text,
)
engine_prompt = TokensPrompt(prompt_token_ids=prompt_inputs["prompt_token_ids"])
if "prompt" in prompt_inputs:
engine_prompt["prompt"] = prompt_inputs["prompt"]
if mm_data is not None:
engine_prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
engine_prompt["multi_modal_uuids"] = mm_uuids
if request.mm_processor_kwargs is not None:
engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
if hasattr(request, "cache_salt") and request.cache_salt is not None:
engine_prompt["cache_salt"] = request.cache_salt
return conversation, [engine_prompt]
async def _process_inputs(
@@ -1341,7 +1286,7 @@ class OpenAIServing:
async def _render_next_turn(
self,
request: ResponsesRequest,
tokenizer: TokenizerLike | None,
renderer: RendererLike,
messages: list[ResponseInputOutputItem],
tool_dicts: list[dict[str, Any]] | None,
tool_parser,
@@ -1354,7 +1299,7 @@ class OpenAIServing:
_, engine_prompts = await self._preprocess_chat(
request,
tokenizer,
renderer,
new_messages,
tool_dicts=tool_dicts,
tool_parser=tool_parser,
@@ -1431,7 +1376,7 @@ class OpenAIServing:
elif isinstance(context, ParsableContext):
engine_prompts = await self._render_next_turn(
context.request,
context.tokenizer,
context.renderer,
context.parser.response_messages,
context.tool_dicts,
context.tool_parser_cls,
@@ -61,6 +61,7 @@ class OpenAIServingModels:
self.input_processor = self.engine_client.input_processor
self.io_processor = self.engine_client.io_processor
self.renderer = self.engine_client.renderer
self.model_config = self.engine_client.model_config
self.max_model_len = self.model_config.max_model_len
+4 -1
View File
@@ -43,6 +43,7 @@ from vllm.entrypoints.openai.responses.protocol import (
from vllm.entrypoints.openai.responses.utils import construct_tool_dicts
from vllm.outputs import RequestOutput
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
from vllm.renderers import RendererLike
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.utils import random_uuid
@@ -260,7 +261,7 @@ class ParsableContext(ConversationContext):
self,
*,
response_messages: list[ResponseInputOutputItem],
tokenizer: TokenizerLike,
renderer: RendererLike,
reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser] | None,
request: ResponsesRequest,
available_tools: list[str] | None,
@@ -279,6 +280,7 @@ class ParsableContext(ConversationContext):
if reasoning_parser_cls is None:
raise ValueError("reasoning_parser_cls must be provided.")
tokenizer = renderer.get_tokenizer()
self.parser = get_responses_parser_for_simple_context(
tokenizer=tokenizer,
reasoning_parser_cls=reasoning_parser_cls,
@@ -288,6 +290,7 @@ class ParsableContext(ConversationContext):
)
self.tool_parser_cls = tool_parser_cls
self.request = request
self.renderer = renderer
self.tokenizer = tokenizer
self.available_tools = available_tools or []
+8 -5
View File
@@ -121,6 +121,7 @@ from vllm.logger import init_logger
from vllm.logprobs import Logprob as SampleLogprob
from vllm.logprobs import SampleLogprobs
from vllm.outputs import CompletionOutput
from vllm.renderers import RendererLike
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
from vllm.tokenizers import TokenizerLike
from vllm.utils import random_uuid
@@ -380,7 +381,8 @@ class OpenAIServingResponses(OpenAIServing):
try:
lora_request = self._maybe_get_adapters(request)
model_name = self.models.model_name(lora_request)
tokenizer = await self.engine_client.get_tokenizer()
renderer = self.engine_client.renderer
tokenizer = renderer.get_tokenizer()
if self.use_harmony:
messages, engine_prompts = self._make_request_with_harmony(
@@ -388,7 +390,7 @@ class OpenAIServingResponses(OpenAIServing):
)
else:
messages, engine_prompts = await self._make_request(
request, prev_response, tokenizer
request, prev_response, renderer
)
except (
@@ -454,7 +456,7 @@ class OpenAIServingResponses(OpenAIServing):
# tokens during generation instead of at the end
context = ParsableContext(
response_messages=messages,
tokenizer=tokenizer,
renderer=renderer,
reasoning_parser_cls=self.reasoning_parser,
request=request,
tool_parser_cls=self.tool_parser,
@@ -585,7 +587,7 @@ class OpenAIServingResponses(OpenAIServing):
self,
request: ResponsesRequest,
prev_response: ResponsesResponse | None,
tokenizer: TokenizerLike,
renderer: RendererLike,
):
tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)
# Construct the input messages.
@@ -607,7 +609,7 @@ class OpenAIServingResponses(OpenAIServing):
_, engine_prompts = await self._preprocess_chat(
request,
tokenizer,
renderer,
messages,
tool_dicts=tool_dicts,
tool_parser=self.tool_parser,
@@ -631,6 +633,7 @@ class OpenAIServingResponses(OpenAIServing):
raise NotImplementedError(
"Only 'auto' tool_choice is supported in response API with Harmony"
)
messages = self._construct_input_messages_with_harmony(request, prev_response)
prompt_token_ids = render_for_completion(messages)
engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)