[Bug] Fix Failure in /v1/chat/completions/render for Multimodal Requests (https://github.com/vllm-project/vllm/issues/35665) (#35684)
This commit is contained in:
@@ -368,6 +368,7 @@ async def init_app_state(
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
)
|
||||
|
||||
@@ -457,6 +458,9 @@ async def init_render_app_state(
|
||||
|
||||
state.openai_serving_models = model_registry
|
||||
|
||||
# Expose tokenization via the render handler (no engine required).
|
||||
state.openai_serving_tokenization = state.openai_serving_render
|
||||
|
||||
state.vllm_config = vllm_config
|
||||
# Disable stats logging — there is no engine to poll.
|
||||
state.log_stats = False
|
||||
|
||||
@@ -17,7 +17,6 @@ from pydantic import (
|
||||
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
@@ -269,53 +268,3 @@ class GenerationError(Exception):
|
||||
def __init__(self, message: str = "Internal server error"):
|
||||
super().__init__(message)
|
||||
self.status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
|
||||
|
||||
####### Tokens IN <> Tokens OUT #######
|
||||
class GenerateRequest(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=random_uuid,
|
||||
description=(
|
||||
"The request_id related to this request. If the caller does "
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
token_ids: list[int]
|
||||
"""The token ids to generate text from."""
|
||||
|
||||
# features: MultiModalFeatureSpec
|
||||
# TODO (NickLucche): implement once Renderer work is completed
|
||||
features: str | None = None
|
||||
"""The processed MM inputs for the model."""
|
||||
|
||||
sampling_params: SamplingParams
|
||||
"""The sampling parameters for the model."""
|
||||
|
||||
model: str | None = None
|
||||
|
||||
stream: bool | None = False
|
||||
stream_options: StreamOptions | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
priority: int = Field(
|
||||
default=0,
|
||||
description=(
|
||||
"The priority of the request (lower means earlier handling; "
|
||||
"default: 0). Any priority other than 0 will raise an error "
|
||||
"if the served model does not use priority scheduling."
|
||||
),
|
||||
)
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
|
||||
@@ -11,7 +11,7 @@ from contextlib import asynccontextmanager
|
||||
from http import HTTPStatus
|
||||
|
||||
import pydantic
|
||||
from fastapi import FastAPI, HTTPException, Request, Response
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.concurrency import iterate_in_threadpool
|
||||
@@ -350,7 +350,8 @@ async def engine_error_handler(
|
||||
server=req.app.state.server,
|
||||
engine=req.app.state.engine_client,
|
||||
)
|
||||
return Response(status_code=HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
|
||||
async def exception_handler(req: Request, exc: Exception):
|
||||
|
||||
@@ -2,20 +2,55 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
SamplingParams,
|
||||
StreamOptions,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import StreamOptions
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
|
||||
####### Tokens IN <> Tokens OUT #######
|
||||
|
||||
|
||||
class PlaceholderRangeInfo(BaseModel):
|
||||
"""Serializable placeholder location for a single multi-modal item."""
|
||||
|
||||
offset: int
|
||||
"""Start index of the placeholder tokens in the prompt."""
|
||||
|
||||
length: int
|
||||
"""Number of placeholder tokens."""
|
||||
|
||||
# TODO: add ``is_embed: list[bool] | None`` once the /generate side
|
||||
# consumes features — some models (e.g. Qwen-VL) use sparse
|
||||
# placeholder masks that cannot be recomputed from offset+length alone.
|
||||
|
||||
|
||||
class MultiModalFeatures(BaseModel):
|
||||
"""Lightweight multimodal metadata produced by the render step.
|
||||
|
||||
Carries hashes (for cache lookup / identification) and placeholder
|
||||
positions so the downstream ``/generate`` service knows *where* in
|
||||
the token sequence each multimodal item lives.
|
||||
|
||||
.. note:: Phase 1 — metadata only.
|
||||
Phase 2 should add ``mm_kwargs`` (processed tensor data) using a
|
||||
binary transport so the ``/generate`` side can skip re-processing.
|
||||
The ``/generate`` endpoint must also be updated to inject these
|
||||
features into ``ProcessorInputs`` before passing to
|
||||
``InputProcessor.process_inputs``.
|
||||
"""
|
||||
|
||||
mm_hashes: dict[str, list[str]]
|
||||
"""Per-modality item hashes, e.g. ``{"image": ["abc", "def"]}``."""
|
||||
|
||||
mm_placeholders: dict[str, list[PlaceholderRangeInfo]]
|
||||
"""Per-modality placeholder ranges in the token sequence."""
|
||||
|
||||
|
||||
class GenerateRequest(BaseModel):
|
||||
request_id: str = Field(
|
||||
default_factory=lambda: f"{random_uuid()}",
|
||||
@@ -28,10 +63,15 @@ class GenerateRequest(BaseModel):
|
||||
token_ids: list[int]
|
||||
"""The token ids to generate text from."""
|
||||
|
||||
# features: MultiModalFeatureSpec
|
||||
# TODO (NickLucche): implement once Renderer work is completed
|
||||
features: str | None = None
|
||||
"""The processed MM inputs for the model."""
|
||||
@field_validator("token_ids")
|
||||
@classmethod
|
||||
def validate_token_ids(cls, v: list[int]) -> list[int]:
|
||||
if any(t < 0 for t in v):
|
||||
raise ValueError("token_ids must not contain negative values")
|
||||
return v
|
||||
|
||||
features: MultiModalFeatures | None = None
|
||||
"""Multimodal hashes and placeholder positions (populated for MM inputs)."""
|
||||
|
||||
sampling_params: SamplingParams
|
||||
"""The sampling parameters for the model."""
|
||||
|
||||
@@ -9,6 +9,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionReque
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.openai.utils import validate_json_request
|
||||
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -24,7 +25,7 @@ def render(request: Request) -> OpenAIServingRender | None:
|
||||
@router.post(
|
||||
"/v1/chat/completions/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=list,
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
@@ -44,13 +45,13 @@ async def render_chat_completion(request: ChatCompletionRequest, raw_request: Re
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/completions/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=list,
|
||||
response_model=list[GenerateRequest],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
@@ -67,7 +68,7 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
|
||||
return JSONResponse(content=result)
|
||||
return JSONResponse(content=[item.model_dump() for item in result])
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
|
||||
@@ -24,14 +24,29 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
parse_chat_inputs_to_harmony_messages,
|
||||
render_for_completion,
|
||||
)
|
||||
from vllm.entrypoints.utils import create_error_response
|
||||
from vllm.entrypoints.serve.disagg.protocol import (
|
||||
GenerateRequest,
|
||||
MultiModalFeatures,
|
||||
PlaceholderRangeInfo,
|
||||
)
|
||||
from vllm.entrypoints.utils import (
|
||||
create_error_response,
|
||||
get_max_tokens,
|
||||
)
|
||||
from vllm.inputs.data import ProcessorInputs, PromptType, SingletonPrompt, TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.inputs import MultiModalHashes, MultiModalPlaceholderDict
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
extract_prompt_len,
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from vllm.utils.mistral import mt as _mt
|
||||
|
||||
@@ -83,10 +98,18 @@ class OpenAIServingRender:
|
||||
self.supports_browsing = False
|
||||
self.supports_code_interpreter = False
|
||||
|
||||
self.default_sampling_params = model_config.get_diff_sampling_param()
|
||||
mc = model_config
|
||||
self.override_max_tokens = (
|
||||
self.default_sampling_params.get("max_tokens")
|
||||
if mc.generation_config not in ("auto", "vllm")
|
||||
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
||||
)
|
||||
|
||||
async def render_chat_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
) -> tuple[list[ConversationMessage], list[ProcessorInputs]] | ErrorResponse:
|
||||
) -> GenerateRequest | ErrorResponse:
|
||||
"""Validate the model and preprocess a chat completion request.
|
||||
|
||||
This is the authoritative implementation used directly by the
|
||||
@@ -96,7 +119,56 @@ class OpenAIServingRender:
|
||||
if error_check_ret is not None:
|
||||
logger.error("Error with model %s", error_check_ret)
|
||||
return error_check_ret
|
||||
return await self.render_chat(request)
|
||||
|
||||
if request.use_beam_search:
|
||||
return self.create_error_response(
|
||||
"Beam search is not supported by the render endpoint"
|
||||
)
|
||||
|
||||
result = await self.render_chat(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
|
||||
_, engine_prompts = result
|
||||
|
||||
if len(engine_prompts) != 1:
|
||||
return self.create_error_response(
|
||||
f"Expected exactly 1 engine prompt, got {len(engine_prompts)}"
|
||||
)
|
||||
|
||||
engine_prompt = engine_prompts[0]
|
||||
|
||||
prompt_components = extract_prompt_components(self.model_config, engine_prompt)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
token_ids = list(token_ids)
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_prompt)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_completion_tokens
|
||||
if request.max_completion_tokens is not None
|
||||
else request.max_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
)
|
||||
params = request.to_sampling_params(max_tokens, self.default_sampling_params)
|
||||
|
||||
request_id = f"chatcmpl-{random_uuid()}"
|
||||
|
||||
return GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
features=self._extract_mm_features(engine_prompt),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
)
|
||||
|
||||
async def render_chat(
|
||||
self,
|
||||
@@ -183,7 +255,7 @@ class OpenAIServingRender:
|
||||
async def render_completion_request(
|
||||
self,
|
||||
request: CompletionRequest,
|
||||
) -> list[ProcessorInputs] | ErrorResponse:
|
||||
) -> list[GenerateRequest] | ErrorResponse:
|
||||
"""Validate the model and preprocess a completion request.
|
||||
|
||||
This is the authoritative implementation used directly by the
|
||||
@@ -192,7 +264,48 @@ class OpenAIServingRender:
|
||||
error_check_ret = await self._check_model(request)
|
||||
if error_check_ret is not None:
|
||||
return error_check_ret
|
||||
return await self.render_completion(request)
|
||||
result = await self.render_completion(request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return result
|
||||
generate_requests: list[GenerateRequest] = []
|
||||
for engine_prompt in result:
|
||||
prompt_components = extract_prompt_components(
|
||||
self.model_config, engine_prompt
|
||||
)
|
||||
token_ids = prompt_components.token_ids
|
||||
if not token_ids:
|
||||
return self.create_error_response("No token_ids rendered")
|
||||
token_ids = list(token_ids)
|
||||
|
||||
input_length = extract_prompt_len(self.model_config, engine_prompt)
|
||||
max_tokens = get_max_tokens(
|
||||
self.model_config.max_model_len,
|
||||
request.max_tokens,
|
||||
input_length,
|
||||
self.default_sampling_params,
|
||||
self.override_max_tokens,
|
||||
)
|
||||
params = request.to_sampling_params(
|
||||
max_tokens, self.default_sampling_params
|
||||
)
|
||||
|
||||
request_id = f"cmpl-{random_uuid()}"
|
||||
|
||||
generate_requests.append(
|
||||
GenerateRequest(
|
||||
request_id=request_id,
|
||||
token_ids=token_ids,
|
||||
features=self._extract_mm_features(engine_prompt),
|
||||
sampling_params=params,
|
||||
model=request.model,
|
||||
stream=bool(request.stream),
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
)
|
||||
)
|
||||
|
||||
return generate_requests
|
||||
|
||||
async def render_completion(
|
||||
self,
|
||||
@@ -223,6 +336,33 @@ class OpenAIServingRender:
|
||||
|
||||
return engine_prompts
|
||||
|
||||
@staticmethod
|
||||
def _extract_mm_features(
|
||||
engine_prompt: ProcessorInputs,
|
||||
) -> MultiModalFeatures | None:
|
||||
"""Extract multimodal metadata from a rendered engine prompt.
|
||||
|
||||
Returns ``None`` for text-only prompts.
|
||||
"""
|
||||
if engine_prompt.get("type") != "multimodal":
|
||||
return None
|
||||
|
||||
# At this point engine_prompt is a MultiModalInputs TypedDict.
|
||||
mm_hashes: MultiModalHashes = engine_prompt["mm_hashes"] # type: ignore[typeddict-item]
|
||||
raw_placeholders: MultiModalPlaceholderDict = engine_prompt["mm_placeholders"] # type: ignore[typeddict-item]
|
||||
|
||||
mm_placeholders = {
|
||||
modality: [
|
||||
PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
|
||||
]
|
||||
for modality, ranges in raw_placeholders.items()
|
||||
}
|
||||
|
||||
return MultiModalFeatures(
|
||||
mm_hashes=mm_hashes,
|
||||
mm_placeholders=mm_placeholders,
|
||||
)
|
||||
|
||||
def _make_request_with_harmony(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
|
||||
@@ -35,6 +35,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
chat_template_content_format: ChatTemplateContentFormatOption,
|
||||
default_chat_template_kwargs: dict[str, Any] | None = None,
|
||||
trust_request_chat_template: bool = False,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
@@ -45,6 +46,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
|
||||
self.chat_template = chat_template
|
||||
self.chat_template_content_format: Final = chat_template_content_format
|
||||
self.default_chat_template_kwargs = default_chat_template_kwargs or {}
|
||||
self.trust_request_chat_template = trust_request_chat_template
|
||||
|
||||
async def create_tokenize(
|
||||
@@ -79,7 +81,7 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
request.messages,
|
||||
default_template=self.chat_template,
|
||||
default_template_content_format=self.chat_template_content_format,
|
||||
default_template_kwargs=None,
|
||||
default_template_kwargs=self.default_chat_template_kwargs,
|
||||
tool_dicts=tool_dicts,
|
||||
)
|
||||
else:
|
||||
@@ -98,8 +100,9 @@ class OpenAIServingTokenization(OpenAIServing):
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
if "prompt_token_ids" in engine_prompt:
|
||||
input_ids.extend(engine_prompt["prompt_token_ids"]) # type: ignore[typeddict-item]
|
||||
prompt_components = self._extract_prompt_components(engine_prompt)
|
||||
if prompt_components.token_ids is not None:
|
||||
input_ids.extend(prompt_components.token_ids)
|
||||
|
||||
token_strs = None
|
||||
if request.return_token_strs:
|
||||
|
||||
Reference in New Issue
Block a user