feat: Enable prompt_embeds Content Part Support in vLLM Chat Completions API (#40720)

Signed-off-by: Luis Robaina <luis@protopia.ai>
Signed-off-by: Luis Robaina 🚀 <luisfabian1545@gmail.com>
Signed-off-by: LuisRobaina <luis@protopia.ai>
Co-authored-by: Andrew Sansom <qthequartermasterman@gmail.com>
This commit is contained in:
Luis 🚀
2026-05-01 10:05:55 +08:00
committed by GitHub
co-authored by Andrew Sansom
parent 1adaa5056b
commit 14043dfecd
28 changed files with 2301 additions and 82 deletions
+201 -9
View File
@@ -11,7 +11,7 @@ from dataclasses import dataclass
from functools import cached_property, lru_cache, partial
from itertools import accumulate
from pathlib import Path
from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, TypeVar, cast
from typing import TYPE_CHECKING, Any, Final, Generic, Literal, TypeAlias, TypeVar, cast
from openai.types.chat import (
ChatCompletionAssistantMessageParam,
@@ -36,7 +36,7 @@ from PIL import Image
from pydantic import BaseModel, ConfigDict, TypeAdapter
# pydantic needs the TypedDict from typing_extensions
from typing_extensions import Required, TypedDict
from typing_extensions import Required, TypedDict, override
from vllm import envs
from vllm.config import ModelConfig
@@ -55,6 +55,10 @@ from vllm.multimodal.inputs import (
)
from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector
from vllm.multimodal.processing import BaseMultiModalProcessor
from vllm.renderers.embed_utils import (
safe_load_prompt_embeds,
safe_load_prompt_embeds_async,
)
from vllm.utils import random_uuid
from vllm.utils.collection_utils import is_list_of
from vllm.utils.import_utils import LazyLoader
@@ -98,9 +102,40 @@ MODALITY_PLACEHOLDERS_MAP = {
"image": "<##IMAGE##>",
"audio": "<##AUDIO##>",
"video": "<##VIDEO##>",
"prompt_embeds": "<##PROMPT_EMBEDS##>",
}
PROMPT_EMBEDS_PLACEHOLDER_TOKEN: Final[str] = "<prompt_embeds>"
"""The special token used as a placeholder for each embedding
position during chat template rendering.
Registered as an additional special token when `--enable-prompt-embeds` is set.
See `_ensure_prompt_embeds_placeholder_token` in `vllm/renderers/hf.py`.
"""
_REQUIRE_MM_PROCESSOR_ERROR: Final[str] = (
"Resolving modality {modality!r} requires a multimodal processor "
"but none is available."
)
_ENABLE_PROMPT_EMBEDS_ERROR: Final[str] = (
"You must set `--enable-prompt-embeds` to input `prompt_embeds`"
)
_PROMPT_EMBEDS_MISSING_DATA_ERROR: Final[str] = (
"prompt_embeds content part requires a non-empty `data` field "
"with base64-encoded tensor bytes."
)
_RESERVED_PLACEHOLDER_IN_TEXT_ERROR: Final[str] = (
"Text content may not contain the reserved placeholder {token!r}. "
"This placeholder is used internally to mark `prompt_embeds` splice "
"positions in the tokenized prompt."
)
class AudioURL(TypedDict, total=False):
url: Required[str]
"""
@@ -147,6 +182,17 @@ class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
"""
class ChatCompletionContentPartPromptEmbedsParam(TypedDict, total=False):
data: Required[str]
"""
Base64-encoded bytes of a serialized `torch.Tensor` of shape
`(num_tokens, hidden_size)`. The tensor's `dtype` and `hidden_size` must
match the model's input embedding layer.
"""
type: Required[Literal["prompt_embeds"]]
"""The type of the content part."""
class VideoURL(TypedDict, total=False):
url: Required[str]
"""
@@ -282,6 +328,7 @@ ChatCompletionContentPartParam: TypeAlias = (
| CustomChatCompletionContentSimpleImageParam
| ChatCompletionContentPartImageEmbedsParam
| ChatCompletionContentPartAudioEmbedsParam
| ChatCompletionContentPartPromptEmbedsParam
| CustomChatCompletionContentSimpleAudioParam
| CustomChatCompletionContentSimpleVideoParam
| CustomChatCompletionContentToolReferenceParam
@@ -367,7 +414,13 @@ ChatTemplateContentFormat = Literal["string", "openai"]
ModalityStr = Literal[
"image", "audio", "video", "image_embeds", "audio_embeds", "vision_chunk"
"image",
"audio",
"video",
"image_embeds",
"audio_embeds",
"vision_chunk",
"prompt_embeds",
]
_T = TypeVar("_T")
@@ -549,7 +602,17 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
An optional uuid can be added which serves as a unique identifier of the
media.
Note:
`prompt_embeds` bypass MM-processor validation because they are
pre-computed embeddings that do not go through any HF processor, encoder,
or model-specific placeholder logic. The corresponding placeholder string is
managed by the parser via `_add_placeholder`, so we return None here.
"""
if modality == "prompt_embeds":
self._items_by_modality["prompt_embeds"].append(item)
return None
input_modality = modality.replace("_embeds", "")
original_modality = modality
use_vision_chunk = (
@@ -660,17 +723,32 @@ def _resolve_vision_chunk_items(
def _resolve_items(
items_by_modality: dict[str, list[tuple[object, str | None]]],
mm_processor: BaseMultiModalProcessor,
mm_processor: BaseMultiModalProcessor | None,
modality_order: dict[str, list[str]],
) -> tuple[MultiModalDataDict, MultiModalUUIDDict]:
"""
Materialize the tracker's per-modality items into `mm_data` / `mm_uuids`.
Note:
`mm_processor` is `None` for text-only models (no registered HF
processor) whose only modality is `prompt_embeds`. Every other
modality requires a processor, enforced by the guard below.
"""
if "image" in items_by_modality and "image_embeds" in items_by_modality:
raise ValueError("Mixing raw image and embedding inputs is not allowed")
if "audio" in items_by_modality and "audio_embeds" in items_by_modality:
raise ValueError("Mixing raw audio and embedding inputs is not allowed")
# `prompt_embeds` bypasses HF MM processors. Every other modality requires one.
processor_modalities = items_by_modality.keys() - {"prompt_embeds"}
if processor_modalities and mm_processor is None:
raise RuntimeError(
_REQUIRE_MM_PROCESSOR_ERROR.format(modality=processor_modalities)
)
mm_data = {}
mm_uuids = {}
if "image_embeds" in items_by_modality:
assert mm_processor is not None
mm_data["image"] = _get_embeds_data(
"image",
[data for data, uuid in items_by_modality["image_embeds"]],
@@ -681,6 +759,7 @@ def _resolve_items(
mm_data["image"] = [data for data, uuid in items_by_modality["image"]]
mm_uuids["image"] = [uuid for data, uuid in items_by_modality["image"]]
if "audio_embeds" in items_by_modality:
assert mm_processor is not None
mm_data["audio"] = _get_embeds_data(
"audio",
[data for data, uuid in items_by_modality["audio_embeds"]],
@@ -694,6 +773,7 @@ def _resolve_items(
mm_data["video"] = [data for data, uuid in items_by_modality["video"]]
mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]]
if "vision_chunk" in items_by_modality:
assert mm_processor is not None
# Process vision_chunk items - extract from (data, modality) tuples
# and convert to VisionChunk types with proper UUID handling
processed_chunks, vision_chunk_uuids = _resolve_vision_chunk_items(
@@ -703,6 +783,10 @@ def _resolve_items(
)
mm_data["vision_chunk"] = processed_chunks
mm_uuids["vision_chunk"] = vision_chunk_uuids
if "prompt_embeds" in items_by_modality:
mm_data["prompt_embeds"] = [
data for data, _uuid in items_by_modality["prompt_embeds"]
]
return mm_data, mm_uuids
@@ -714,8 +798,16 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]
if not self._items_by_modality:
return None, None
# Text-only models (`is_multimodal_model=False`) with inputs of
# modality `prompt_embeds` have no MM processor since `prompt_embeds` are
# pre-computed and require no processing, so we pass `None`.
mm_processor = (
self.mm_processor if self._model_config.is_multimodal_model else None
)
return _resolve_items(
dict(self._items_by_modality), self.mm_processor, self._modality_order
dict(self._items_by_modality),
mm_processor,
self._modality_order,
)
def create_parser(
@@ -738,8 +830,13 @@ class AsyncMultiModalItemTracker(
for modality, coros in self._items_by_modality.items()
}
mm_processor = (
self.mm_processor if self._model_config.is_multimodal_model else None
)
return _resolve_items(
resolved_items_by_modality, self.mm_processor, self._modality_order
resolved_items_by_modality,
mm_processor,
self._modality_order,
)
def create_parser(
@@ -758,10 +855,16 @@ class BaseMultiModalContentParser(ABC):
# general MM placeholder:
# {
# "<##IMAGE##>": ["<image>", "<image>", "<image>"],
# "<##AUDIO##>": ["<audio>", "<audio>"]
# "<##AUDIO##>": ["<audio>", "<audio>"],
# "<##PROMPT_EMBEDS##>": ["<prompt_embeds>", "<prompt_embeds>"]
# }
self._placeholder_storage: dict[str, list] = defaultdict(list)
@property
@abstractmethod
def model_config(self) -> ModelConfig:
raise NotImplementedError
def _add_placeholder(self, modality: ModalityStr, placeholder: str | None):
mod_placeholder = MODALITY_PLACEHOLDERS_MAP[modality]
if placeholder:
@@ -806,6 +909,10 @@ class BaseMultiModalContentParser(ABC):
) -> None:
raise NotImplementedError
@abstractmethod
def parse_prompt_embeds(self, data: str) -> None:
raise NotImplementedError
@abstractmethod
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
raise NotImplementedError
@@ -834,6 +941,21 @@ class MultiModalContentParser(BaseMultiModalContentParser):
def model_config(self) -> ModelConfig:
return self._tracker.model_config
@override
def parse_prompt_embeds(self, data: str) -> None:
"""Decode a base64 prompt embeds tensor and store it in the tracker.
Emits a single `PROMPT_EMBEDS_PLACEHOLDER_TOKEN` sentinel per
content part. The renderer later expands each sentinel to a span of
`tensor.shape[0]` placeholder tokens after tokenization.
"""
if not self.model_config.enable_prompt_embeds:
raise ValueError(_ENABLE_PROMPT_EMBEDS_ERROR)
tensor = safe_load_prompt_embeds(self.model_config, data.encode())
self._tracker.add("prompt_embeds", (tensor, None))
self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
image = self._connector.fetch_image(image_url) if image_url else None
@@ -958,6 +1080,29 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
def model_config(self) -> ModelConfig:
return self._tracker.model_config
@override
def parse_prompt_embeds(self, data: str) -> None:
"""Schedule async prompt embeds decode and store the coroutine in the tracker.
Like the sync variant, emits a single sentinel `PROMPT_EMBEDS_PLACEHOLDER_TOKEN`
per content part. Unlike the sync variant, the tensor decode is deferred to a
thread-pool executor via `safe_load_prompt_embeds_async`.
"""
if not self.model_config.enable_prompt_embeds:
raise ValueError(_ENABLE_PROMPT_EMBEDS_ERROR)
coro = self._load_prompt_embeds_async(data.encode())
self._tracker.add("prompt_embeds", coro)
self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
async def _load_prompt_embeds_async(
self, data_bytes: bytes
) -> tuple[torch.Tensor, None]:
# Second tuple slot fills the tracker's generic `(item, uuid | None)`
# contract. prompt_embeds has no UUID concept, so it's always `None`.
tensor = await safe_load_prompt_embeds_async(self.model_config, data_bytes)
return tensor, None
async def _image_with_uuid_async(self, image_url: str | None, uuid: str | None):
image = (
await self._connector.fetch_image_async(image_url) if image_url else None
@@ -1269,6 +1414,7 @@ def _get_full_multimodal_text_prompt(
_TextParser = partial(cast, ChatCompletionContentPartTextParam)
_ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)
_AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam)
_PromptEmbedsParser = partial(cast, ChatCompletionContentPartPromptEmbedsParam)
_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
_PILImageParser = partial(cast, CustomChatCompletionContentPILImageParam)
@@ -1294,6 +1440,7 @@ MM_PARSER_MAP: dict[
"image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
"image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None),
"audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None),
"prompt_embeds": lambda part: _PromptEmbedsParser(part).get("data", None),
"image_pil": lambda part: _PILImageParser(part).get("image_pil", None),
"audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),
"input_audio": lambda part: _InputAudioParser(part).get("input_audio", None),
@@ -1372,6 +1519,11 @@ def _parse_chat_message_content_mm_part(
)
audio_embeds = audio_params.get("audio_embeds", None)
return "audio_embeds", audio_embeds
if "prompt_embeds" in part:
prompt_embeds_params = cast( # type: ignore[assignment]
ChatCompletionContentPartPromptEmbedsParam, part
)
return "prompt_embeds", prompt_embeds_params.get("data", None)
if "audio_url" in part:
audio_params = cast( # type: ignore[assignment]
CustomChatCompletionContentSimpleAudioParam, part
@@ -1455,6 +1607,24 @@ def _parse_chat_message_content_parts(
return [ConversationMessage(role=role, content=text_prompt)]
def _reject_reserved_placeholder_in_text(text: str, model_config: ModelConfig) -> None:
"""Reject user-supplied text parts that contains the reserved `prompt_embeds`
placeholder sentinel.
When the server accepts `prompt_embeds`, the placeholder token is
registered as a single unsplittable special token on the tokenizer. Any
user text that happens to contain the literal sequence would tokenize to
the same ID and be mistaken for a splice point by the renderer, letting a
caller move or inject splice positions via plain text content.
"""
if model_config.enable_prompt_embeds and PROMPT_EMBEDS_PLACEHOLDER_TOKEN in text:
raise ValueError(
_RESERVED_PLACEHOLDER_IN_TEXT_ERROR.format(
token=PROMPT_EMBEDS_PLACEHOLDER_TOKEN
)
)
def _parse_chat_message_content_part(
part: ChatCompletionContentPartParam,
mm_parser: BaseMultiModalContentParser,
@@ -1470,6 +1640,7 @@ def _parse_chat_message_content_part(
with multimodal placeholders.
"""
if isinstance(part, str): # Handle plain text parts
_reject_reserved_placeholder_in_text(part, mm_parser.model_config)
if wrap_dicts:
return {"type": "text", "text": part}
return part
@@ -1488,6 +1659,7 @@ def _parse_chat_message_content_part(
if part_type in ("text", "input_text", "output_text", "refusal", "thinking"):
str_content = cast(str, content)
_reject_reserved_placeholder_in_text(str_content, mm_parser.model_config)
if wrap_dicts:
return {"type": "text", "text": str_content}
else:
@@ -1516,6 +1688,11 @@ def _parse_chat_message_content_part(
content = cast(str | dict[str, str], content) if content is not None else None
mm_parser.parse_audio_embeds(content, uuid)
modality = "audio"
elif part_type == "prompt_embeds":
if not content:
raise ValueError(_PROMPT_EMBEDS_MISSING_DATA_ERROR)
mm_parser.parse_prompt_embeds(cast(str, content))
modality = "prompt_embeds"
elif part_type == "audio_url":
str_content = cast(str, content)
mm_parser.parse_audio(str_content, uuid)
@@ -1544,7 +1721,18 @@ def _parse_chat_message_content_part(
)
if wrap_dicts:
if modality == "prompt_embeds":
# Chat templates don't know about the "prompt_embeds" modality,
# emit the single sentinel token as text so the template renders
# it inline. The renderer later expands it to N tokens post-tokenize.
return {"type": "text", "text": PROMPT_EMBEDS_PLACEHOLDER_TOKEN}
return {"type": modality}
if modality == "prompt_embeds":
# Emit the renderer token inline regardless of `interleave_strings`,
# prompt_embeds are spliced at the token offset so position matters.
# Falling back to front-padding via `missing_placeholders` would
# reorder them relative to surrounding text.
return PROMPT_EMBEDS_PLACEHOLDER_TOKEN
return MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None
@@ -1668,7 +1856,10 @@ def parse_chat_messages(
MultiModalUUIDDict | None,
]:
conversation: list[ConversationMessage] = []
mm_tracker = MultiModalItemTracker(model_config, media_io_kwargs=media_io_kwargs)
mm_tracker = MultiModalItemTracker(
model_config,
media_io_kwargs=media_io_kwargs,
)
for msg in messages:
sub_messages = _parse_chat_message_content(
@@ -1705,7 +1896,8 @@ async def parse_chat_messages_async(
]:
conversation: list[ConversationMessage] = []
mm_tracker = AsyncMultiModalItemTracker(
model_config, media_io_kwargs=media_io_kwargs
model_config,
media_io_kwargs=media_io_kwargs,
)
for msg in messages:
+4 -1
View File
@@ -926,7 +926,10 @@ class LLM:
add_generation_prompt=add_generation_prompt,
continue_final_message=continue_final_message,
tools=tools,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
tokenize=(
is_mistral_tokenizer(renderer.tokenizer)
or self.model_config.enable_prompt_embeds
),
),
),
mm_processor_kwargs=mm_processor_kwargs,
+4 -1
View File
@@ -541,7 +541,10 @@ class OpenAIServingRender:
default_template_kwargs,
dict(
tools=tool_dicts,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
tokenize=(
is_mistral_tokenizer(renderer.tokenizer)
or self.model_config.enable_prompt_embeds
),
),
)
+19
View File
@@ -71,12 +71,27 @@ class EmbedsInput(_InputOptions):
prompt: NotRequired[str]
"""The prompt text corresponding to the token IDs, if available."""
prompt_token_ids: NotRequired[list[int]]
"""Token IDs of the rendered prompt. Only set for mixed-mode inputs
(chat completion with `prompt_embeds` content parts). When present,
`is_token_ids` MUST also be present and have the same length.
For pure-embeds inputs this field is absent."""
is_token_ids: NotRequired[list[bool]]
"""Per-position mask for mixed-mode inputs. `True` means the position
is a real token ID (use the model's embedding layer); `False` means
the position uses a pre-computed embedding row from `prompt_embeds`.
Length MUST equal `len(prompt_token_ids)`.
For pure-embeds inputs this field is absent."""
def embeds_input(
prompt_embeds: "torch.Tensor",
*,
prompt: str | None = None,
cache_salt: str | None = None,
prompt_token_ids: list[int] | None = None,
is_token_ids: list[bool] | None = None,
) -> EmbedsInput:
"""
Construct [`EmbedsInput`][vllm.inputs.engine.EmbedsInput]
@@ -88,6 +103,10 @@ def embeds_input(
inputs["prompt"] = prompt
if cache_salt is not None:
inputs["cache_salt"] = cache_salt
if prompt_token_ids is not None:
inputs["prompt_token_ids"] = prompt_token_ids
if is_token_ids is not None:
inputs["is_token_ids"] = is_token_ids
return inputs
+11
View File
@@ -125,6 +125,17 @@ class EmbedsPrompt(_PromptOptions):
prompt: NotRequired[str]
"""The prompt text corresponding to the token embeddings, if available."""
prompt_token_ids: NotRequired[list[int]]
"""Token IDs for mixed-mode inputs (chat completion with
`prompt_embeds` content parts). The tokens at positions where
`prompt_is_token_ids` is `False` are placeholder tokens that
get replaced by entries from `prompt_embeds` in the forward pass."""
prompt_is_token_ids: NotRequired[list[bool]]
"""Per-position mask, `True` uses the real token ID, `False` uses
the corresponding entry from `prompt_embeds`.
Must be the same length as `prompt_token_ids` when both are set."""
DecoderOnlyPrompt: TypeAlias = (
str | TextPrompt | list[int] | TokensPrompt | EmbedsPrompt
+2
View File
@@ -769,6 +769,8 @@ class BaseRenderer(ABC, Generic[_T]):
return embeds_input(
prompt_embeds=prompt_embeds,
cache_salt=prompt.get("cache_salt"),
prompt_token_ids=prompt.get("prompt_token_ids"),
is_token_ids=prompt.get("prompt_is_token_ids"),
)
async def _process_tokens_async(
+45 -6
View File
@@ -7,6 +7,7 @@ import pybase64
import torch
from vllm.exceptions import VLLMValidationError
from vllm.utils.async_utils import make_async
if TYPE_CHECKING:
from vllm.config import ModelConfig
@@ -30,15 +31,53 @@ def safe_load_prompt_embeds(
weights_only=True,
map_location=torch.device("cpu"),
)
assert isinstance(tensor, torch.Tensor) and tensor.dtype in (
torch.float32,
torch.bfloat16,
torch.float16,
)
if not isinstance(tensor, torch.Tensor):
raise VLLMValidationError(
"`prompt_embeds` payload did not deserialize to a torch.Tensor.",
parameter="prompt_embeds",
)
tensor = tensor.to_dense()
if tensor.dim() > 2:
tensor = tensor.squeeze(0)
assert tensor.dim() == 2
if tensor.dim() != 2:
raise VLLMValidationError(
"`prompt_embeds` must be a 2D tensor of shape "
f"(num_tokens, hidden_size); got shape {tuple(tensor.shape)}.",
parameter="prompt_embeds",
)
# Pin each tensor to the model's hidden_size. Validating here
# also transitively guarantees cross-tensor consistency for requests that
# include multiple `prompt_embeds` parts, which is required by downstream
# concatenation in `_build_mixed_prompt_embeds`.
expected_hidden_size = model_config.get_hidden_size()
if tensor.shape[1] != expected_hidden_size:
raise VLLMValidationError(
f"`prompt_embeds` hidden_size {tensor.shape[1]} does not match "
f"the model's hidden_size {expected_hidden_size}.",
parameter="prompt_embeds",
)
# Cast to the model's dtype so API clients don't need to know the server's
# `--dtype` setting ahead of time. Only floating-point source dtypes are
# allowed. integer / bool / complex inputs almost certainly indicate caller
# error (e.g. quantized payloads, wrong tensor), and a silent `.to()`
# could hide a real mistake.
expected_dtype = model_config.dtype
if tensor.dtype != expected_dtype:
if not tensor.is_floating_point():
raise VLLMValidationError(
f"`prompt_embeds` dtype {tensor.dtype} is not a floating-point "
f"type, cannot safely cast to the model's dtype {expected_dtype}.",
parameter="prompt_embeds",
)
tensor = tensor.to(expected_dtype)
return tensor
safe_load_prompt_embeds_async = make_async(safe_load_prompt_embeds)
"""Async variant of `safe_load_prompt_embeds` that defers the decode to a
thread-pool executor, so the asyncio event loop is not blocked by the base64
decode + `torch.load` work."""
+440 -18
View File
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import inspect
import itertools
import weakref
from collections import defaultdict, deque
from collections.abc import Set
from collections.abc import Sequence
from functools import lru_cache
from typing import Any, Literal, cast, overload
from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
import jinja2
import jinja2.ext
@@ -13,20 +16,32 @@ import jinja2.meta
import jinja2.nodes
import jinja2.parser
import jinja2.sandbox
import torch
from typing_extensions import override
from vllm.config import ModelConfig, VllmConfig
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormat,
ChatTemplateContentFormatOption,
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
ChatTemplateResolutionError,
ConversationMessage,
load_chat_template,
parse_chat_messages,
parse_chat_messages_async,
)
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
from vllm.inputs import EmbedsPrompt
from vllm.inputs.engine import MultiModalInput
from vllm.logger import init_logger
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.inputs import (
MultiModalFieldElem,
MultiModalKwargsItem,
MultiModalKwargsItems,
MultiModalSharedField,
PlaceholderRange,
)
from vllm.multimodal.processing.processor import (
PromptReplacement,
apply_token_matches,
find_mm_placeholders,
)
from vllm.tokenizers.hf import HfTokenizer
from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
from vllm.transformers_utils.processor import cached_get_processor
@@ -34,13 +49,166 @@ from vllm.utils.async_utils import make_async
from vllm.utils.func_utils import supports_kw
from .base import BaseRenderer
from .inputs import DictPrompt
from .inputs.preprocess import parse_dec_only_prompt
from .params import ChatParams
if TYPE_CHECKING:
from collections.abc import Set
from vllm.config import ModelConfig, VllmConfig
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormat,
ChatTemplateContentFormatOption,
ConversationMessage,
)
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict, TokensPrompt
from vllm.inputs.engine import TokensInput
from vllm.multimodal.processing.processor import (
MultiModalPromptUpdates,
ResolvedPromptUpdate,
)
from .inputs import DictPrompt
from .params import ChatParams
logger = init_logger(__name__)
# Cache of `tokenizer -> prompt_embeds placeholder token ID`. Keyed by the
# tokenizer object (not `id(tokenizer)`) so a fresh tokenizer landing at a
# recycled memory address can't pick up a stale tid. Entries evict atomically
# with the tokenizer's garbage-collection.
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_CACHE: Final[
weakref.WeakKeyDictionary[HfTokenizer, int]
] = weakref.WeakKeyDictionary()
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_ERROR: Final[str] = (
"Expected {token!r} to tokenize to exactly 1 token, got {num_ids} ({ids!r})."
)
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR: Final[str] = (
"Expected {expected} prompt_embeds placeholder spans in the "
"tokenized prompt, found {actual}."
)
_MISSING_PROMPT_TOKEN_IDS_ERROR: Final[str] = (
"Expected prompt_token_ids in rendered prompt when prompt_embeds "
"are present. This indicates the chat template was invoked with "
"tokenize=False."
)
_TOKENIZE_OVERRIDE_WARNING: Final[str] = (
"Overriding `tokenize=False` to `True` because `prompt_embeds` "
"post-processing requires tokenized IDs."
)
def _ensure_prompt_embeds_placeholder_token(tokenizer: HfTokenizer) -> int:
"""Register `PROMPT_EMBEDS_PLACEHOLDER_TOKEN` as a special token and return
its token ID."""
cached = _PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_CACHE.get(tokenizer)
if cached is not None:
return cached
tokenizer.add_special_tokens(
{"additional_special_tokens": [PROMPT_EMBEDS_PLACEHOLDER_TOKEN]}
)
ids = tokenizer.encode(PROMPT_EMBEDS_PLACEHOLDER_TOKEN, add_special_tokens=False)
if len(ids) != 1:
raise RuntimeError(
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_ERROR.format(
token=PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
num_ids=len(ids),
ids=ids,
)
)
token_id = ids[0]
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_CACHE[tokenizer] = token_id
return token_id
def _build_prompt_embeds_updates(
prompt_embeds_tensors: Sequence[torch.Tensor],
placeholder_token_id: int,
) -> MultiModalPromptUpdates:
"""Build `MultiModalPromptUpdates` for `prompt_embeds` expansion.
Each tensor produces a `PromptReplacement` that maps
`[placeholder_token_id]` -> `[placeholder_token_id] x N`
(where `N = tensor.shape[0]`).
"""
updates: list[Sequence[ResolvedPromptUpdate]] = []
for i, tensor in enumerate(prompt_embeds_tensors):
update = PromptReplacement(
modality="prompt_embeds",
target=[placeholder_token_id],
replacement=[placeholder_token_id] * tensor.shape[0],
)
updates.append([update.resolve(item_idx=i)])
return {"prompt_embeds": updates}
def _expand_prompt_embeds_placeholders(
token_ids: list[int],
mm_prompt_updates: MultiModalPromptUpdates,
) -> list[int]:
"""Expand each 1-token `prompt_embeds` sentinel into an N-token span.
Uses `apply_token_matches`. Each single placeholder token in
`token_ids` is replaced with a consecutive span of
`tensor.shape[0]` copies, following tensors in order.
"""
expanded, _ = apply_token_matches(token_ids, mm_prompt_updates, tokenizer=None)
return expanded
def _build_prompt_embeds_positions(
token_ids: list[int],
num_tensors: int,
mm_prompt_updates: MultiModalPromptUpdates,
) -> list[tuple[int, int]]:
"""Locate each prompt_embeds placeholder span in `token_ids`.
Expects `token_ids` to already contain expanded N-token spans.
Returns `[(start_idx, length), ...]` aligned with the tensors.
"""
placeholders = find_mm_placeholders(
prompt=token_ids,
mm_prompt_updates=mm_prompt_updates,
tokenizer=None,
)
features = placeholders.get("prompt_embeds", [])
if len(features) != num_tensors:
raise ValueError(
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR.format(
expected=num_tensors,
actual=len(features),
)
)
return [(f.start_idx, f.length) for f in features]
def _build_mixed_prompt_embeds(
token_ids: list[int],
prompt_embeds_tensors: Sequence[torch.Tensor],
positions: list[tuple[int, int]],
) -> tuple[torch.Tensor, list[bool]]:
"""Build the full-length `prompt_embeds` tensor and the `is_token_ids`
mask aligned to `token_ids`."""
total_len = len(token_ids)
hidden_size = prompt_embeds_tensors[0].shape[1]
dtype = prompt_embeds_tensors[0].dtype
full_embeds = torch.zeros(total_len, hidden_size, dtype=dtype)
is_token_ids = torch.ones(total_len, dtype=torch.bool)
for (start, length), tensor in zip(positions, prompt_embeds_tensors, strict=True):
full_embeds[start : start + length] = tensor
is_token_ids[start : start + length] = False
return full_embeds, is_token_ids.tolist()
_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], str | None]()
"""
Used in `_try_get_processor_chat_template` to avoid calling
@@ -98,7 +266,7 @@ def resolve_chat_template(
chat_template: str | None,
tools: list[dict[str, Any]] | None,
*,
model_config: "ModelConfig",
model_config: ModelConfig,
) -> str | None:
# 1st priority: The given chat template
if chat_template is not None:
@@ -281,7 +449,7 @@ def _resolve_chat_template_content_format(
tools: list[dict[str, Any]] | None,
tokenizer: HfTokenizer,
*,
model_config: "ModelConfig",
model_config: ModelConfig,
) -> ChatTemplateContentFormat:
resolved_chat_template = resolve_chat_template(
tokenizer,
@@ -335,7 +503,7 @@ def resolve_chat_template_content_format(
given_format: ChatTemplateContentFormatOption,
tokenizer: HfTokenizer,
*,
model_config: "ModelConfig",
model_config: ModelConfig,
) -> ChatTemplateContentFormat:
if given_format != "auto":
return given_format
@@ -437,7 +605,7 @@ def resolve_chat_template_kwargs(
@overload
def safe_apply_chat_template(
model_config: "ModelConfig",
model_config: ModelConfig,
tokenizer: HfTokenizer,
conversation: list[ConversationMessage],
*,
@@ -448,7 +616,7 @@ def safe_apply_chat_template(
) -> list[int]: ...
@overload
def safe_apply_chat_template(
model_config: "ModelConfig",
model_config: ModelConfig,
tokenizer: HfTokenizer,
conversation: list[ConversationMessage],
*,
@@ -458,7 +626,7 @@ def safe_apply_chat_template(
**kwargs,
) -> str: ...
def safe_apply_chat_template(
model_config: "ModelConfig",
model_config: ModelConfig,
tokenizer: HfTokenizer,
conversation: list[ConversationMessage],
*,
@@ -486,6 +654,14 @@ def safe_apply_chat_template(
chat_template_kwargs=kwargs,
)
# transformers v5 changed the default of `return_dict` to True, which
# makes `apply_chat_template(tokenize=True)` return a `BatchEncoding`
# instead of `list[int]`. Force `return_dict=False` so downstream code
# that expects a flat token list (e.g. `parse_dec_only_prompt`) works
# consistently across v4 and v5.
if tokenize and "return_dict" not in resolved_kwargs:
resolved_kwargs["return_dict"] = False
try:
return tokenizer.apply_chat_template(
conversation=conversation, # type: ignore[arg-type]
@@ -627,6 +803,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
model_config = self.model_config
tokenizer = self.get_tokenizer()
prompt_embeds_placeholder_token_id: int | None = None
if model_config.enable_prompt_embeds:
prompt_embeds_placeholder_token_id = (
_ensure_prompt_embeds_placeholder_token(tokenizer)
)
conversation, mm_data, mm_uuids = parse_chat_messages(
messages,
model_config,
@@ -641,11 +823,30 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
mm_processor_kwargs=params.mm_processor_kwargs,
)
# prompt_embeds tensors are carried by the tracker through mm_data,
# but they must NOT be fed to the MM processor (which would reject
# the unknown key). Extract them here.
prompt_embeds_tensors: list[torch.Tensor] | None = None
if mm_data is not None and "prompt_embeds" in mm_data:
prompt_embeds_tensors = list(
cast(Sequence[torch.Tensor], mm_data["prompt_embeds"])
)
mm_data = {k: v for k, v in mm_data.items() if k != "prompt_embeds"}
if not mm_data:
mm_data = None
chat_template_kwargs = params.get_apply_chat_template_kwargs()
if prompt_embeds_tensors:
# prompt_embeds post-processing requires prompt_token_ids.
if chat_template_kwargs.get("tokenize") is False:
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
chat_template_kwargs["tokenize"] = True
prompt_raw = safe_apply_chat_template(
model_config,
tokenizer,
conversation,
**params.get_apply_chat_template_kwargs(),
**chat_template_kwargs,
)
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
@@ -671,6 +872,29 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
)
prompt = parse_dec_only_prompt(prompt_raw)
# When `prompt_embeds` is mixed with other modality data,
# `_process_tokens` runs `_process_multimodal` first (expanding
# `<|AUDIO|>` / `<|IMAGE|>` placeholders) and then
# `_apply_prompt_embeds_to_engine_input` augments the result.
# Stash the tensors and placeholder ID for that override to consume.
if prompt_embeds_tensors and mm_data:
assert prompt_embeds_placeholder_token_id is not None
cast(dict, prompt)["_prompt_embeds"] = (
prompt_embeds_tensors,
prompt_embeds_placeholder_token_id,
)
if params.mm_processor_kwargs:
cast(dict, prompt)["mm_processor_kwargs"] = params.mm_processor_kwargs
elif prompt_embeds_tensors:
# Pure mode: no other MM data, mutate prompt to EmbedsPrompt shape.
assert prompt_embeds_placeholder_token_id is not None
self._apply_prompt_embeds_to_prompt(
prompt,
prompt_embeds_tensors,
prompt_embeds_placeholder_token_id,
)
if mm_data is not None:
prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
@@ -686,6 +910,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
model_config = self.model_config
tokenizer = self.get_tokenizer()
prompt_embeds_placeholder_token_id: int | None = None
if model_config.enable_prompt_embeds:
prompt_embeds_placeholder_token_id = (
_ensure_prompt_embeds_placeholder_token(tokenizer)
)
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
messages,
model_config,
@@ -700,11 +930,27 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
mm_processor_kwargs=params.mm_processor_kwargs,
)
prompt_embeds_tensors: list[torch.Tensor] | None = None
if mm_data is not None and "prompt_embeds" in mm_data:
prompt_embeds_tensors = list(
cast(Sequence[torch.Tensor], mm_data["prompt_embeds"])
)
mm_data = {k: v for k, v in mm_data.items() if k != "prompt_embeds"}
if not mm_data:
mm_data = None
chat_template_kwargs = params.get_apply_chat_template_kwargs()
if prompt_embeds_tensors:
# prompt_embeds post-processing requires prompt_token_ids.
if chat_template_kwargs.get("tokenize") is False:
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
chat_template_kwargs["tokenize"] = True
prompt_raw = await self._apply_chat_template_async(
model_config,
tokenizer,
conversation,
**params.get_apply_chat_template_kwargs(),
**chat_template_kwargs,
)
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
@@ -728,9 +974,185 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
)
prompt = parse_dec_only_prompt(prompt_raw)
# See `render_messages` for the rationale.
if prompt_embeds_tensors and mm_data:
assert prompt_embeds_placeholder_token_id is not None
cast(dict, prompt)["_prompt_embeds"] = (
prompt_embeds_tensors,
prompt_embeds_placeholder_token_id,
)
if params.mm_processor_kwargs:
cast(dict, prompt)["mm_processor_kwargs"] = params.mm_processor_kwargs
elif prompt_embeds_tensors:
assert prompt_embeds_placeholder_token_id is not None
self._apply_prompt_embeds_to_prompt(
prompt,
prompt_embeds_tensors,
prompt_embeds_placeholder_token_id,
)
if mm_data is not None:
prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
prompt["multi_modal_uuids"] = mm_uuids
return conversation, prompt
@override
def _process_tokens(
self,
prompt: TokensPrompt,
*,
skip_mm_cache: bool = False,
) -> TokensInput | MultiModalInput:
"""Pre-expand `prompt_embeds` sentinels before delegating to the MM
processor, then attach `prompt_embeds` modality data to the result.
Mixed mode only: the `_prompt_embeds` stash is set by
`render_messages` when `prompt_embeds` co-exist with other MM data
(images, audio, …). We expand each 1-token sentinel to an N-token
span *before* calling `super()._process_tokens()` so the MM
processor records all placeholder offsets in the final (post-expansion)
coordinate space, no offset shifting needed afterwards.
"""
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
if prompt_embeds_info is not None:
tensors, placeholder_token_id = prompt_embeds_info
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
list(prompt["prompt_token_ids"]), mm_updates
)
engine_input = super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
if prompt_embeds_info is not None:
tensors, _ = prompt_embeds_info
self._apply_prompt_embeds_to_engine_input(
cast(MultiModalInput, engine_input),
tensors,
mm_updates,
)
return engine_input
@override
async def _process_tokens_async(
self,
prompt: TokensPrompt,
*,
skip_mm_cache: bool = False,
) -> TokensInput | MultiModalInput:
"""Async equivalent of `_process_tokens`."""
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
if prompt_embeds_info is not None:
tensors, placeholder_token_id = prompt_embeds_info
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
list(prompt["prompt_token_ids"]), mm_updates
)
engine_input = await super()._process_tokens_async(
prompt, skip_mm_cache=skip_mm_cache
)
if prompt_embeds_info is not None:
tensors, _ = prompt_embeds_info
self._apply_prompt_embeds_to_engine_input(
cast(MultiModalInput, engine_input),
tensors,
mm_updates,
)
return engine_input
@staticmethod
def _apply_prompt_embeds_to_prompt(
prompt: DictPrompt,
prompt_embeds_tensors: list[torch.Tensor],
placeholder_token_id: int,
) -> None:
"""Mutate `prompt` from `TokensPrompt` to `EmbedsPrompt` shape.
Pure `prompt_embeds` path only (no other MM modalities). Expands
each `<prompt_embeds>` sentinel token into an N-token span and builds
the full-length `prompt_embeds` tensor + `prompt_is_token_ids` mask
that the engine's `enable_prompt_embeds` worker branch consumes.
"""
token_ids = cast(list[int] | None, prompt.get("prompt_token_ids"))
if token_ids is None:
raise RuntimeError(_MISSING_PROMPT_TOKEN_IDS_ERROR)
embeds_orig_positions: list[int] = [
i for i, tok in enumerate(token_ids) if tok == placeholder_token_id
]
if len(embeds_orig_positions) != len(prompt_embeds_tensors):
raise ValueError(
f"Expected {len(prompt_embeds_tensors)} prompt_embeds "
f"placeholder tokens in the rendered prompt, found "
f"{len(embeds_orig_positions)}."
)
mm_updates = _build_prompt_embeds_updates(
prompt_embeds_tensors, placeholder_token_id
)
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
positions = _build_prompt_embeds_positions(
expanded, len(prompt_embeds_tensors), mm_updates
)
embeds_prompt = cast(EmbedsPrompt, prompt)
embeds_prompt["prompt_token_ids"] = expanded
full_embeds, is_token_ids_mask = _build_mixed_prompt_embeds(
expanded, prompt_embeds_tensors, positions
)
embeds_prompt["prompt_embeds"] = full_embeds
embeds_prompt["prompt_is_token_ids"] = is_token_ids_mask
@staticmethod
def _apply_prompt_embeds_to_engine_input(
engine_input: MultiModalInput,
prompt_embeds_tensors: list[torch.Tensor],
mm_updates: MultiModalPromptUpdates,
) -> None:
"""Augment `engine_input` in-place with a `prompt_embeds` modality.
Mixed mode: called after `_process_multimodal` has already run on the
pre-expanded token IDs (expansion was done in `_process_tokens` before
calling `super()`). Locates the already-expanded `prompt_embeds` spans
and adds `prompt_embeds` entries to `mm_kwargs`, `mm_hashes`, and
`mm_placeholders`.
"""
# token_ids already contain the pre-expanded N-token spans.
token_ids = list(engine_input["prompt_token_ids"])
positions = _build_prompt_embeds_positions(
token_ids, len(prompt_embeds_tensors), mm_updates
)
pe_kwargs_items: list[MultiModalKwargsItem] = []
pe_hashes: list[str] = []
pe_placeholders: list[PlaceholderRange] = []
for tensor, (start, length) in zip(
prompt_embeds_tensors, positions, strict=True
):
pe_kwargs_items.append(
MultiModalKwargsItem(
{
"embedding": MultiModalFieldElem(
data=tensor,
field=MultiModalSharedField(batch_size=1),
)
}
)
)
pe_hashes.append(MultiModalHasher.hash_kwargs(prompt_embeds=tensor))
# `is_embed=None` matches the existing image_embeds-style
# "no encoder, just splice the tensor directly" semantics.
pe_placeholders.append(
PlaceholderRange(offset=start, length=length, is_embed=None)
)
cast(
MultiModalKwargsItems[MultiModalKwargsItem | None],
engine_input["mm_kwargs"],
)["prompt_embeds"] = pe_kwargs_items
engine_input["mm_hashes"] = {
**engine_input["mm_hashes"],
"prompt_embeds": pe_hashes,
}
cast(dict, engine_input["mm_placeholders"])["prompt_embeds"] = pe_placeholders
+2
View File
@@ -38,6 +38,7 @@ class NewRequestData:
num_computed_tokens: int
lora_request: LoRARequest | None
prompt_embeds: "torch.Tensor | None" = None
prompt_is_token_ids: list[bool] | None = None
# Only used for v2 model runner.
prefill_token_ids: list[int] | None = None
@@ -59,6 +60,7 @@ class NewRequestData:
num_computed_tokens=request.num_computed_tokens,
lora_request=request.lora_request,
prompt_embeds=request.prompt_embeds,
prompt_is_token_ids=request.prompt_is_token_ids,
prefill_token_ids=prefill_token_ids,
)
+6
View File
@@ -94,6 +94,12 @@ class EngineCoreRequest(
data_parallel_rank: int | None
prompt_embeds: torch.Tensor | None = None
# Per-position mask for mixed-mode inputs (e.g chat completion with
# prompt_embeds content parts). `True` means the position is a real
# token ID; `False` means the position uses a pre-computed entry from
# `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
prompt_is_token_ids: list[bool] | None = None
# Index of the client, used to ensure outputs are sent back to the same
# client for this request when scaling out the front-end.
client_index: int = 0
+4 -1
View File
@@ -292,11 +292,13 @@ class InputProcessor:
# Mypy can be conservative for TypedDict unions; normalize access.
if decoder_inputs["type"] == "embeds":
prompt_token_ids = None
prompt_embeds = decoder_inputs["prompt_embeds"]
prompt_token_ids = decoder_inputs.get("prompt_token_ids")
prompt_is_token_ids = decoder_inputs.get("is_token_ids")
else:
prompt_token_ids = decoder_inputs["prompt_token_ids"]
prompt_embeds = None
prompt_is_token_ids = None
sampling_params = None
pooling_params = None
@@ -361,6 +363,7 @@ class InputProcessor:
request_id=request_id,
prompt_token_ids=prompt_token_ids,
prompt_embeds=prompt_embeds,
prompt_is_token_ids=prompt_is_token_ids,
mm_features=mm_features,
sampling_params=sampling_params,
pooling_params=pooling_params,
+6
View File
@@ -66,6 +66,7 @@ class Request:
client_index: int = 0,
arrival_time: float | None = None,
prompt_embeds: torch.Tensor | None = None,
prompt_is_token_ids: list[bool] | None = None,
mm_features: list[MultiModalFeatureSpec] | None = None,
lora_request: "LoRARequest | None" = None,
cache_salt: str | None = None,
@@ -114,6 +115,10 @@ class Request:
self.prompt_token_ids = prompt_token_ids
self.prompt_embeds = prompt_embeds
# Per-position mask used in mixed-mode (chat completion with
# prompt_embeds). `None` except when both `prompt_token_ids` and
# `prompt_embeds` are set and their positions are interleaved.
self.prompt_is_token_ids = prompt_is_token_ids
# Cache per-block prompt-embed hashes to avoid rehashing the same
# tensor slices when generating extra keys.
self._prompt_embeds_per_block_hashes: dict[tuple[int, int], bytes] = {}
@@ -184,6 +189,7 @@ class Request:
client_index=request.client_index,
prompt_token_ids=request.prompt_token_ids,
prompt_embeds=request.prompt_embeds,
prompt_is_token_ids=request.prompt_is_token_ids,
mm_features=request.mm_features,
sampling_params=request.sampling_params,
pooling_params=request.pooling_params,
+10 -1
View File
@@ -50,6 +50,10 @@ class CachedRequestState:
lora_request: LoRARequest | None = None
prompt_embeds: torch.Tensor | None = None
# Per-position mask for mixed-mode inputs (e.g chat completion with
# prompt_embeds content parts). See `Request.prompt_is_token_ids`.
prompt_is_token_ids: list[bool] | None = None
# Used when both async_scheduling and spec_decode are enabled.
prev_num_draft_len: int = 0
@@ -356,7 +360,12 @@ class InputBatch:
end_idx = start_idx + len(request.output_token_ids)
if request.prompt_token_ids is not None:
self.token_ids_cpu[req_index, :num_prompt_tokens] = request.prompt_token_ids
self.is_token_ids[req_index, :num_prompt_tokens] = True
if request.prompt_is_token_ids is not None:
self.is_token_ids[req_index, :num_prompt_tokens] = (
request.prompt_is_token_ids
)
else:
self.is_token_ids[req_index, :num_prompt_tokens] = True
else:
self.is_token_ids[req_index, :num_prompt_tokens] = False
if request.prompt_embeds is not None:
+35 -1
View File
@@ -1163,6 +1163,7 @@ class GPUModelRunner(
req_id=req_id,
prompt_token_ids=new_req_data.prompt_token_ids,
prompt_embeds=new_req_data.prompt_embeds,
prompt_is_token_ids=new_req_data.prompt_is_token_ids,
mm_features=new_req_data.mm_features,
sampling_params=sampling_params,
pooling_params=pooling_params,
@@ -1505,10 +1506,16 @@ class GPUModelRunner(
)
mrope_model = cast(SupportsMRoPE, model)
# `prompt_embeds` is a passthrough modality (no grid_thw), models'
# M-RoPE code assumes per-feature grid info, so filter it out. The
# prompt_embeds positions are treated as text positions for M-RoPE.
mrope_features = [
f for f in req_state.mm_features if f.modality != "prompt_embeds"
]
req_state.mrope_positions, req_state.mrope_position_delta = (
mrope_model.get_mrope_input_positions(
req_state.prompt_token_ids,
req_state.mm_features,
mrope_features,
)
)
@@ -2744,6 +2751,33 @@ class GPUModelRunner(
if not mm_kwargs:
return []
# `prompt_embeds` is a passthrough modality, the tensor is already in
# the model embedding space, so no encoder runs. Inject each
# `prompt_embeds` tensor directly into the encoder cache here so that
# `_gather_mm_embeddings` can splice it via the standard `is_mm_embed`
# path.
pe_indices = [
i
for i, (modality, _) in enumerate(mm_kwargs)
if modality == "prompt_embeds"
]
if pe_indices:
for i in pe_indices:
pe_tensor = mm_kwargs[i][1]["embedding"].data
assert isinstance(pe_tensor, torch.Tensor)
self.encoder_cache[mm_hashes[i]] = pe_tensor.to(self.device)
self.maybe_save_ec_to_connector(self.encoder_cache, mm_hashes[i])
# Filter out `prompt_embeds` items from mm_kwargs/mm_hashes/mm_lora_refs
# since they don't require further encoder processing.
mm_hashes = [h for i, h in enumerate(mm_hashes) if i not in pe_indices]
mm_kwargs = [k for i, k in enumerate(mm_kwargs) if i not in pe_indices]
mm_lora_refs = [
r for i, r in enumerate(mm_lora_refs) if i not in pe_indices
]
if not mm_kwargs:
return [] # nothing left to encode after filtering out `prompt_embeds`
should_time = bool(
self.observability_config
and self.observability_config.enable_mm_processor_stats