feat: Enable prompt_embeds Content Part Support in vLLM Chat Completions API (#40720)
Signed-off-by: Luis Robaina <luis@protopia.ai> Signed-off-by: Luis Robaina 🚀 <luisfabian1545@gmail.com> Signed-off-by: LuisRobaina <luis@protopia.ai> Co-authored-by: Andrew Sansom <qthequartermasterman@gmail.com>
This commit is contained in:
@@ -11,7 +11,7 @@ from dataclasses import dataclass
|
||||
from functools import cached_property, lru_cache, partial
|
||||
from itertools import accumulate
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, TypeVar, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, Generic, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
from openai.types.chat import (
|
||||
ChatCompletionAssistantMessageParam,
|
||||
@@ -36,7 +36,7 @@ from PIL import Image
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
# pydantic needs the TypedDict from typing_extensions
|
||||
from typing_extensions import Required, TypedDict
|
||||
from typing_extensions import Required, TypedDict, override
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import ModelConfig
|
||||
@@ -55,6 +55,10 @@ from vllm.multimodal.inputs import (
|
||||
)
|
||||
from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor
|
||||
from vllm.renderers.embed_utils import (
|
||||
safe_load_prompt_embeds,
|
||||
safe_load_prompt_embeds_async,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
@@ -98,9 +102,40 @@ MODALITY_PLACEHOLDERS_MAP = {
|
||||
"image": "<##IMAGE##>",
|
||||
"audio": "<##AUDIO##>",
|
||||
"video": "<##VIDEO##>",
|
||||
"prompt_embeds": "<##PROMPT_EMBEDS##>",
|
||||
}
|
||||
|
||||
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN: Final[str] = "<prompt_embeds>"
|
||||
"""The special token used as a placeholder for each embedding
|
||||
position during chat template rendering.
|
||||
|
||||
Registered as an additional special token when `--enable-prompt-embeds` is set.
|
||||
See `_ensure_prompt_embeds_placeholder_token` in `vllm/renderers/hf.py`.
|
||||
"""
|
||||
|
||||
|
||||
_REQUIRE_MM_PROCESSOR_ERROR: Final[str] = (
|
||||
"Resolving modality {modality!r} requires a multimodal processor "
|
||||
"but none is available."
|
||||
)
|
||||
|
||||
_ENABLE_PROMPT_EMBEDS_ERROR: Final[str] = (
|
||||
"You must set `--enable-prompt-embeds` to input `prompt_embeds`"
|
||||
)
|
||||
|
||||
_PROMPT_EMBEDS_MISSING_DATA_ERROR: Final[str] = (
|
||||
"prompt_embeds content part requires a non-empty `data` field "
|
||||
"with base64-encoded tensor bytes."
|
||||
)
|
||||
|
||||
_RESERVED_PLACEHOLDER_IN_TEXT_ERROR: Final[str] = (
|
||||
"Text content may not contain the reserved placeholder {token!r}. "
|
||||
"This placeholder is used internally to mark `prompt_embeds` splice "
|
||||
"positions in the tokenized prompt."
|
||||
)
|
||||
|
||||
|
||||
class AudioURL(TypedDict, total=False):
|
||||
url: Required[str]
|
||||
"""
|
||||
@@ -147,6 +182,17 @@ class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
|
||||
"""
|
||||
|
||||
|
||||
class ChatCompletionContentPartPromptEmbedsParam(TypedDict, total=False):
|
||||
data: Required[str]
|
||||
"""
|
||||
Base64-encoded bytes of a serialized `torch.Tensor` of shape
|
||||
`(num_tokens, hidden_size)`. The tensor's `dtype` and `hidden_size` must
|
||||
match the model's input embedding layer.
|
||||
"""
|
||||
type: Required[Literal["prompt_embeds"]]
|
||||
"""The type of the content part."""
|
||||
|
||||
|
||||
class VideoURL(TypedDict, total=False):
|
||||
url: Required[str]
|
||||
"""
|
||||
@@ -282,6 +328,7 @@ ChatCompletionContentPartParam: TypeAlias = (
|
||||
| CustomChatCompletionContentSimpleImageParam
|
||||
| ChatCompletionContentPartImageEmbedsParam
|
||||
| ChatCompletionContentPartAudioEmbedsParam
|
||||
| ChatCompletionContentPartPromptEmbedsParam
|
||||
| CustomChatCompletionContentSimpleAudioParam
|
||||
| CustomChatCompletionContentSimpleVideoParam
|
||||
| CustomChatCompletionContentToolReferenceParam
|
||||
@@ -367,7 +414,13 @@ ChatTemplateContentFormat = Literal["string", "openai"]
|
||||
|
||||
|
||||
ModalityStr = Literal[
|
||||
"image", "audio", "video", "image_embeds", "audio_embeds", "vision_chunk"
|
||||
"image",
|
||||
"audio",
|
||||
"video",
|
||||
"image_embeds",
|
||||
"audio_embeds",
|
||||
"vision_chunk",
|
||||
"prompt_embeds",
|
||||
]
|
||||
_T = TypeVar("_T")
|
||||
|
||||
@@ -549,7 +602,17 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
|
||||
|
||||
An optional uuid can be added which serves as a unique identifier of the
|
||||
media.
|
||||
|
||||
Note:
|
||||
`prompt_embeds` bypass MM-processor validation because they are
|
||||
pre-computed embeddings that do not go through any HF processor, encoder,
|
||||
or model-specific placeholder logic. The corresponding placeholder string is
|
||||
managed by the parser via `_add_placeholder`, so we return None here.
|
||||
"""
|
||||
if modality == "prompt_embeds":
|
||||
self._items_by_modality["prompt_embeds"].append(item)
|
||||
return None
|
||||
|
||||
input_modality = modality.replace("_embeds", "")
|
||||
original_modality = modality
|
||||
use_vision_chunk = (
|
||||
@@ -660,17 +723,32 @@ def _resolve_vision_chunk_items(
|
||||
|
||||
def _resolve_items(
|
||||
items_by_modality: dict[str, list[tuple[object, str | None]]],
|
||||
mm_processor: BaseMultiModalProcessor,
|
||||
mm_processor: BaseMultiModalProcessor | None,
|
||||
modality_order: dict[str, list[str]],
|
||||
) -> tuple[MultiModalDataDict, MultiModalUUIDDict]:
|
||||
"""
|
||||
Materialize the tracker's per-modality items into `mm_data` / `mm_uuids`.
|
||||
|
||||
Note:
|
||||
`mm_processor` is `None` for text-only models (no registered HF
|
||||
processor) whose only modality is `prompt_embeds`. Every other
|
||||
modality requires a processor, enforced by the guard below.
|
||||
"""
|
||||
if "image" in items_by_modality and "image_embeds" in items_by_modality:
|
||||
raise ValueError("Mixing raw image and embedding inputs is not allowed")
|
||||
if "audio" in items_by_modality and "audio_embeds" in items_by_modality:
|
||||
raise ValueError("Mixing raw audio and embedding inputs is not allowed")
|
||||
# `prompt_embeds` bypasses HF MM processors. Every other modality requires one.
|
||||
processor_modalities = items_by_modality.keys() - {"prompt_embeds"}
|
||||
if processor_modalities and mm_processor is None:
|
||||
raise RuntimeError(
|
||||
_REQUIRE_MM_PROCESSOR_ERROR.format(modality=processor_modalities)
|
||||
)
|
||||
|
||||
mm_data = {}
|
||||
mm_uuids = {}
|
||||
if "image_embeds" in items_by_modality:
|
||||
assert mm_processor is not None
|
||||
mm_data["image"] = _get_embeds_data(
|
||||
"image",
|
||||
[data for data, uuid in items_by_modality["image_embeds"]],
|
||||
@@ -681,6 +759,7 @@ def _resolve_items(
|
||||
mm_data["image"] = [data for data, uuid in items_by_modality["image"]]
|
||||
mm_uuids["image"] = [uuid for data, uuid in items_by_modality["image"]]
|
||||
if "audio_embeds" in items_by_modality:
|
||||
assert mm_processor is not None
|
||||
mm_data["audio"] = _get_embeds_data(
|
||||
"audio",
|
||||
[data for data, uuid in items_by_modality["audio_embeds"]],
|
||||
@@ -694,6 +773,7 @@ def _resolve_items(
|
||||
mm_data["video"] = [data for data, uuid in items_by_modality["video"]]
|
||||
mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]]
|
||||
if "vision_chunk" in items_by_modality:
|
||||
assert mm_processor is not None
|
||||
# Process vision_chunk items - extract from (data, modality) tuples
|
||||
# and convert to VisionChunk types with proper UUID handling
|
||||
processed_chunks, vision_chunk_uuids = _resolve_vision_chunk_items(
|
||||
@@ -703,6 +783,10 @@ def _resolve_items(
|
||||
)
|
||||
mm_data["vision_chunk"] = processed_chunks
|
||||
mm_uuids["vision_chunk"] = vision_chunk_uuids
|
||||
if "prompt_embeds" in items_by_modality:
|
||||
mm_data["prompt_embeds"] = [
|
||||
data for data, _uuid in items_by_modality["prompt_embeds"]
|
||||
]
|
||||
|
||||
return mm_data, mm_uuids
|
||||
|
||||
@@ -714,8 +798,16 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]
|
||||
if not self._items_by_modality:
|
||||
return None, None
|
||||
|
||||
# Text-only models (`is_multimodal_model=False`) with inputs of
|
||||
# modality `prompt_embeds` have no MM processor since `prompt_embeds` are
|
||||
# pre-computed and require no processing, so we pass `None`.
|
||||
mm_processor = (
|
||||
self.mm_processor if self._model_config.is_multimodal_model else None
|
||||
)
|
||||
return _resolve_items(
|
||||
dict(self._items_by_modality), self.mm_processor, self._modality_order
|
||||
dict(self._items_by_modality),
|
||||
mm_processor,
|
||||
self._modality_order,
|
||||
)
|
||||
|
||||
def create_parser(
|
||||
@@ -738,8 +830,13 @@ class AsyncMultiModalItemTracker(
|
||||
for modality, coros in self._items_by_modality.items()
|
||||
}
|
||||
|
||||
mm_processor = (
|
||||
self.mm_processor if self._model_config.is_multimodal_model else None
|
||||
)
|
||||
return _resolve_items(
|
||||
resolved_items_by_modality, self.mm_processor, self._modality_order
|
||||
resolved_items_by_modality,
|
||||
mm_processor,
|
||||
self._modality_order,
|
||||
)
|
||||
|
||||
def create_parser(
|
||||
@@ -758,10 +855,16 @@ class BaseMultiModalContentParser(ABC):
|
||||
# general MM placeholder:
|
||||
# {
|
||||
# "<##IMAGE##>": ["<image>", "<image>", "<image>"],
|
||||
# "<##AUDIO##>": ["<audio>", "<audio>"]
|
||||
# "<##AUDIO##>": ["<audio>", "<audio>"],
|
||||
# "<##PROMPT_EMBEDS##>": ["<prompt_embeds>", "<prompt_embeds>"]
|
||||
# }
|
||||
self._placeholder_storage: dict[str, list] = defaultdict(list)
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def model_config(self) -> ModelConfig:
|
||||
raise NotImplementedError
|
||||
|
||||
def _add_placeholder(self, modality: ModalityStr, placeholder: str | None):
|
||||
mod_placeholder = MODALITY_PLACEHOLDERS_MAP[modality]
|
||||
if placeholder:
|
||||
@@ -806,6 +909,10 @@ class BaseMultiModalContentParser(ABC):
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def parse_prompt_embeds(self, data: str) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
|
||||
raise NotImplementedError
|
||||
@@ -834,6 +941,21 @@ class MultiModalContentParser(BaseMultiModalContentParser):
|
||||
def model_config(self) -> ModelConfig:
|
||||
return self._tracker.model_config
|
||||
|
||||
@override
|
||||
def parse_prompt_embeds(self, data: str) -> None:
|
||||
"""Decode a base64 prompt embeds tensor and store it in the tracker.
|
||||
|
||||
Emits a single `PROMPT_EMBEDS_PLACEHOLDER_TOKEN` sentinel per
|
||||
content part. The renderer later expands each sentinel to a span of
|
||||
`tensor.shape[0]` placeholder tokens after tokenization.
|
||||
"""
|
||||
if not self.model_config.enable_prompt_embeds:
|
||||
raise ValueError(_ENABLE_PROMPT_EMBEDS_ERROR)
|
||||
|
||||
tensor = safe_load_prompt_embeds(self.model_config, data.encode())
|
||||
self._tracker.add("prompt_embeds", (tensor, None))
|
||||
self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
|
||||
|
||||
def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
|
||||
image = self._connector.fetch_image(image_url) if image_url else None
|
||||
|
||||
@@ -958,6 +1080,29 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
|
||||
def model_config(self) -> ModelConfig:
|
||||
return self._tracker.model_config
|
||||
|
||||
@override
|
||||
def parse_prompt_embeds(self, data: str) -> None:
|
||||
"""Schedule async prompt embeds decode and store the coroutine in the tracker.
|
||||
|
||||
Like the sync variant, emits a single sentinel `PROMPT_EMBEDS_PLACEHOLDER_TOKEN`
|
||||
per content part. Unlike the sync variant, the tensor decode is deferred to a
|
||||
thread-pool executor via `safe_load_prompt_embeds_async`.
|
||||
"""
|
||||
if not self.model_config.enable_prompt_embeds:
|
||||
raise ValueError(_ENABLE_PROMPT_EMBEDS_ERROR)
|
||||
|
||||
coro = self._load_prompt_embeds_async(data.encode())
|
||||
self._tracker.add("prompt_embeds", coro)
|
||||
self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN)
|
||||
|
||||
async def _load_prompt_embeds_async(
|
||||
self, data_bytes: bytes
|
||||
) -> tuple[torch.Tensor, None]:
|
||||
# Second tuple slot fills the tracker's generic `(item, uuid | None)`
|
||||
# contract. prompt_embeds has no UUID concept, so it's always `None`.
|
||||
tensor = await safe_load_prompt_embeds_async(self.model_config, data_bytes)
|
||||
return tensor, None
|
||||
|
||||
async def _image_with_uuid_async(self, image_url: str | None, uuid: str | None):
|
||||
image = (
|
||||
await self._connector.fetch_image_async(image_url) if image_url else None
|
||||
@@ -1269,6 +1414,7 @@ def _get_full_multimodal_text_prompt(
|
||||
_TextParser = partial(cast, ChatCompletionContentPartTextParam)
|
||||
_ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)
|
||||
_AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam)
|
||||
_PromptEmbedsParser = partial(cast, ChatCompletionContentPartPromptEmbedsParam)
|
||||
_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
|
||||
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
|
||||
_PILImageParser = partial(cast, CustomChatCompletionContentPILImageParam)
|
||||
@@ -1294,6 +1440,7 @@ MM_PARSER_MAP: dict[
|
||||
"image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
|
||||
"image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None),
|
||||
"audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None),
|
||||
"prompt_embeds": lambda part: _PromptEmbedsParser(part).get("data", None),
|
||||
"image_pil": lambda part: _PILImageParser(part).get("image_pil", None),
|
||||
"audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),
|
||||
"input_audio": lambda part: _InputAudioParser(part).get("input_audio", None),
|
||||
@@ -1372,6 +1519,11 @@ def _parse_chat_message_content_mm_part(
|
||||
)
|
||||
audio_embeds = audio_params.get("audio_embeds", None)
|
||||
return "audio_embeds", audio_embeds
|
||||
if "prompt_embeds" in part:
|
||||
prompt_embeds_params = cast( # type: ignore[assignment]
|
||||
ChatCompletionContentPartPromptEmbedsParam, part
|
||||
)
|
||||
return "prompt_embeds", prompt_embeds_params.get("data", None)
|
||||
if "audio_url" in part:
|
||||
audio_params = cast( # type: ignore[assignment]
|
||||
CustomChatCompletionContentSimpleAudioParam, part
|
||||
@@ -1455,6 +1607,24 @@ def _parse_chat_message_content_parts(
|
||||
return [ConversationMessage(role=role, content=text_prompt)]
|
||||
|
||||
|
||||
def _reject_reserved_placeholder_in_text(text: str, model_config: ModelConfig) -> None:
|
||||
"""Reject user-supplied text parts that contains the reserved `prompt_embeds`
|
||||
placeholder sentinel.
|
||||
|
||||
When the server accepts `prompt_embeds`, the placeholder token is
|
||||
registered as a single unsplittable special token on the tokenizer. Any
|
||||
user text that happens to contain the literal sequence would tokenize to
|
||||
the same ID and be mistaken for a splice point by the renderer, letting a
|
||||
caller move or inject splice positions via plain text content.
|
||||
"""
|
||||
if model_config.enable_prompt_embeds and PROMPT_EMBEDS_PLACEHOLDER_TOKEN in text:
|
||||
raise ValueError(
|
||||
_RESERVED_PLACEHOLDER_IN_TEXT_ERROR.format(
|
||||
token=PROMPT_EMBEDS_PLACEHOLDER_TOKEN
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _parse_chat_message_content_part(
|
||||
part: ChatCompletionContentPartParam,
|
||||
mm_parser: BaseMultiModalContentParser,
|
||||
@@ -1470,6 +1640,7 @@ def _parse_chat_message_content_part(
|
||||
with multimodal placeholders.
|
||||
"""
|
||||
if isinstance(part, str): # Handle plain text parts
|
||||
_reject_reserved_placeholder_in_text(part, mm_parser.model_config)
|
||||
if wrap_dicts:
|
||||
return {"type": "text", "text": part}
|
||||
return part
|
||||
@@ -1488,6 +1659,7 @@ def _parse_chat_message_content_part(
|
||||
|
||||
if part_type in ("text", "input_text", "output_text", "refusal", "thinking"):
|
||||
str_content = cast(str, content)
|
||||
_reject_reserved_placeholder_in_text(str_content, mm_parser.model_config)
|
||||
if wrap_dicts:
|
||||
return {"type": "text", "text": str_content}
|
||||
else:
|
||||
@@ -1516,6 +1688,11 @@ def _parse_chat_message_content_part(
|
||||
content = cast(str | dict[str, str], content) if content is not None else None
|
||||
mm_parser.parse_audio_embeds(content, uuid)
|
||||
modality = "audio"
|
||||
elif part_type == "prompt_embeds":
|
||||
if not content:
|
||||
raise ValueError(_PROMPT_EMBEDS_MISSING_DATA_ERROR)
|
||||
mm_parser.parse_prompt_embeds(cast(str, content))
|
||||
modality = "prompt_embeds"
|
||||
elif part_type == "audio_url":
|
||||
str_content = cast(str, content)
|
||||
mm_parser.parse_audio(str_content, uuid)
|
||||
@@ -1544,7 +1721,18 @@ def _parse_chat_message_content_part(
|
||||
)
|
||||
|
||||
if wrap_dicts:
|
||||
if modality == "prompt_embeds":
|
||||
# Chat templates don't know about the "prompt_embeds" modality,
|
||||
# emit the single sentinel token as text so the template renders
|
||||
# it inline. The renderer later expands it to N tokens post-tokenize.
|
||||
return {"type": "text", "text": PROMPT_EMBEDS_PLACEHOLDER_TOKEN}
|
||||
return {"type": modality}
|
||||
if modality == "prompt_embeds":
|
||||
# Emit the renderer token inline regardless of `interleave_strings`,
|
||||
# prompt_embeds are spliced at the token offset so position matters.
|
||||
# Falling back to front-padding via `missing_placeholders` would
|
||||
# reorder them relative to surrounding text.
|
||||
return PROMPT_EMBEDS_PLACEHOLDER_TOKEN
|
||||
return MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None
|
||||
|
||||
|
||||
@@ -1668,7 +1856,10 @@ def parse_chat_messages(
|
||||
MultiModalUUIDDict | None,
|
||||
]:
|
||||
conversation: list[ConversationMessage] = []
|
||||
mm_tracker = MultiModalItemTracker(model_config, media_io_kwargs=media_io_kwargs)
|
||||
mm_tracker = MultiModalItemTracker(
|
||||
model_config,
|
||||
media_io_kwargs=media_io_kwargs,
|
||||
)
|
||||
|
||||
for msg in messages:
|
||||
sub_messages = _parse_chat_message_content(
|
||||
@@ -1705,7 +1896,8 @@ async def parse_chat_messages_async(
|
||||
]:
|
||||
conversation: list[ConversationMessage] = []
|
||||
mm_tracker = AsyncMultiModalItemTracker(
|
||||
model_config, media_io_kwargs=media_io_kwargs
|
||||
model_config,
|
||||
media_io_kwargs=media_io_kwargs,
|
||||
)
|
||||
|
||||
for msg in messages:
|
||||
|
||||
@@ -926,7 +926,10 @@ class LLM:
|
||||
add_generation_prompt=add_generation_prompt,
|
||||
continue_final_message=continue_final_message,
|
||||
tools=tools,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
tokenize=(
|
||||
is_mistral_tokenizer(renderer.tokenizer)
|
||||
or self.model_config.enable_prompt_embeds
|
||||
),
|
||||
),
|
||||
),
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
|
||||
@@ -541,7 +541,10 @@ class OpenAIServingRender:
|
||||
default_template_kwargs,
|
||||
dict(
|
||||
tools=tool_dicts,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
tokenize=(
|
||||
is_mistral_tokenizer(renderer.tokenizer)
|
||||
or self.model_config.enable_prompt_embeds
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -71,12 +71,27 @@ class EmbedsInput(_InputOptions):
|
||||
prompt: NotRequired[str]
|
||||
"""The prompt text corresponding to the token IDs, if available."""
|
||||
|
||||
prompt_token_ids: NotRequired[list[int]]
|
||||
"""Token IDs of the rendered prompt. Only set for mixed-mode inputs
|
||||
(chat completion with `prompt_embeds` content parts). When present,
|
||||
`is_token_ids` MUST also be present and have the same length.
|
||||
For pure-embeds inputs this field is absent."""
|
||||
|
||||
is_token_ids: NotRequired[list[bool]]
|
||||
"""Per-position mask for mixed-mode inputs. `True` means the position
|
||||
is a real token ID (use the model's embedding layer); `False` means
|
||||
the position uses a pre-computed embedding row from `prompt_embeds`.
|
||||
Length MUST equal `len(prompt_token_ids)`.
|
||||
For pure-embeds inputs this field is absent."""
|
||||
|
||||
|
||||
def embeds_input(
|
||||
prompt_embeds: "torch.Tensor",
|
||||
*,
|
||||
prompt: str | None = None,
|
||||
cache_salt: str | None = None,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
is_token_ids: list[bool] | None = None,
|
||||
) -> EmbedsInput:
|
||||
"""
|
||||
Construct [`EmbedsInput`][vllm.inputs.engine.EmbedsInput]
|
||||
@@ -88,6 +103,10 @@ def embeds_input(
|
||||
inputs["prompt"] = prompt
|
||||
if cache_salt is not None:
|
||||
inputs["cache_salt"] = cache_salt
|
||||
if prompt_token_ids is not None:
|
||||
inputs["prompt_token_ids"] = prompt_token_ids
|
||||
if is_token_ids is not None:
|
||||
inputs["is_token_ids"] = is_token_ids
|
||||
|
||||
return inputs
|
||||
|
||||
|
||||
@@ -125,6 +125,17 @@ class EmbedsPrompt(_PromptOptions):
|
||||
prompt: NotRequired[str]
|
||||
"""The prompt text corresponding to the token embeddings, if available."""
|
||||
|
||||
prompt_token_ids: NotRequired[list[int]]
|
||||
"""Token IDs for mixed-mode inputs (chat completion with
|
||||
`prompt_embeds` content parts). The tokens at positions where
|
||||
`prompt_is_token_ids` is `False` are placeholder tokens that
|
||||
get replaced by entries from `prompt_embeds` in the forward pass."""
|
||||
|
||||
prompt_is_token_ids: NotRequired[list[bool]]
|
||||
"""Per-position mask, `True` uses the real token ID, `False` uses
|
||||
the corresponding entry from `prompt_embeds`.
|
||||
Must be the same length as `prompt_token_ids` when both are set."""
|
||||
|
||||
|
||||
DecoderOnlyPrompt: TypeAlias = (
|
||||
str | TextPrompt | list[int] | TokensPrompt | EmbedsPrompt
|
||||
|
||||
@@ -769,6 +769,8 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
return embeds_input(
|
||||
prompt_embeds=prompt_embeds,
|
||||
cache_salt=prompt.get("cache_salt"),
|
||||
prompt_token_ids=prompt.get("prompt_token_ids"),
|
||||
is_token_ids=prompt.get("prompt_is_token_ids"),
|
||||
)
|
||||
|
||||
async def _process_tokens_async(
|
||||
|
||||
@@ -7,6 +7,7 @@ import pybase64
|
||||
import torch
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.async_utils import make_async
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
@@ -30,15 +31,53 @@ def safe_load_prompt_embeds(
|
||||
weights_only=True,
|
||||
map_location=torch.device("cpu"),
|
||||
)
|
||||
assert isinstance(tensor, torch.Tensor) and tensor.dtype in (
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
torch.float16,
|
||||
)
|
||||
if not isinstance(tensor, torch.Tensor):
|
||||
raise VLLMValidationError(
|
||||
"`prompt_embeds` payload did not deserialize to a torch.Tensor.",
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
tensor = tensor.to_dense()
|
||||
|
||||
if tensor.dim() > 2:
|
||||
tensor = tensor.squeeze(0)
|
||||
assert tensor.dim() == 2
|
||||
if tensor.dim() != 2:
|
||||
raise VLLMValidationError(
|
||||
"`prompt_embeds` must be a 2D tensor of shape "
|
||||
f"(num_tokens, hidden_size); got shape {tuple(tensor.shape)}.",
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
|
||||
# Pin each tensor to the model's hidden_size. Validating here
|
||||
# also transitively guarantees cross-tensor consistency for requests that
|
||||
# include multiple `prompt_embeds` parts, which is required by downstream
|
||||
# concatenation in `_build_mixed_prompt_embeds`.
|
||||
expected_hidden_size = model_config.get_hidden_size()
|
||||
if tensor.shape[1] != expected_hidden_size:
|
||||
raise VLLMValidationError(
|
||||
f"`prompt_embeds` hidden_size {tensor.shape[1]} does not match "
|
||||
f"the model's hidden_size {expected_hidden_size}.",
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
|
||||
# Cast to the model's dtype so API clients don't need to know the server's
|
||||
# `--dtype` setting ahead of time. Only floating-point source dtypes are
|
||||
# allowed. integer / bool / complex inputs almost certainly indicate caller
|
||||
# error (e.g. quantized payloads, wrong tensor), and a silent `.to()`
|
||||
# could hide a real mistake.
|
||||
expected_dtype = model_config.dtype
|
||||
if tensor.dtype != expected_dtype:
|
||||
if not tensor.is_floating_point():
|
||||
raise VLLMValidationError(
|
||||
f"`prompt_embeds` dtype {tensor.dtype} is not a floating-point "
|
||||
f"type, cannot safely cast to the model's dtype {expected_dtype}.",
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
tensor = tensor.to(expected_dtype)
|
||||
|
||||
return tensor
|
||||
|
||||
|
||||
safe_load_prompt_embeds_async = make_async(safe_load_prompt_embeds)
|
||||
"""Async variant of `safe_load_prompt_embeds` that defers the decode to a
|
||||
thread-pool executor, so the asyncio event loop is not blocked by the base64
|
||||
decode + `torch.load` work."""
|
||||
|
||||
+440
-18
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import itertools
|
||||
import weakref
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Set
|
||||
from collections.abc import Sequence
|
||||
from functools import lru_cache
|
||||
from typing import Any, Literal, cast, overload
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
|
||||
|
||||
import jinja2
|
||||
import jinja2.ext
|
||||
@@ -13,20 +16,32 @@ import jinja2.meta
|
||||
import jinja2.nodes
|
||||
import jinja2.parser
|
||||
import jinja2.sandbox
|
||||
import torch
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormat,
|
||||
ChatTemplateContentFormatOption,
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
ChatTemplateResolutionError,
|
||||
ConversationMessage,
|
||||
load_chat_template,
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||
from vllm.inputs import EmbedsPrompt
|
||||
from vllm.inputs.engine import MultiModalInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalSharedField,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
apply_token_matches,
|
||||
find_mm_placeholders,
|
||||
)
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
|
||||
from vllm.transformers_utils.processor import cached_get_processor
|
||||
@@ -34,13 +49,166 @@ from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.func_utils import supports_kw
|
||||
|
||||
from .base import BaseRenderer
|
||||
from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Set
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormat,
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict, TokensPrompt
|
||||
from vllm.inputs.engine import TokensInput
|
||||
from vllm.multimodal.processing.processor import (
|
||||
MultiModalPromptUpdates,
|
||||
ResolvedPromptUpdate,
|
||||
)
|
||||
|
||||
from .inputs import DictPrompt
|
||||
from .params import ChatParams
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Cache of `tokenizer -> prompt_embeds placeholder token ID`. Keyed by the
|
||||
# tokenizer object (not `id(tokenizer)`) so a fresh tokenizer landing at a
|
||||
# recycled memory address can't pick up a stale tid. Entries evict atomically
|
||||
# with the tokenizer's garbage-collection.
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_CACHE: Final[
|
||||
weakref.WeakKeyDictionary[HfTokenizer, int]
|
||||
] = weakref.WeakKeyDictionary()
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_ERROR: Final[str] = (
|
||||
"Expected {token!r} to tokenize to exactly 1 token, got {num_ids} ({ids!r})."
|
||||
)
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR: Final[str] = (
|
||||
"Expected {expected} prompt_embeds placeholder spans in the "
|
||||
"tokenized prompt, found {actual}."
|
||||
)
|
||||
_MISSING_PROMPT_TOKEN_IDS_ERROR: Final[str] = (
|
||||
"Expected prompt_token_ids in rendered prompt when prompt_embeds "
|
||||
"are present. This indicates the chat template was invoked with "
|
||||
"tokenize=False."
|
||||
)
|
||||
_TOKENIZE_OVERRIDE_WARNING: Final[str] = (
|
||||
"Overriding `tokenize=False` to `True` because `prompt_embeds` "
|
||||
"post-processing requires tokenized IDs."
|
||||
)
|
||||
|
||||
|
||||
def _ensure_prompt_embeds_placeholder_token(tokenizer: HfTokenizer) -> int:
|
||||
"""Register `PROMPT_EMBEDS_PLACEHOLDER_TOKEN` as a special token and return
|
||||
its token ID."""
|
||||
cached = _PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_CACHE.get(tokenizer)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
tokenizer.add_special_tokens(
|
||||
{"additional_special_tokens": [PROMPT_EMBEDS_PLACEHOLDER_TOKEN]}
|
||||
)
|
||||
|
||||
ids = tokenizer.encode(PROMPT_EMBEDS_PLACEHOLDER_TOKEN, add_special_tokens=False)
|
||||
if len(ids) != 1:
|
||||
raise RuntimeError(
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_ERROR.format(
|
||||
token=PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
num_ids=len(ids),
|
||||
ids=ids,
|
||||
)
|
||||
)
|
||||
|
||||
token_id = ids[0]
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_CACHE[tokenizer] = token_id
|
||||
return token_id
|
||||
|
||||
|
||||
def _build_prompt_embeds_updates(
|
||||
prompt_embeds_tensors: Sequence[torch.Tensor],
|
||||
placeholder_token_id: int,
|
||||
) -> MultiModalPromptUpdates:
|
||||
"""Build `MultiModalPromptUpdates` for `prompt_embeds` expansion.
|
||||
|
||||
Each tensor produces a `PromptReplacement` that maps
|
||||
`[placeholder_token_id]` -> `[placeholder_token_id] x N`
|
||||
(where `N = tensor.shape[0]`).
|
||||
"""
|
||||
updates: list[Sequence[ResolvedPromptUpdate]] = []
|
||||
for i, tensor in enumerate(prompt_embeds_tensors):
|
||||
update = PromptReplacement(
|
||||
modality="prompt_embeds",
|
||||
target=[placeholder_token_id],
|
||||
replacement=[placeholder_token_id] * tensor.shape[0],
|
||||
)
|
||||
updates.append([update.resolve(item_idx=i)])
|
||||
return {"prompt_embeds": updates}
|
||||
|
||||
|
||||
def _expand_prompt_embeds_placeholders(
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> list[int]:
|
||||
"""Expand each 1-token `prompt_embeds` sentinel into an N-token span.
|
||||
|
||||
Uses `apply_token_matches`. Each single placeholder token in
|
||||
`token_ids` is replaced with a consecutive span of
|
||||
`tensor.shape[0]` copies, following tensors in order.
|
||||
"""
|
||||
expanded, _ = apply_token_matches(token_ids, mm_prompt_updates, tokenizer=None)
|
||||
return expanded
|
||||
|
||||
|
||||
def _build_prompt_embeds_positions(
|
||||
token_ids: list[int],
|
||||
num_tensors: int,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""Locate each prompt_embeds placeholder span in `token_ids`.
|
||||
|
||||
Expects `token_ids` to already contain expanded N-token spans.
|
||||
Returns `[(start_idx, length), ...]` aligned with the tensors.
|
||||
"""
|
||||
placeholders = find_mm_placeholders(
|
||||
prompt=token_ids,
|
||||
mm_prompt_updates=mm_prompt_updates,
|
||||
tokenizer=None,
|
||||
)
|
||||
features = placeholders.get("prompt_embeds", [])
|
||||
|
||||
if len(features) != num_tensors:
|
||||
raise ValueError(
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR.format(
|
||||
expected=num_tensors,
|
||||
actual=len(features),
|
||||
)
|
||||
)
|
||||
|
||||
return [(f.start_idx, f.length) for f in features]
|
||||
|
||||
|
||||
def _build_mixed_prompt_embeds(
|
||||
token_ids: list[int],
|
||||
prompt_embeds_tensors: Sequence[torch.Tensor],
|
||||
positions: list[tuple[int, int]],
|
||||
) -> tuple[torch.Tensor, list[bool]]:
|
||||
"""Build the full-length `prompt_embeds` tensor and the `is_token_ids`
|
||||
mask aligned to `token_ids`."""
|
||||
total_len = len(token_ids)
|
||||
hidden_size = prompt_embeds_tensors[0].shape[1]
|
||||
dtype = prompt_embeds_tensors[0].dtype
|
||||
|
||||
full_embeds = torch.zeros(total_len, hidden_size, dtype=dtype)
|
||||
is_token_ids = torch.ones(total_len, dtype=torch.bool)
|
||||
|
||||
for (start, length), tensor in zip(positions, prompt_embeds_tensors, strict=True):
|
||||
full_embeds[start : start + length] = tensor
|
||||
is_token_ids[start : start + length] = False
|
||||
|
||||
return full_embeds, is_token_ids.tolist()
|
||||
|
||||
|
||||
_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], str | None]()
|
||||
"""
|
||||
Used in `_try_get_processor_chat_template` to avoid calling
|
||||
@@ -98,7 +266,7 @@ def resolve_chat_template(
|
||||
chat_template: str | None,
|
||||
tools: list[dict[str, Any]] | None,
|
||||
*,
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
) -> str | None:
|
||||
# 1st priority: The given chat template
|
||||
if chat_template is not None:
|
||||
@@ -281,7 +449,7 @@ def _resolve_chat_template_content_format(
|
||||
tools: list[dict[str, Any]] | None,
|
||||
tokenizer: HfTokenizer,
|
||||
*,
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
) -> ChatTemplateContentFormat:
|
||||
resolved_chat_template = resolve_chat_template(
|
||||
tokenizer,
|
||||
@@ -335,7 +503,7 @@ def resolve_chat_template_content_format(
|
||||
given_format: ChatTemplateContentFormatOption,
|
||||
tokenizer: HfTokenizer,
|
||||
*,
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
) -> ChatTemplateContentFormat:
|
||||
if given_format != "auto":
|
||||
return given_format
|
||||
@@ -437,7 +605,7 @@ def resolve_chat_template_kwargs(
|
||||
|
||||
@overload
|
||||
def safe_apply_chat_template(
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
tokenizer: HfTokenizer,
|
||||
conversation: list[ConversationMessage],
|
||||
*,
|
||||
@@ -448,7 +616,7 @@ def safe_apply_chat_template(
|
||||
) -> list[int]: ...
|
||||
@overload
|
||||
def safe_apply_chat_template(
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
tokenizer: HfTokenizer,
|
||||
conversation: list[ConversationMessage],
|
||||
*,
|
||||
@@ -458,7 +626,7 @@ def safe_apply_chat_template(
|
||||
**kwargs,
|
||||
) -> str: ...
|
||||
def safe_apply_chat_template(
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
tokenizer: HfTokenizer,
|
||||
conversation: list[ConversationMessage],
|
||||
*,
|
||||
@@ -486,6 +654,14 @@ def safe_apply_chat_template(
|
||||
chat_template_kwargs=kwargs,
|
||||
)
|
||||
|
||||
# transformers v5 changed the default of `return_dict` to True, which
|
||||
# makes `apply_chat_template(tokenize=True)` return a `BatchEncoding`
|
||||
# instead of `list[int]`. Force `return_dict=False` so downstream code
|
||||
# that expects a flat token list (e.g. `parse_dec_only_prompt`) works
|
||||
# consistently across v4 and v5.
|
||||
if tokenize and "return_dict" not in resolved_kwargs:
|
||||
resolved_kwargs["return_dict"] = False
|
||||
|
||||
try:
|
||||
return tokenizer.apply_chat_template(
|
||||
conversation=conversation, # type: ignore[arg-type]
|
||||
@@ -627,6 +803,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
model_config = self.model_config
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
prompt_embeds_placeholder_token_id: int | None = None
|
||||
if model_config.enable_prompt_embeds:
|
||||
prompt_embeds_placeholder_token_id = (
|
||||
_ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
)
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
model_config,
|
||||
@@ -641,11 +823,30 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
mm_processor_kwargs=params.mm_processor_kwargs,
|
||||
)
|
||||
|
||||
# prompt_embeds tensors are carried by the tracker through mm_data,
|
||||
# but they must NOT be fed to the MM processor (which would reject
|
||||
# the unknown key). Extract them here.
|
||||
prompt_embeds_tensors: list[torch.Tensor] | None = None
|
||||
if mm_data is not None and "prompt_embeds" in mm_data:
|
||||
prompt_embeds_tensors = list(
|
||||
cast(Sequence[torch.Tensor], mm_data["prompt_embeds"])
|
||||
)
|
||||
mm_data = {k: v for k, v in mm_data.items() if k != "prompt_embeds"}
|
||||
if not mm_data:
|
||||
mm_data = None
|
||||
|
||||
chat_template_kwargs = params.get_apply_chat_template_kwargs()
|
||||
if prompt_embeds_tensors:
|
||||
# prompt_embeds post-processing requires prompt_token_ids.
|
||||
if chat_template_kwargs.get("tokenize") is False:
|
||||
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
|
||||
chat_template_kwargs["tokenize"] = True
|
||||
|
||||
prompt_raw = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
@@ -671,6 +872,29 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
)
|
||||
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
|
||||
# When `prompt_embeds` is mixed with other modality data,
|
||||
# `_process_tokens` runs `_process_multimodal` first (expanding
|
||||
# `<|AUDIO|>` / `<|IMAGE|>` placeholders) and then
|
||||
# `_apply_prompt_embeds_to_engine_input` augments the result.
|
||||
# Stash the tensors and placeholder ID for that override to consume.
|
||||
if prompt_embeds_tensors and mm_data:
|
||||
assert prompt_embeds_placeholder_token_id is not None
|
||||
cast(dict, prompt)["_prompt_embeds"] = (
|
||||
prompt_embeds_tensors,
|
||||
prompt_embeds_placeholder_token_id,
|
||||
)
|
||||
if params.mm_processor_kwargs:
|
||||
cast(dict, prompt)["mm_processor_kwargs"] = params.mm_processor_kwargs
|
||||
elif prompt_embeds_tensors:
|
||||
# Pure mode: no other MM data, mutate prompt to EmbedsPrompt shape.
|
||||
assert prompt_embeds_placeholder_token_id is not None
|
||||
self._apply_prompt_embeds_to_prompt(
|
||||
prompt,
|
||||
prompt_embeds_tensors,
|
||||
prompt_embeds_placeholder_token_id,
|
||||
)
|
||||
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
@@ -686,6 +910,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
model_config = self.model_config
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
prompt_embeds_placeholder_token_id: int | None = None
|
||||
if model_config.enable_prompt_embeds:
|
||||
prompt_embeds_placeholder_token_id = (
|
||||
_ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
)
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
model_config,
|
||||
@@ -700,11 +930,27 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
mm_processor_kwargs=params.mm_processor_kwargs,
|
||||
)
|
||||
|
||||
prompt_embeds_tensors: list[torch.Tensor] | None = None
|
||||
if mm_data is not None and "prompt_embeds" in mm_data:
|
||||
prompt_embeds_tensors = list(
|
||||
cast(Sequence[torch.Tensor], mm_data["prompt_embeds"])
|
||||
)
|
||||
mm_data = {k: v for k, v in mm_data.items() if k != "prompt_embeds"}
|
||||
if not mm_data:
|
||||
mm_data = None
|
||||
|
||||
chat_template_kwargs = params.get_apply_chat_template_kwargs()
|
||||
if prompt_embeds_tensors:
|
||||
# prompt_embeds post-processing requires prompt_token_ids.
|
||||
if chat_template_kwargs.get("tokenize") is False:
|
||||
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
|
||||
chat_template_kwargs["tokenize"] = True
|
||||
|
||||
prompt_raw = await self._apply_chat_template_async(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
@@ -728,9 +974,185 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
)
|
||||
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
|
||||
# See `render_messages` for the rationale.
|
||||
if prompt_embeds_tensors and mm_data:
|
||||
assert prompt_embeds_placeholder_token_id is not None
|
||||
cast(dict, prompt)["_prompt_embeds"] = (
|
||||
prompt_embeds_tensors,
|
||||
prompt_embeds_placeholder_token_id,
|
||||
)
|
||||
if params.mm_processor_kwargs:
|
||||
cast(dict, prompt)["mm_processor_kwargs"] = params.mm_processor_kwargs
|
||||
elif prompt_embeds_tensors:
|
||||
assert prompt_embeds_placeholder_token_id is not None
|
||||
self._apply_prompt_embeds_to_prompt(
|
||||
prompt,
|
||||
prompt_embeds_tensors,
|
||||
prompt_embeds_placeholder_token_id,
|
||||
)
|
||||
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt
|
||||
|
||||
@override
|
||||
def _process_tokens(
|
||||
self,
|
||||
prompt: TokensPrompt,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> TokensInput | MultiModalInput:
|
||||
"""Pre-expand `prompt_embeds` sentinels before delegating to the MM
|
||||
processor, then attach `prompt_embeds` modality data to the result.
|
||||
|
||||
Mixed mode only: the `_prompt_embeds` stash is set by
|
||||
`render_messages` when `prompt_embeds` co-exist with other MM data
|
||||
(images, audio, …). We expand each 1-token sentinel to an N-token
|
||||
span *before* calling `super()._process_tokens()` so the MM
|
||||
processor records all placeholder offsets in the final (post-expansion)
|
||||
coordinate space, no offset shifting needed afterwards.
|
||||
"""
|
||||
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
|
||||
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
|
||||
list(prompt["prompt_token_ids"]), mm_updates
|
||||
)
|
||||
engine_input = super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, _ = prompt_embeds_info
|
||||
self._apply_prompt_embeds_to_engine_input(
|
||||
cast(MultiModalInput, engine_input),
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
return engine_input
|
||||
|
||||
@override
|
||||
async def _process_tokens_async(
|
||||
self,
|
||||
prompt: TokensPrompt,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> TokensInput | MultiModalInput:
|
||||
"""Async equivalent of `_process_tokens`."""
|
||||
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
|
||||
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
|
||||
list(prompt["prompt_token_ids"]), mm_updates
|
||||
)
|
||||
engine_input = await super()._process_tokens_async(
|
||||
prompt, skip_mm_cache=skip_mm_cache
|
||||
)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, _ = prompt_embeds_info
|
||||
self._apply_prompt_embeds_to_engine_input(
|
||||
cast(MultiModalInput, engine_input),
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
return engine_input
|
||||
|
||||
@staticmethod
|
||||
def _apply_prompt_embeds_to_prompt(
|
||||
prompt: DictPrompt,
|
||||
prompt_embeds_tensors: list[torch.Tensor],
|
||||
placeholder_token_id: int,
|
||||
) -> None:
|
||||
"""Mutate `prompt` from `TokensPrompt` to `EmbedsPrompt` shape.
|
||||
|
||||
Pure `prompt_embeds` path only (no other MM modalities). Expands
|
||||
each `<prompt_embeds>` sentinel token into an N-token span and builds
|
||||
the full-length `prompt_embeds` tensor + `prompt_is_token_ids` mask
|
||||
that the engine's `enable_prompt_embeds` worker branch consumes.
|
||||
"""
|
||||
token_ids = cast(list[int] | None, prompt.get("prompt_token_ids"))
|
||||
if token_ids is None:
|
||||
raise RuntimeError(_MISSING_PROMPT_TOKEN_IDS_ERROR)
|
||||
|
||||
embeds_orig_positions: list[int] = [
|
||||
i for i, tok in enumerate(token_ids) if tok == placeholder_token_id
|
||||
]
|
||||
if len(embeds_orig_positions) != len(prompt_embeds_tensors):
|
||||
raise ValueError(
|
||||
f"Expected {len(prompt_embeds_tensors)} prompt_embeds "
|
||||
f"placeholder tokens in the rendered prompt, found "
|
||||
f"{len(embeds_orig_positions)}."
|
||||
)
|
||||
|
||||
mm_updates = _build_prompt_embeds_updates(
|
||||
prompt_embeds_tensors, placeholder_token_id
|
||||
)
|
||||
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
|
||||
positions = _build_prompt_embeds_positions(
|
||||
expanded, len(prompt_embeds_tensors), mm_updates
|
||||
)
|
||||
|
||||
embeds_prompt = cast(EmbedsPrompt, prompt)
|
||||
embeds_prompt["prompt_token_ids"] = expanded
|
||||
full_embeds, is_token_ids_mask = _build_mixed_prompt_embeds(
|
||||
expanded, prompt_embeds_tensors, positions
|
||||
)
|
||||
embeds_prompt["prompt_embeds"] = full_embeds
|
||||
embeds_prompt["prompt_is_token_ids"] = is_token_ids_mask
|
||||
|
||||
@staticmethod
|
||||
def _apply_prompt_embeds_to_engine_input(
|
||||
engine_input: MultiModalInput,
|
||||
prompt_embeds_tensors: list[torch.Tensor],
|
||||
mm_updates: MultiModalPromptUpdates,
|
||||
) -> None:
|
||||
"""Augment `engine_input` in-place with a `prompt_embeds` modality.
|
||||
|
||||
Mixed mode: called after `_process_multimodal` has already run on the
|
||||
pre-expanded token IDs (expansion was done in `_process_tokens` before
|
||||
calling `super()`). Locates the already-expanded `prompt_embeds` spans
|
||||
and adds `prompt_embeds` entries to `mm_kwargs`, `mm_hashes`, and
|
||||
`mm_placeholders`.
|
||||
"""
|
||||
# token_ids already contain the pre-expanded N-token spans.
|
||||
token_ids = list(engine_input["prompt_token_ids"])
|
||||
|
||||
positions = _build_prompt_embeds_positions(
|
||||
token_ids, len(prompt_embeds_tensors), mm_updates
|
||||
)
|
||||
|
||||
pe_kwargs_items: list[MultiModalKwargsItem] = []
|
||||
pe_hashes: list[str] = []
|
||||
pe_placeholders: list[PlaceholderRange] = []
|
||||
for tensor, (start, length) in zip(
|
||||
prompt_embeds_tensors, positions, strict=True
|
||||
):
|
||||
pe_kwargs_items.append(
|
||||
MultiModalKwargsItem(
|
||||
{
|
||||
"embedding": MultiModalFieldElem(
|
||||
data=tensor,
|
||||
field=MultiModalSharedField(batch_size=1),
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
pe_hashes.append(MultiModalHasher.hash_kwargs(prompt_embeds=tensor))
|
||||
# `is_embed=None` matches the existing image_embeds-style
|
||||
# "no encoder, just splice the tensor directly" semantics.
|
||||
pe_placeholders.append(
|
||||
PlaceholderRange(offset=start, length=length, is_embed=None)
|
||||
)
|
||||
|
||||
cast(
|
||||
MultiModalKwargsItems[MultiModalKwargsItem | None],
|
||||
engine_input["mm_kwargs"],
|
||||
)["prompt_embeds"] = pe_kwargs_items
|
||||
engine_input["mm_hashes"] = {
|
||||
**engine_input["mm_hashes"],
|
||||
"prompt_embeds": pe_hashes,
|
||||
}
|
||||
cast(dict, engine_input["mm_placeholders"])["prompt_embeds"] = pe_placeholders
|
||||
|
||||
@@ -38,6 +38,7 @@ class NewRequestData:
|
||||
num_computed_tokens: int
|
||||
lora_request: LoRARequest | None
|
||||
prompt_embeds: "torch.Tensor | None" = None
|
||||
prompt_is_token_ids: list[bool] | None = None
|
||||
|
||||
# Only used for v2 model runner.
|
||||
prefill_token_ids: list[int] | None = None
|
||||
@@ -59,6 +60,7 @@ class NewRequestData:
|
||||
num_computed_tokens=request.num_computed_tokens,
|
||||
lora_request=request.lora_request,
|
||||
prompt_embeds=request.prompt_embeds,
|
||||
prompt_is_token_ids=request.prompt_is_token_ids,
|
||||
prefill_token_ids=prefill_token_ids,
|
||||
)
|
||||
|
||||
|
||||
@@ -94,6 +94,12 @@ class EngineCoreRequest(
|
||||
data_parallel_rank: int | None
|
||||
prompt_embeds: torch.Tensor | None = None
|
||||
|
||||
# Per-position mask for mixed-mode inputs (e.g chat completion with
|
||||
# prompt_embeds content parts). `True` means the position is a real
|
||||
# token ID; `False` means the position uses a pre-computed entry from
|
||||
# `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
|
||||
prompt_is_token_ids: list[bool] | None = None
|
||||
|
||||
# Index of the client, used to ensure outputs are sent back to the same
|
||||
# client for this request when scaling out the front-end.
|
||||
client_index: int = 0
|
||||
|
||||
@@ -292,11 +292,13 @@ class InputProcessor:
|
||||
|
||||
# Mypy can be conservative for TypedDict unions; normalize access.
|
||||
if decoder_inputs["type"] == "embeds":
|
||||
prompt_token_ids = None
|
||||
prompt_embeds = decoder_inputs["prompt_embeds"]
|
||||
prompt_token_ids = decoder_inputs.get("prompt_token_ids")
|
||||
prompt_is_token_ids = decoder_inputs.get("is_token_ids")
|
||||
else:
|
||||
prompt_token_ids = decoder_inputs["prompt_token_ids"]
|
||||
prompt_embeds = None
|
||||
prompt_is_token_ids = None
|
||||
|
||||
sampling_params = None
|
||||
pooling_params = None
|
||||
@@ -361,6 +363,7 @@ class InputProcessor:
|
||||
request_id=request_id,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
prompt_embeds=prompt_embeds,
|
||||
prompt_is_token_ids=prompt_is_token_ids,
|
||||
mm_features=mm_features,
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=pooling_params,
|
||||
|
||||
@@ -66,6 +66,7 @@ class Request:
|
||||
client_index: int = 0,
|
||||
arrival_time: float | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
prompt_is_token_ids: list[bool] | None = None,
|
||||
mm_features: list[MultiModalFeatureSpec] | None = None,
|
||||
lora_request: "LoRARequest | None" = None,
|
||||
cache_salt: str | None = None,
|
||||
@@ -114,6 +115,10 @@ class Request:
|
||||
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
self.prompt_embeds = prompt_embeds
|
||||
# Per-position mask used in mixed-mode (chat completion with
|
||||
# prompt_embeds). `None` except when both `prompt_token_ids` and
|
||||
# `prompt_embeds` are set and their positions are interleaved.
|
||||
self.prompt_is_token_ids = prompt_is_token_ids
|
||||
# Cache per-block prompt-embed hashes to avoid rehashing the same
|
||||
# tensor slices when generating extra keys.
|
||||
self._prompt_embeds_per_block_hashes: dict[tuple[int, int], bytes] = {}
|
||||
@@ -184,6 +189,7 @@ class Request:
|
||||
client_index=request.client_index,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
prompt_embeds=request.prompt_embeds,
|
||||
prompt_is_token_ids=request.prompt_is_token_ids,
|
||||
mm_features=request.mm_features,
|
||||
sampling_params=request.sampling_params,
|
||||
pooling_params=request.pooling_params,
|
||||
|
||||
@@ -50,6 +50,10 @@ class CachedRequestState:
|
||||
lora_request: LoRARequest | None = None
|
||||
prompt_embeds: torch.Tensor | None = None
|
||||
|
||||
# Per-position mask for mixed-mode inputs (e.g chat completion with
|
||||
# prompt_embeds content parts). See `Request.prompt_is_token_ids`.
|
||||
prompt_is_token_ids: list[bool] | None = None
|
||||
|
||||
# Used when both async_scheduling and spec_decode are enabled.
|
||||
prev_num_draft_len: int = 0
|
||||
|
||||
@@ -356,7 +360,12 @@ class InputBatch:
|
||||
end_idx = start_idx + len(request.output_token_ids)
|
||||
if request.prompt_token_ids is not None:
|
||||
self.token_ids_cpu[req_index, :num_prompt_tokens] = request.prompt_token_ids
|
||||
self.is_token_ids[req_index, :num_prompt_tokens] = True
|
||||
if request.prompt_is_token_ids is not None:
|
||||
self.is_token_ids[req_index, :num_prompt_tokens] = (
|
||||
request.prompt_is_token_ids
|
||||
)
|
||||
else:
|
||||
self.is_token_ids[req_index, :num_prompt_tokens] = True
|
||||
else:
|
||||
self.is_token_ids[req_index, :num_prompt_tokens] = False
|
||||
if request.prompt_embeds is not None:
|
||||
|
||||
@@ -1163,6 +1163,7 @@ class GPUModelRunner(
|
||||
req_id=req_id,
|
||||
prompt_token_ids=new_req_data.prompt_token_ids,
|
||||
prompt_embeds=new_req_data.prompt_embeds,
|
||||
prompt_is_token_ids=new_req_data.prompt_is_token_ids,
|
||||
mm_features=new_req_data.mm_features,
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=pooling_params,
|
||||
@@ -1505,10 +1506,16 @@ class GPUModelRunner(
|
||||
)
|
||||
mrope_model = cast(SupportsMRoPE, model)
|
||||
|
||||
# `prompt_embeds` is a passthrough modality (no grid_thw), models'
|
||||
# M-RoPE code assumes per-feature grid info, so filter it out. The
|
||||
# prompt_embeds positions are treated as text positions for M-RoPE.
|
||||
mrope_features = [
|
||||
f for f in req_state.mm_features if f.modality != "prompt_embeds"
|
||||
]
|
||||
req_state.mrope_positions, req_state.mrope_position_delta = (
|
||||
mrope_model.get_mrope_input_positions(
|
||||
req_state.prompt_token_ids,
|
||||
req_state.mm_features,
|
||||
mrope_features,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -2744,6 +2751,33 @@ class GPUModelRunner(
|
||||
if not mm_kwargs:
|
||||
return []
|
||||
|
||||
# `prompt_embeds` is a passthrough modality, the tensor is already in
|
||||
# the model embedding space, so no encoder runs. Inject each
|
||||
# `prompt_embeds` tensor directly into the encoder cache here so that
|
||||
# `_gather_mm_embeddings` can splice it via the standard `is_mm_embed`
|
||||
# path.
|
||||
pe_indices = [
|
||||
i
|
||||
for i, (modality, _) in enumerate(mm_kwargs)
|
||||
if modality == "prompt_embeds"
|
||||
]
|
||||
if pe_indices:
|
||||
for i in pe_indices:
|
||||
pe_tensor = mm_kwargs[i][1]["embedding"].data
|
||||
assert isinstance(pe_tensor, torch.Tensor)
|
||||
|
||||
self.encoder_cache[mm_hashes[i]] = pe_tensor.to(self.device)
|
||||
self.maybe_save_ec_to_connector(self.encoder_cache, mm_hashes[i])
|
||||
# Filter out `prompt_embeds` items from mm_kwargs/mm_hashes/mm_lora_refs
|
||||
# since they don't require further encoder processing.
|
||||
mm_hashes = [h for i, h in enumerate(mm_hashes) if i not in pe_indices]
|
||||
mm_kwargs = [k for i, k in enumerate(mm_kwargs) if i not in pe_indices]
|
||||
mm_lora_refs = [
|
||||
r for i, r in enumerate(mm_lora_refs) if i not in pe_indices
|
||||
]
|
||||
if not mm_kwargs:
|
||||
return [] # nothing left to encode after filtering out `prompt_embeds`
|
||||
|
||||
should_time = bool(
|
||||
self.observability_config
|
||||
and self.observability_config.enable_mm_processor_stats
|
||||
|
||||
Reference in New Issue
Block a user