feat: Enable prompt_embeds Content Part Support in vLLM Chat Completions API (#40720)
Signed-off-by: Luis Robaina <luis@protopia.ai> Signed-off-by: Luis Robaina 🚀 <luisfabian1545@gmail.com> Signed-off-by: LuisRobaina <luis@protopia.ai> Co-authored-by: Andrew Sansom <qthequartermasterman@gmail.com>
This commit is contained in:
@@ -769,6 +769,8 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
return embeds_input(
|
||||
prompt_embeds=prompt_embeds,
|
||||
cache_salt=prompt.get("cache_salt"),
|
||||
prompt_token_ids=prompt.get("prompt_token_ids"),
|
||||
is_token_ids=prompt.get("prompt_is_token_ids"),
|
||||
)
|
||||
|
||||
async def _process_tokens_async(
|
||||
|
||||
@@ -7,6 +7,7 @@ import pybase64
|
||||
import torch
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.utils.async_utils import make_async
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
@@ -30,15 +31,53 @@ def safe_load_prompt_embeds(
|
||||
weights_only=True,
|
||||
map_location=torch.device("cpu"),
|
||||
)
|
||||
assert isinstance(tensor, torch.Tensor) and tensor.dtype in (
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
torch.float16,
|
||||
)
|
||||
if not isinstance(tensor, torch.Tensor):
|
||||
raise VLLMValidationError(
|
||||
"`prompt_embeds` payload did not deserialize to a torch.Tensor.",
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
tensor = tensor.to_dense()
|
||||
|
||||
if tensor.dim() > 2:
|
||||
tensor = tensor.squeeze(0)
|
||||
assert tensor.dim() == 2
|
||||
if tensor.dim() != 2:
|
||||
raise VLLMValidationError(
|
||||
"`prompt_embeds` must be a 2D tensor of shape "
|
||||
f"(num_tokens, hidden_size); got shape {tuple(tensor.shape)}.",
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
|
||||
# Pin each tensor to the model's hidden_size. Validating here
|
||||
# also transitively guarantees cross-tensor consistency for requests that
|
||||
# include multiple `prompt_embeds` parts, which is required by downstream
|
||||
# concatenation in `_build_mixed_prompt_embeds`.
|
||||
expected_hidden_size = model_config.get_hidden_size()
|
||||
if tensor.shape[1] != expected_hidden_size:
|
||||
raise VLLMValidationError(
|
||||
f"`prompt_embeds` hidden_size {tensor.shape[1]} does not match "
|
||||
f"the model's hidden_size {expected_hidden_size}.",
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
|
||||
# Cast to the model's dtype so API clients don't need to know the server's
|
||||
# `--dtype` setting ahead of time. Only floating-point source dtypes are
|
||||
# allowed. integer / bool / complex inputs almost certainly indicate caller
|
||||
# error (e.g. quantized payloads, wrong tensor), and a silent `.to()`
|
||||
# could hide a real mistake.
|
||||
expected_dtype = model_config.dtype
|
||||
if tensor.dtype != expected_dtype:
|
||||
if not tensor.is_floating_point():
|
||||
raise VLLMValidationError(
|
||||
f"`prompt_embeds` dtype {tensor.dtype} is not a floating-point "
|
||||
f"type, cannot safely cast to the model's dtype {expected_dtype}.",
|
||||
parameter="prompt_embeds",
|
||||
)
|
||||
tensor = tensor.to(expected_dtype)
|
||||
|
||||
return tensor
|
||||
|
||||
|
||||
safe_load_prompt_embeds_async = make_async(safe_load_prompt_embeds)
|
||||
"""Async variant of `safe_load_prompt_embeds` that defers the decode to a
|
||||
thread-pool executor, so the asyncio event loop is not blocked by the base64
|
||||
decode + `torch.load` work."""
|
||||
|
||||
+440
-18
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import itertools
|
||||
import weakref
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Set
|
||||
from collections.abc import Sequence
|
||||
from functools import lru_cache
|
||||
from typing import Any, Literal, cast, overload
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload
|
||||
|
||||
import jinja2
|
||||
import jinja2.ext
|
||||
@@ -13,20 +16,32 @@ import jinja2.meta
|
||||
import jinja2.nodes
|
||||
import jinja2.parser
|
||||
import jinja2.sandbox
|
||||
import torch
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormat,
|
||||
ChatTemplateContentFormatOption,
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
ChatTemplateResolutionError,
|
||||
ConversationMessage,
|
||||
load_chat_template,
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||
from vllm.inputs import EmbedsPrompt
|
||||
from vllm.inputs.engine import MultiModalInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
MultiModalSharedField,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import (
|
||||
PromptReplacement,
|
||||
apply_token_matches,
|
||||
find_mm_placeholders,
|
||||
)
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
|
||||
from vllm.transformers_utils.processor import cached_get_processor
|
||||
@@ -34,13 +49,166 @@ from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.func_utils import supports_kw
|
||||
|
||||
from .base import BaseRenderer
|
||||
from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Set
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormat,
|
||||
ChatTemplateContentFormatOption,
|
||||
ConversationMessage,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict, TokensPrompt
|
||||
from vllm.inputs.engine import TokensInput
|
||||
from vllm.multimodal.processing.processor import (
|
||||
MultiModalPromptUpdates,
|
||||
ResolvedPromptUpdate,
|
||||
)
|
||||
|
||||
from .inputs import DictPrompt
|
||||
from .params import ChatParams
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Cache of `tokenizer -> prompt_embeds placeholder token ID`. Keyed by the
|
||||
# tokenizer object (not `id(tokenizer)`) so a fresh tokenizer landing at a
|
||||
# recycled memory address can't pick up a stale tid. Entries evict atomically
|
||||
# with the tokenizer's garbage-collection.
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_CACHE: Final[
|
||||
weakref.WeakKeyDictionary[HfTokenizer, int]
|
||||
] = weakref.WeakKeyDictionary()
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_ERROR: Final[str] = (
|
||||
"Expected {token!r} to tokenize to exactly 1 token, got {num_ids} ({ids!r})."
|
||||
)
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR: Final[str] = (
|
||||
"Expected {expected} prompt_embeds placeholder spans in the "
|
||||
"tokenized prompt, found {actual}."
|
||||
)
|
||||
_MISSING_PROMPT_TOKEN_IDS_ERROR: Final[str] = (
|
||||
"Expected prompt_token_ids in rendered prompt when prompt_embeds "
|
||||
"are present. This indicates the chat template was invoked with "
|
||||
"tokenize=False."
|
||||
)
|
||||
_TOKENIZE_OVERRIDE_WARNING: Final[str] = (
|
||||
"Overriding `tokenize=False` to `True` because `prompt_embeds` "
|
||||
"post-processing requires tokenized IDs."
|
||||
)
|
||||
|
||||
|
||||
def _ensure_prompt_embeds_placeholder_token(tokenizer: HfTokenizer) -> int:
|
||||
"""Register `PROMPT_EMBEDS_PLACEHOLDER_TOKEN` as a special token and return
|
||||
its token ID."""
|
||||
cached = _PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_CACHE.get(tokenizer)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
tokenizer.add_special_tokens(
|
||||
{"additional_special_tokens": [PROMPT_EMBEDS_PLACEHOLDER_TOKEN]}
|
||||
)
|
||||
|
||||
ids = tokenizer.encode(PROMPT_EMBEDS_PLACEHOLDER_TOKEN, add_special_tokens=False)
|
||||
if len(ids) != 1:
|
||||
raise RuntimeError(
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_ERROR.format(
|
||||
token=PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
num_ids=len(ids),
|
||||
ids=ids,
|
||||
)
|
||||
)
|
||||
|
||||
token_id = ids[0]
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_TOKEN_ID_CACHE[tokenizer] = token_id
|
||||
return token_id
|
||||
|
||||
|
||||
def _build_prompt_embeds_updates(
|
||||
prompt_embeds_tensors: Sequence[torch.Tensor],
|
||||
placeholder_token_id: int,
|
||||
) -> MultiModalPromptUpdates:
|
||||
"""Build `MultiModalPromptUpdates` for `prompt_embeds` expansion.
|
||||
|
||||
Each tensor produces a `PromptReplacement` that maps
|
||||
`[placeholder_token_id]` -> `[placeholder_token_id] x N`
|
||||
(where `N = tensor.shape[0]`).
|
||||
"""
|
||||
updates: list[Sequence[ResolvedPromptUpdate]] = []
|
||||
for i, tensor in enumerate(prompt_embeds_tensors):
|
||||
update = PromptReplacement(
|
||||
modality="prompt_embeds",
|
||||
target=[placeholder_token_id],
|
||||
replacement=[placeholder_token_id] * tensor.shape[0],
|
||||
)
|
||||
updates.append([update.resolve(item_idx=i)])
|
||||
return {"prompt_embeds": updates}
|
||||
|
||||
|
||||
def _expand_prompt_embeds_placeholders(
|
||||
token_ids: list[int],
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> list[int]:
|
||||
"""Expand each 1-token `prompt_embeds` sentinel into an N-token span.
|
||||
|
||||
Uses `apply_token_matches`. Each single placeholder token in
|
||||
`token_ids` is replaced with a consecutive span of
|
||||
`tensor.shape[0]` copies, following tensors in order.
|
||||
"""
|
||||
expanded, _ = apply_token_matches(token_ids, mm_prompt_updates, tokenizer=None)
|
||||
return expanded
|
||||
|
||||
|
||||
def _build_prompt_embeds_positions(
|
||||
token_ids: list[int],
|
||||
num_tensors: int,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""Locate each prompt_embeds placeholder span in `token_ids`.
|
||||
|
||||
Expects `token_ids` to already contain expanded N-token spans.
|
||||
Returns `[(start_idx, length), ...]` aligned with the tensors.
|
||||
"""
|
||||
placeholders = find_mm_placeholders(
|
||||
prompt=token_ids,
|
||||
mm_prompt_updates=mm_prompt_updates,
|
||||
tokenizer=None,
|
||||
)
|
||||
features = placeholders.get("prompt_embeds", [])
|
||||
|
||||
if len(features) != num_tensors:
|
||||
raise ValueError(
|
||||
_PROMPT_EMBEDS_PLACEHOLDER_SPAN_MISMATCH_ERROR.format(
|
||||
expected=num_tensors,
|
||||
actual=len(features),
|
||||
)
|
||||
)
|
||||
|
||||
return [(f.start_idx, f.length) for f in features]
|
||||
|
||||
|
||||
def _build_mixed_prompt_embeds(
|
||||
token_ids: list[int],
|
||||
prompt_embeds_tensors: Sequence[torch.Tensor],
|
||||
positions: list[tuple[int, int]],
|
||||
) -> tuple[torch.Tensor, list[bool]]:
|
||||
"""Build the full-length `prompt_embeds` tensor and the `is_token_ids`
|
||||
mask aligned to `token_ids`."""
|
||||
total_len = len(token_ids)
|
||||
hidden_size = prompt_embeds_tensors[0].shape[1]
|
||||
dtype = prompt_embeds_tensors[0].dtype
|
||||
|
||||
full_embeds = torch.zeros(total_len, hidden_size, dtype=dtype)
|
||||
is_token_ids = torch.ones(total_len, dtype=torch.bool)
|
||||
|
||||
for (start, length), tensor in zip(positions, prompt_embeds_tensors, strict=True):
|
||||
full_embeds[start : start + length] = tensor
|
||||
is_token_ids[start : start + length] = False
|
||||
|
||||
return full_embeds, is_token_ids.tolist()
|
||||
|
||||
|
||||
_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], str | None]()
|
||||
"""
|
||||
Used in `_try_get_processor_chat_template` to avoid calling
|
||||
@@ -98,7 +266,7 @@ def resolve_chat_template(
|
||||
chat_template: str | None,
|
||||
tools: list[dict[str, Any]] | None,
|
||||
*,
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
) -> str | None:
|
||||
# 1st priority: The given chat template
|
||||
if chat_template is not None:
|
||||
@@ -281,7 +449,7 @@ def _resolve_chat_template_content_format(
|
||||
tools: list[dict[str, Any]] | None,
|
||||
tokenizer: HfTokenizer,
|
||||
*,
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
) -> ChatTemplateContentFormat:
|
||||
resolved_chat_template = resolve_chat_template(
|
||||
tokenizer,
|
||||
@@ -335,7 +503,7 @@ def resolve_chat_template_content_format(
|
||||
given_format: ChatTemplateContentFormatOption,
|
||||
tokenizer: HfTokenizer,
|
||||
*,
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
) -> ChatTemplateContentFormat:
|
||||
if given_format != "auto":
|
||||
return given_format
|
||||
@@ -437,7 +605,7 @@ def resolve_chat_template_kwargs(
|
||||
|
||||
@overload
|
||||
def safe_apply_chat_template(
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
tokenizer: HfTokenizer,
|
||||
conversation: list[ConversationMessage],
|
||||
*,
|
||||
@@ -448,7 +616,7 @@ def safe_apply_chat_template(
|
||||
) -> list[int]: ...
|
||||
@overload
|
||||
def safe_apply_chat_template(
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
tokenizer: HfTokenizer,
|
||||
conversation: list[ConversationMessage],
|
||||
*,
|
||||
@@ -458,7 +626,7 @@ def safe_apply_chat_template(
|
||||
**kwargs,
|
||||
) -> str: ...
|
||||
def safe_apply_chat_template(
|
||||
model_config: "ModelConfig",
|
||||
model_config: ModelConfig,
|
||||
tokenizer: HfTokenizer,
|
||||
conversation: list[ConversationMessage],
|
||||
*,
|
||||
@@ -486,6 +654,14 @@ def safe_apply_chat_template(
|
||||
chat_template_kwargs=kwargs,
|
||||
)
|
||||
|
||||
# transformers v5 changed the default of `return_dict` to True, which
|
||||
# makes `apply_chat_template(tokenize=True)` return a `BatchEncoding`
|
||||
# instead of `list[int]`. Force `return_dict=False` so downstream code
|
||||
# that expects a flat token list (e.g. `parse_dec_only_prompt`) works
|
||||
# consistently across v4 and v5.
|
||||
if tokenize and "return_dict" not in resolved_kwargs:
|
||||
resolved_kwargs["return_dict"] = False
|
||||
|
||||
try:
|
||||
return tokenizer.apply_chat_template(
|
||||
conversation=conversation, # type: ignore[arg-type]
|
||||
@@ -627,6 +803,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
model_config = self.model_config
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
prompt_embeds_placeholder_token_id: int | None = None
|
||||
if model_config.enable_prompt_embeds:
|
||||
prompt_embeds_placeholder_token_id = (
|
||||
_ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
)
|
||||
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
model_config,
|
||||
@@ -641,11 +823,30 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
mm_processor_kwargs=params.mm_processor_kwargs,
|
||||
)
|
||||
|
||||
# prompt_embeds tensors are carried by the tracker through mm_data,
|
||||
# but they must NOT be fed to the MM processor (which would reject
|
||||
# the unknown key). Extract them here.
|
||||
prompt_embeds_tensors: list[torch.Tensor] | None = None
|
||||
if mm_data is not None and "prompt_embeds" in mm_data:
|
||||
prompt_embeds_tensors = list(
|
||||
cast(Sequence[torch.Tensor], mm_data["prompt_embeds"])
|
||||
)
|
||||
mm_data = {k: v for k, v in mm_data.items() if k != "prompt_embeds"}
|
||||
if not mm_data:
|
||||
mm_data = None
|
||||
|
||||
chat_template_kwargs = params.get_apply_chat_template_kwargs()
|
||||
if prompt_embeds_tensors:
|
||||
# prompt_embeds post-processing requires prompt_token_ids.
|
||||
if chat_template_kwargs.get("tokenize") is False:
|
||||
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
|
||||
chat_template_kwargs["tokenize"] = True
|
||||
|
||||
prompt_raw = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
@@ -671,6 +872,29 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
)
|
||||
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
|
||||
# When `prompt_embeds` is mixed with other modality data,
|
||||
# `_process_tokens` runs `_process_multimodal` first (expanding
|
||||
# `<|AUDIO|>` / `<|IMAGE|>` placeholders) and then
|
||||
# `_apply_prompt_embeds_to_engine_input` augments the result.
|
||||
# Stash the tensors and placeholder ID for that override to consume.
|
||||
if prompt_embeds_tensors and mm_data:
|
||||
assert prompt_embeds_placeholder_token_id is not None
|
||||
cast(dict, prompt)["_prompt_embeds"] = (
|
||||
prompt_embeds_tensors,
|
||||
prompt_embeds_placeholder_token_id,
|
||||
)
|
||||
if params.mm_processor_kwargs:
|
||||
cast(dict, prompt)["mm_processor_kwargs"] = params.mm_processor_kwargs
|
||||
elif prompt_embeds_tensors:
|
||||
# Pure mode: no other MM data, mutate prompt to EmbedsPrompt shape.
|
||||
assert prompt_embeds_placeholder_token_id is not None
|
||||
self._apply_prompt_embeds_to_prompt(
|
||||
prompt,
|
||||
prompt_embeds_tensors,
|
||||
prompt_embeds_placeholder_token_id,
|
||||
)
|
||||
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
@@ -686,6 +910,12 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
model_config = self.model_config
|
||||
tokenizer = self.get_tokenizer()
|
||||
|
||||
prompt_embeds_placeholder_token_id: int | None = None
|
||||
if model_config.enable_prompt_embeds:
|
||||
prompt_embeds_placeholder_token_id = (
|
||||
_ensure_prompt_embeds_placeholder_token(tokenizer)
|
||||
)
|
||||
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
model_config,
|
||||
@@ -700,11 +930,27 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
mm_processor_kwargs=params.mm_processor_kwargs,
|
||||
)
|
||||
|
||||
prompt_embeds_tensors: list[torch.Tensor] | None = None
|
||||
if mm_data is not None and "prompt_embeds" in mm_data:
|
||||
prompt_embeds_tensors = list(
|
||||
cast(Sequence[torch.Tensor], mm_data["prompt_embeds"])
|
||||
)
|
||||
mm_data = {k: v for k, v in mm_data.items() if k != "prompt_embeds"}
|
||||
if not mm_data:
|
||||
mm_data = None
|
||||
|
||||
chat_template_kwargs = params.get_apply_chat_template_kwargs()
|
||||
if prompt_embeds_tensors:
|
||||
# prompt_embeds post-processing requires prompt_token_ids.
|
||||
if chat_template_kwargs.get("tokenize") is False:
|
||||
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
|
||||
chat_template_kwargs["tokenize"] = True
|
||||
|
||||
prompt_raw = await self._apply_chat_template_async(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**params.get_apply_chat_template_kwargs(),
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
@@ -728,9 +974,185 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
)
|
||||
|
||||
prompt = parse_dec_only_prompt(prompt_raw)
|
||||
|
||||
# See `render_messages` for the rationale.
|
||||
if prompt_embeds_tensors and mm_data:
|
||||
assert prompt_embeds_placeholder_token_id is not None
|
||||
cast(dict, prompt)["_prompt_embeds"] = (
|
||||
prompt_embeds_tensors,
|
||||
prompt_embeds_placeholder_token_id,
|
||||
)
|
||||
if params.mm_processor_kwargs:
|
||||
cast(dict, prompt)["mm_processor_kwargs"] = params.mm_processor_kwargs
|
||||
elif prompt_embeds_tensors:
|
||||
assert prompt_embeds_placeholder_token_id is not None
|
||||
self._apply_prompt_embeds_to_prompt(
|
||||
prompt,
|
||||
prompt_embeds_tensors,
|
||||
prompt_embeds_placeholder_token_id,
|
||||
)
|
||||
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt
|
||||
|
||||
@override
|
||||
def _process_tokens(
|
||||
self,
|
||||
prompt: TokensPrompt,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> TokensInput | MultiModalInput:
|
||||
"""Pre-expand `prompt_embeds` sentinels before delegating to the MM
|
||||
processor, then attach `prompt_embeds` modality data to the result.
|
||||
|
||||
Mixed mode only: the `_prompt_embeds` stash is set by
|
||||
`render_messages` when `prompt_embeds` co-exist with other MM data
|
||||
(images, audio, …). We expand each 1-token sentinel to an N-token
|
||||
span *before* calling `super()._process_tokens()` so the MM
|
||||
processor records all placeholder offsets in the final (post-expansion)
|
||||
coordinate space, no offset shifting needed afterwards.
|
||||
"""
|
||||
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
|
||||
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
|
||||
list(prompt["prompt_token_ids"]), mm_updates
|
||||
)
|
||||
engine_input = super()._process_tokens(prompt, skip_mm_cache=skip_mm_cache)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, _ = prompt_embeds_info
|
||||
self._apply_prompt_embeds_to_engine_input(
|
||||
cast(MultiModalInput, engine_input),
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
return engine_input
|
||||
|
||||
@override
|
||||
async def _process_tokens_async(
|
||||
self,
|
||||
prompt: TokensPrompt,
|
||||
*,
|
||||
skip_mm_cache: bool = False,
|
||||
) -> TokensInput | MultiModalInput:
|
||||
"""Async equivalent of `_process_tokens`."""
|
||||
prompt_embeds_info = cast(dict, prompt).pop("_prompt_embeds", None)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, placeholder_token_id = prompt_embeds_info
|
||||
mm_updates = _build_prompt_embeds_updates(tensors, placeholder_token_id)
|
||||
cast(dict, prompt)["prompt_token_ids"] = _expand_prompt_embeds_placeholders(
|
||||
list(prompt["prompt_token_ids"]), mm_updates
|
||||
)
|
||||
engine_input = await super()._process_tokens_async(
|
||||
prompt, skip_mm_cache=skip_mm_cache
|
||||
)
|
||||
if prompt_embeds_info is not None:
|
||||
tensors, _ = prompt_embeds_info
|
||||
self._apply_prompt_embeds_to_engine_input(
|
||||
cast(MultiModalInput, engine_input),
|
||||
tensors,
|
||||
mm_updates,
|
||||
)
|
||||
return engine_input
|
||||
|
||||
@staticmethod
|
||||
def _apply_prompt_embeds_to_prompt(
|
||||
prompt: DictPrompt,
|
||||
prompt_embeds_tensors: list[torch.Tensor],
|
||||
placeholder_token_id: int,
|
||||
) -> None:
|
||||
"""Mutate `prompt` from `TokensPrompt` to `EmbedsPrompt` shape.
|
||||
|
||||
Pure `prompt_embeds` path only (no other MM modalities). Expands
|
||||
each `<prompt_embeds>` sentinel token into an N-token span and builds
|
||||
the full-length `prompt_embeds` tensor + `prompt_is_token_ids` mask
|
||||
that the engine's `enable_prompt_embeds` worker branch consumes.
|
||||
"""
|
||||
token_ids = cast(list[int] | None, prompt.get("prompt_token_ids"))
|
||||
if token_ids is None:
|
||||
raise RuntimeError(_MISSING_PROMPT_TOKEN_IDS_ERROR)
|
||||
|
||||
embeds_orig_positions: list[int] = [
|
||||
i for i, tok in enumerate(token_ids) if tok == placeholder_token_id
|
||||
]
|
||||
if len(embeds_orig_positions) != len(prompt_embeds_tensors):
|
||||
raise ValueError(
|
||||
f"Expected {len(prompt_embeds_tensors)} prompt_embeds "
|
||||
f"placeholder tokens in the rendered prompt, found "
|
||||
f"{len(embeds_orig_positions)}."
|
||||
)
|
||||
|
||||
mm_updates = _build_prompt_embeds_updates(
|
||||
prompt_embeds_tensors, placeholder_token_id
|
||||
)
|
||||
expanded = _expand_prompt_embeds_placeholders(token_ids, mm_updates)
|
||||
positions = _build_prompt_embeds_positions(
|
||||
expanded, len(prompt_embeds_tensors), mm_updates
|
||||
)
|
||||
|
||||
embeds_prompt = cast(EmbedsPrompt, prompt)
|
||||
embeds_prompt["prompt_token_ids"] = expanded
|
||||
full_embeds, is_token_ids_mask = _build_mixed_prompt_embeds(
|
||||
expanded, prompt_embeds_tensors, positions
|
||||
)
|
||||
embeds_prompt["prompt_embeds"] = full_embeds
|
||||
embeds_prompt["prompt_is_token_ids"] = is_token_ids_mask
|
||||
|
||||
@staticmethod
|
||||
def _apply_prompt_embeds_to_engine_input(
|
||||
engine_input: MultiModalInput,
|
||||
prompt_embeds_tensors: list[torch.Tensor],
|
||||
mm_updates: MultiModalPromptUpdates,
|
||||
) -> None:
|
||||
"""Augment `engine_input` in-place with a `prompt_embeds` modality.
|
||||
|
||||
Mixed mode: called after `_process_multimodal` has already run on the
|
||||
pre-expanded token IDs (expansion was done in `_process_tokens` before
|
||||
calling `super()`). Locates the already-expanded `prompt_embeds` spans
|
||||
and adds `prompt_embeds` entries to `mm_kwargs`, `mm_hashes`, and
|
||||
`mm_placeholders`.
|
||||
"""
|
||||
# token_ids already contain the pre-expanded N-token spans.
|
||||
token_ids = list(engine_input["prompt_token_ids"])
|
||||
|
||||
positions = _build_prompt_embeds_positions(
|
||||
token_ids, len(prompt_embeds_tensors), mm_updates
|
||||
)
|
||||
|
||||
pe_kwargs_items: list[MultiModalKwargsItem] = []
|
||||
pe_hashes: list[str] = []
|
||||
pe_placeholders: list[PlaceholderRange] = []
|
||||
for tensor, (start, length) in zip(
|
||||
prompt_embeds_tensors, positions, strict=True
|
||||
):
|
||||
pe_kwargs_items.append(
|
||||
MultiModalKwargsItem(
|
||||
{
|
||||
"embedding": MultiModalFieldElem(
|
||||
data=tensor,
|
||||
field=MultiModalSharedField(batch_size=1),
|
||||
)
|
||||
}
|
||||
)
|
||||
)
|
||||
pe_hashes.append(MultiModalHasher.hash_kwargs(prompt_embeds=tensor))
|
||||
# `is_embed=None` matches the existing image_embeds-style
|
||||
# "no encoder, just splice the tensor directly" semantics.
|
||||
pe_placeholders.append(
|
||||
PlaceholderRange(offset=start, length=length, is_embed=None)
|
||||
)
|
||||
|
||||
cast(
|
||||
MultiModalKwargsItems[MultiModalKwargsItem | None],
|
||||
engine_input["mm_kwargs"],
|
||||
)["prompt_embeds"] = pe_kwargs_items
|
||||
engine_input["mm_hashes"] = {
|
||||
**engine_input["mm_hashes"],
|
||||
"prompt_embeds": pe_hashes,
|
||||
}
|
||||
cast(dict, engine_input["mm_placeholders"])["prompt_embeds"] = pe_placeholders
|
||||
|
||||
Reference in New Issue
Block a user