[3/N] Harden Transformers modelling backend multi-modal path (#51827)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-20 10:24:41 +00:00
committed by GitHub
parent 1eab6fef01
commit 6e85feb1b7
4 changed files with 838 additions and 161 deletions
@@ -22,6 +22,8 @@ from contextlib import ExitStack, contextmanager
from typing import TYPE_CHECKING, Any
import torch
import transformers
from packaging.version import Version
from vllm.compilation.decorators import should_torch_compile_mm_encoder
from vllm.config.utils import getattr_iter
@@ -50,7 +52,9 @@ from vllm.multimodal.processing import (
BaseMultiModalProcessor,
BaseProcessingInfo,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
TimingContext,
)
from vllm.sequence import IntermediateTensors
@@ -65,6 +69,12 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
_MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3}
_MODALITY_SIZE_KEYS = {"audio": "num_audio_tokens", "image": "num_image_patches"}
def _get_embed_token_id(replacement_ids: torch.Tensor) -> int:
"""The token an expansion repeats is the one holding the embeddings."""
return int(replacement_ids.mode().values)
class MultiModalProcessingInfo(BaseProcessingInfo):
@@ -94,21 +104,6 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
)
return modalities
def _get_audio_token_id(self) -> int:
processor = self.get_hf_processor()
if hasattr(processor, "audio_token_id"):
return processor.audio_token_id
config = self.get_hf_config()
val = getattr_iter(config, ("audio_token_id", "audio_token_index"))
if val is not None:
return val
if hasattr(processor, "audio_token"):
tokenizer = self.get_tokenizer()
vocab = tokenizer.get_vocab()
if processor.audio_token in vocab:
return vocab[processor.audio_token]
raise ValueError("Cannot find audio_token_id on processor or model config")
def _get_audio_sampling_rate(self) -> float:
sub = self._get_audio_processor()
if sub is not None and hasattr(sub, "sampling_rate"):
@@ -169,7 +164,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
processor = self.info.get_hf_processor()
audio_token = getattr(processor, "audio_token", "")
# Separated so that `_apply_audio` can tell the placeholders apart
# Separated so that adjacent placeholders stay distinguishable
text += " ".join([audio_token] * num_audios)
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
processor = self.info.get_hf_processor()
@@ -210,23 +205,26 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
return data
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
def _get_prompt_updates(
class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]):
"""Processing common to both Transformers backend processors: calling the HF
processor, sizing images, and attributing its outputs to a modality.
Subclasses add the strategy for locating placeholders in the prompt.
"""
def _get_hf_mm_data(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
"""No updates: `apply` locates placeholders via `mm_token_type_ids` instead.
HF processors have no generic contract for the token sequence they insert,
so it cannot be expressed as a `PromptUpdate`. Returning nothing is only
safe because `apply` is overridden; the base class would reject it.
"""
return []
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""Rename the parser's `audios` key to the `audio` argument HF audio
processors take."""
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
if self.info._is_audio_model() and "audios" in processor_data:
processor_data["audio"] = processor_data.pop("audios")
return processor_data, passthrough_data
def _get_modality_field_names(self, modality: str) -> set[str]:
"""Field names the sub-processor for `modality` produces."""
"""Names of the fields the sub-processor for `modality` produces."""
# TODO: use else branch only once huggingface/transformers#44394 lands.
if modality == "audio":
sub_processor = self.info._get_audio_processor()
@@ -273,6 +271,18 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
return owned
def _get_slice_dim(self, data: Any, total_rows: int) -> int:
"""Which dimension of a field holds the rows belonging to each item.
Some processors (e.g., Idefics3) return image fields with a leading batch
dimension, putting the rows one dimension further in.
"""
if not isinstance(data, torch.Tensor) or data.ndim < 2:
return 0
if data.shape[0] != total_rows and data.shape[1] == total_rows:
return 1
return 0
def _get_mm_fields_config(
self,
hf_inputs: "BatchFeature",
@@ -281,29 +291,50 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
# HF Processors always return a mask but vLLM doesn't need it
hf_inputs.pop("attention_mask", None)
# Written by `_apply_audio`/`_apply_vision`; absent if the modality had no items
# Absent if the modality had no items
sizes = {
"audio": hf_inputs.get("num_audio_tokens"),
"image": hf_inputs.get("num_image_patches"),
modality: hf_inputs.get(key)
for modality, key in _MODALITY_SIZE_KEYS.items()
}
modalities = [m for m, size in sizes.items() if size is not None]
size_keys = {"num_audio_tokens", "num_image_patches"}
keys = [key for key in hf_inputs if key not in size_keys]
# Keys we wrote ourselves, rather than ones a sub-processor produced
own_keys = set(_MODALITY_SIZE_KEYS.values()) | {
f"{modality}_replacement_{suffix}"
for modality in modalities
for suffix in ("ids", "sizes")
}
keys = [key for key in hf_inputs if key not in own_keys]
owned = self._partition_keys_by_modality(keys, modalities)
# Un-padded fields are already one entry per item, so index rather than slice
mm_fields: dict[str, MultiModalFieldConfig] = {
key: MultiModalFieldConfig.flat_from_sizes(modality, sizes[modality])
key: MultiModalFieldConfig.batched(modality)
if modality == "audio" or isinstance(hf_inputs[key], list)
else MultiModalFieldConfig.flat_from_sizes(
modality,
sizes[modality],
dim=self._get_slice_dim(hf_inputs[key], int(sizes[modality].sum())),
)
for modality in modalities
for key in owned[modality]
}
# Keep these as batched, as they always have batch size as first dim
if "audio" in modalities:
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched(
"audio", keep_on_cpu=True
for modality in modalities:
# One row per item, and only ever read on the CPU
mm_fields[_MODALITY_SIZE_KEYS[modality]] = MultiModalFieldConfig.batched(
modality, keep_on_cpu=True
)
replacement_sizes = hf_inputs.get(f"{modality}_replacement_sizes")
if replacement_sizes is not None:
mm_fields[f"{modality}_replacement_ids"] = (
MultiModalFieldConfig.flat_from_sizes(
modality, replacement_sizes, keep_on_cpu=True
)
)
if "image" in modalities:
# Always one row per item, whatever they describe
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
@@ -311,25 +342,154 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
return mm_fields
def _get_hf_mm_data(
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Run the HF processor and unpad inputs."""
try:
hf_inputs = super()._call_hf_processor(
prompt, mm_data, mm_kwargs, tok_kwargs
)
except ValueError:
if any(mm_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs)
hf_inputs = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs)
return hf_inputs
def _unpad_audios(
self,
hf_inputs: "BatchFeature",
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> None:
"""Replace the audio fields with each audio processed on its own.
Processors pad every audio up to the longest in the call, which would leave
an item's data dependent on what it was processed with. Unlike images,
nothing in the output states how long each one really is, and processors
pad a lone audio too, so the only way to know what an audio produces by
itself is to process it by itself.
"""
audios = mm_data.get("audio")
if not audios or len({len(audio) for audio in audios}) == 1:
return
alone = [
self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**mm_kwargs),
dict(
text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
),
dict(**mm_kwargs, **tok_kwargs),
)
for audio in audios
]
for key in self._get_modality_field_names("audio"):
if isinstance(hf_inputs.get(key), torch.Tensor):
hf_inputs[key] = [output[key][0] for output in alone]
def _unpad_images(self, hf_inputs: "BatchFeature") -> None:
"""Trim each image back to its own size when the processor padded them all
to the largest in the batch.
An image's data has to depend on nothing but that image, or the multi-modal
processor cache would store it under that image's hash and later reuse it
beside a different neighbour. Padding is re-applied when the encoder runs.
"""
pixel_values = hf_inputs.get("pixel_values")
image_sizes = hf_inputs.get("image_sizes")
if not isinstance(pixel_values, torch.Tensor):
return
if not isinstance(image_sizes, torch.Tensor):
return
if pixel_values.ndim != 4 or len(pixel_values) != len(image_sizes):
return
# The sizes describe the trailing dimensions only if the largest of them is
# what the batch was padded up to. Otherwise they mean something else, as
# in llava-onevision, where they are the sizes before any processing.
maxima = image_sizes.max(dim=0).values
if maxima.tolist() != list(pixel_values.shape[-2:]):
return
hf_inputs["pixel_values"] = [
image[..., :height, :width]
for image, (height, width) in zip(pixel_values, image_sizes.tolist())
]
class LegacyMultiModalProcessor(_MultiModalProcessorBase):
"""Locates placeholders by searching the prompt the HF processor has already
expanded for the tokens of each modality.
Serves transformers versions with no `return_text_replacement_offsets`.
Placeholders found this way cannot be rebuilt from an unexpanded prompt, so
this processor overrides `apply` and gets no multi-modal processor cache.
Remove it once `requirements/common.txt` requires `transformers>=5.15.0`.
"""
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
"""Empty, because `apply` writes the placeholder ranges itself rather than
deriving them from updates."""
return []
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask which modality owns each token of the expanded prompt, which is the
only marking of the tokens an expansion adds around an item."""
if any(mm_data.values()):
mm_data = {**mm_data, "return_mm_token_type_ids": True}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
def _get_mm_token_ids(self, modality: str) -> list[int]:
"""Token ids marking where `modality` sits in the prompt, which for some
processors differ from the placeholder written into it.
The expanded prompt is all this path has to go on, so it takes the
processor at its word about which tokens belong to the modality.
"""
In contrast to the base class, this method requests
`return_mm_token_type_ids` and remaps the `audios` key to `audio` for
audio models.
"""
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
if self.info._is_audio_model() and "audios" in processor_data:
processor_data["audio"] = processor_data.pop("audios")
processor_data["return_mm_token_type_ids"] = True
return processor_data, passthrough_data
info = self.info
processor = info.get_hf_processor()
declared = getattr(processor, f"{modality}_token_ids", None) or ()
if ids := [token_id for token_id in declared if token_id is not None]:
return ids
config = info.get_hf_config()
names = (f"{modality}_token_id", f"{modality}_token_index")
token_id = getattr(processor, names[0], getattr_iter(config, names))
if token_id is None:
token = getattr(processor, f"{modality}_token", None)
token_id = info.get_tokenizer().get_vocab().get(token)
if token_id is None:
raise ValueError(
f"Cannot find {modality}_token_id on processor or model config"
)
return [token_id]
def _apply_audio(
self,
@@ -337,12 +497,17 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
processed_data: "BatchFeature",
num_audios: int,
) -> dict[str, list[PlaceholderRange]]:
audio_token_id = self.info._get_audio_token_id()
"""Take each contiguous run of the audio token as one item's placeholder,
and record how many tokens the run holds."""
audio_token_ids = self._get_mm_token_ids("audio")
prompt_tensor = torch.tensor(prompt_ids)
is_audio = prompt_tensor == audio_token_id
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
if not is_audio.any():
return {}
raise ValueError(
f"{num_audios} audio item(s) were passed but the prompt "
"contains no audio token. Add one placeholder per audio item."
)
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
transitions = padded.int().diff()
@@ -365,6 +530,23 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
processed_data["num_audio_tokens"] = lengths
return {"audio": ranges}
def _get_num_multimodal_tokens(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, list[int]]:
"""Ask the HF processor how many tokens and patches each image expands to."""
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
(size.height, size.width)
for size in map(images.get_image_size, range(len(images)))
]
return processor._get_num_multimodal_tokens(
image_sizes=image_sizes,
**self.info.ctx.get_merged_mm_kwargs({}),
)
def _apply_vision(
self,
prompt_ids: list[int],
@@ -373,47 +555,46 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
hf_processor_mm_kwargs: Mapping[str, object],
mm_token_type_ids: torch.Tensor | None,
) -> dict[str, list[PlaceholderRange]]:
# Placeholders can't be located without them, so give up rather than guess
if mm_token_type_ids is None:
return {}
"""Split the positions the processor marks as image into one placeholder per
item, sized by the token count it reports for each image."""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if mm_token_type_ids is None:
raise ValueError(
f"{type(hf_processor).__name__} returned no `mm_token_type_ids`, so "
"the Transformers modeling backend cannot locate the placeholder of "
"each image."
)
# We can infer vLLM style placeholder from token type ids, if we split
# it for each input `mm_data`.
mm_positions = torch.where(mm_token_type_ids == 1)[1]
images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = []
for item_idx in range(len(images)):
image_size = images.get_image_size(item_idx)
image_sizes.append((image_size.height, image_size.width))
mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
image_sizes=image_sizes,
**self.info.ctx.get_merged_mm_kwargs({}),
# These mark the tokens structuring an image as well as the image itself,
# which is what the counts below are over
mm_positions = torch.where(mm_token_type_ids[0] == 1)[0]
mm_tokens_per_modality = self._get_num_multimodal_tokens(
mm_items, hf_processor_mm_kwargs
)
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
split_sizes = mm_tokens_per_modality["num_image_tokens"]
if split_sizes:
image_token_ids = getattr(hf_processor, "image_token_ids", None)
if image_token_ids is None:
# Transformers <5.10.0
image_token_ids = [hf_processor.image_token_id]
image_token_ids = torch.tensor(
[i for i in image_token_ids if i is not None]
if sum(split_sizes) != len(mm_positions):
raise ValueError(
f"The expanded prompt holds {len(mm_positions)} image token(s) but "
f"{type(hf_processor).__name__} accounts for {sum(split_sizes)} "
f"across {mm_items.get_count('image')} image item(s)."
)
chunked_mm_positions = torch.split(mm_positions, split_sizes)
mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
if split_sizes:
image_token_ids = torch.tensor(self._get_mm_token_ids("image"))
mm_tokens = torch.tensor(prompt_ids)[mm_positions]
ranges = [
PlaceholderRange(
offset=positions[0].item(),
length=positions.shape[0],
is_embed=torch.isin(mm_tokens, image_token_ids),
# Only some of the span carries embeddings
is_embed=torch.isin(tokens, image_token_ids),
)
for positions, tokens in zip(
torch.split(mm_positions, split_sizes),
torch.split(mm_tokens, split_sizes),
)
for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
]
mm_placeholders = {"image": ranges}
@@ -427,12 +608,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalInput:
"""
Process multi-modal inputs to be used in vLLM.
Apply HF Processor on prompt text and multi-modal data together,
outputting token IDs and processed tensors.
"""
"""Process the prompt and every multi-modal item in one HF processor call,
then read the placeholder ranges out of the token ids it returns."""
prompt = inputs.prompt
mm_items = inputs.mm_data_items
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
@@ -468,7 +645,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
self.info.ctx.get_mm_config().mm_hasher_algorithm,
)
# For gemma3 we check `token_type_ids` as the key
# Gemma3 reports them under the key the model uses for its own token types
mm_token_type_ids = processed_data.pop("token_type_ids", None)
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
@@ -516,6 +693,201 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
)
class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"""Locates placeholders from the `text_replacement_offsets` the HF processor
reports, expressing each one as a `PromptUpdate`.
Stating the expansion as an update is what lets it be rebuilt from an
unexpanded prompt, so this processor takes the base class's processing path
and with it the multi-modal processor cache.
"""
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
"""Replace each modality's placeholder token with the token ids that item's
replacement text encodes to, marking which of them hold embeddings."""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
updates = []
for modality, items in out_mm_kwargs.items():
# Popped so they are neither cached nor sent to the model; the updates
# they produce are cached alongside the item instead
replacements = [
PromptUpdateDetails.select_token_id(
(ids := item.pop(f"{modality}_replacement_ids").data).tolist(),
_get_embed_token_id(ids),
)
for item in items
]
updates.append(
PromptReplacement(
modality=modality,
target=getattr(hf_processor, f"{modality}_token"),
replacement=replacements.__getitem__,
)
)
return updates
def _apply_hf_processor_main(
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
*,
enable_hf_prompt_update: bool,
) -> tuple[list[int], "BatchFeature", bool]:
"""Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice
in, whatever `enable_hf_prompt_update` asks for.
This differs from `super()` only for a text prompt with
`enable_hf_prompt_update`, where `super()` would keep the expanded token
ids the HF processor returns. Both routes give the same ids, so forcing
this one costs an extra processor call and buys the guarantee that a
request bypassing the cache is identical to a cached one.
A text prompt only arrives with `enable_hf_prompt_update` when there is
no multi-modal processor cache. Requests carrying pre-computed
embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported
here, so they fail validation shortly afterwards.
"""
if isinstance(prompt, str) and enable_hf_prompt_update:
logger.warning_once(
"Disabling the multi-modal processor cache is extra slow with the "
"Transformers modeling backend: the prompt is still tokenized "
"unexpanded and the expansion spliced in, to keep the token ids "
"identical to what the cache produces, which costs an extra HF "
"processor call per request."
)
return super()._apply_hf_processor_main(
prompt,
mm_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
enable_hf_prompt_update=False,
)
def _get_num_image_patches(
self,
hf_inputs: "BatchFeature",
mm_data: Mapping[str, object],
num_images: int,
) -> torch.Tensor:
"""How many rows of the image fields belong to each image.
Taken from whichever per-image count the processor reports,
and checked against the data it has to slice.
"""
if (grid := hf_inputs.get("image_grid_thw")) is not None:
num_patches = grid.prod(-1)
elif (counts := self._get_num_patches_per_image(mm_data)) is not None:
num_patches = torch.tensor(counts)
else:
num_patches = torch.ones(num_images, dtype=torch.long)
image_data = hf_inputs.get("pixel_values", hf_inputs.get("image_patches"))
if isinstance(image_data, torch.Tensor):
total = int(num_patches.sum())
rows = image_data.shape[self._get_slice_dim(image_data, total)]
if rows != total:
raise ValueError(
f"{type(self.info.get_hf_processor()).__name__} returned "
f"{rows} row(s) of image data for {num_images} image(s), which "
f"cannot be split into the {num_patches.tolist()} row(s) per "
"image derived from its outputs, so the rows cannot be "
"attributed to an image. Gemma3 does this when "
"`do_pan_and_scan` crops an image."
)
return num_patches
def _get_num_patches_per_image(
self, mm_data: Mapping[str, object]
) -> list[int] | None:
"""Ask the HF processor how many rows of image data each image produces."""
images = mm_data.get("images")
if not images:
return None
try:
sizes = [(image.height, image.width) for image in images]
mm_tokens = self.info.get_hf_processor()._get_num_multimodal_tokens(
image_sizes=sizes, **self.info.ctx.get_merged_mm_kwargs({})
)
return list(mm_tokens["num_image_patches"])
except (AttributeError, KeyError, TypeError):
return None
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask for the replacement each placeholder expands to, and record it as
per-item fields: its token ids, and the tokens or patches behind them."""
if has_mm_data := any(mm_data.values()):
mm_data = {**mm_data, "return_text_replacement_offsets": True}
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
# Drop the inputs the model would reject
hf_inputs.pop("mm_token_type_ids", None)
hf_inputs.pop("token_type_ids", None)
offsets = hf_inputs.pop("text_replacement_offsets", None)
# Some processors return an empty batch as a tensor rather than a list
if offsets is None or len(offsets) == 0 or len(offsets[0]) == 0:
if has_mm_data:
raise ValueError(
f"{type(self.info.get_hf_processor()).__name__} returned no "
"text replacement offsets, so the Transformers modeling backend "
"cannot locate the placeholder of each item. Its `__call__` has "
"to reach `ProcessorMixin.get_text_with_replacements` with one "
"replacement per item, which usually means implementing "
"`replace_<modality>_token`. Please report this to transformers "
"so it can be fixed, and install `transformers<5.15.0` in the "
"meantime to locate placeholders in the expanded prompt instead."
)
return hf_inputs
tokenizer = self.info.get_tokenizer()
replacements = defaultdict[str, list[list[int]]](list)
for entry in offsets[0]:
replacements[entry["type"]].append(
tokenizer.encode(entry["replacement"], add_special_tokens=False)
)
for modality, seqs in replacements.items():
hf_inputs[f"{modality}_replacement_ids"] = torch.tensor(
[token_id for seq in seqs for token_id in seq]
)
hf_inputs[f"{modality}_replacement_sizes"] = torch.tensor(
[len(seq) for seq in seqs]
)
if modality == "image":
hf_inputs["num_image_patches"] = self._get_num_image_patches(
hf_inputs, mm_data, len(seqs)
)
elif modality == "audio":
counts = []
for seq in seqs:
ids = torch.tensor(seq)
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
return hf_inputs
# From this version on, a processor reporting no offsets is an error rather than a
# fallback to searching the expanded prompt
MultiModalProcessor = (
LegacyMultiModalProcessor
if Version(transformers.__version__) < Version("5.15.0")
else OffsetsMultiModalProcessor
)
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
# Skip SupportsMRoPE.__init__ and call the next class in MRO
@@ -733,18 +1105,32 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
kwargs.pop("token_type_ids", None)
kwargs.pop("mm_token_type_ids", None)
# HuggingFace's `get_audio_features` implementations branch on
# per-sample feature lengths internally.
with gpu_sync_allowed():
audio_output = self.model.get_audio_features(
input_features, return_dict=True, **kwargs
)
audio_embeddings = audio_output.pooler_output
# Per-audio token counts are needed as Python ints to split.
with gpu_sync_allowed():
split_sizes = num_audio_tokens.flatten().tolist()
return self._split_embeddings(audio_embeddings, split_sizes)
if isinstance(input_features, torch.Tensor):
# HuggingFace's `get_audio_features` implementations branch on
# per-sample feature lengths internally.
with gpu_sync_allowed():
audio_output = self.model.get_audio_features(
input_features, return_dict=True, **kwargs
)
return self._split_embeddings(audio_output.pooler_output, split_sizes)
# Audios the processor left un-padded arrive as a list once their
# lengths differ. Encode them one at a time so that none of them is
# padded to match another.
embeddings: list[torch.Tensor] = []
for index, features in enumerate(input_features):
audio_output = self.model.get_audio_features(
features.unsqueeze(0),
return_dict=True,
**self._select_item_kwargs(kwargs, index, len(input_features)),
)
embeddings.extend(
self._split_embeddings(audio_output.pooler_output, [split_sizes[index]])
)
return embeddings
def _process_image_input(self, **kwargs) -> list[torch.Tensor] | None:
pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
@@ -761,6 +1147,42 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
num_image_patches = kwargs.pop("num_image_patches")
split_sizes = num_image_patches.flatten().tolist()
if isinstance(pixel_values, torch.Tensor):
vision_embeddings = self._get_image_features(pixel_values, **kwargs)
if isinstance(vision_embeddings, torch.Tensor):
return self._split_embeddings(vision_embeddings, split_sizes)
return list(vision_embeddings)
# Images the processor left un-padded arrive as a list once their
# shapes differ. Encode them one at a time so that none of them is
# padded to match another.
embeddings: list[torch.Tensor] = []
for index, image in enumerate(pixel_values):
features = self._get_image_features(
image.unsqueeze(0),
**self._select_item_kwargs(kwargs, index, len(pixel_values)),
)
# Encoders which return one entry per image return a single entry
if not isinstance(features, torch.Tensor):
features = torch.cat(list(features))
embeddings.extend(self._split_embeddings(features, [split_sizes[index]]))
return embeddings
def _select_item_kwargs(
self, kwargs: dict[str, Any], index: int, num_items: int
) -> dict[str, Any]:
"""Narrow the entries of `kwargs` that hold one row per item down to the item
at `index`. Length is all there is to match on, so an unrelated entry of the
same length is narrowed too."""
return {
key: value[index : index + 1]
if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
else value
for key, value in kwargs.items()
}
def _get_image_features(self, pixel_values: torch.Tensor, **kwargs) -> Any:
# grid_thw fields are registered keep_on_cpu; restore the on-device
# placement that HF get_image_features implementations expect.
for key, value in kwargs.items():
@@ -772,22 +1194,17 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
# padding images via boolean-mask indexing, LlavaOnevision
# branches on per-sample batch counts).
with gpu_sync_allowed():
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
features = self.model.get_image_features(pixel_values, **kwargs)
# Transformers `v5`, `self.get_image_features` returns a tuple
# containing the features and optionally attentions/hidden_states
# After v5 is settled, we can enable qwen3-vl with several outputs
# from `self.get_image_features`
if isinstance(vision_embeddings, tuple):
vision_embeddings = vision_embeddings[0]
elif isinstance(vision_embeddings, dict):
vision_embeddings = vision_embeddings.pooler_output
if isinstance(vision_embeddings, torch.Tensor):
split_sizes = num_image_patches.flatten().tolist()
return self._split_embeddings(vision_embeddings, split_sizes)
return list(vision_embeddings)
if isinstance(features, tuple):
return features[0]
if isinstance(features, dict):
return features.pooler_output
return features
def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings:
# Each helper detects its own inputs. We are called once per modality, so the