[3/N] Harden Transformers modelling backend multi-modal path (#51827)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -22,6 +22,8 @@ from contextlib import ExitStack, contextmanager
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
|
||||
from vllm.compilation.decorators import should_torch_compile_mm_encoder
|
||||
from vllm.config.utils import getattr_iter
|
||||
@@ -50,7 +52,9 @@ from vllm.multimodal.processing import (
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -65,6 +69,12 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3}
|
||||
_MODALITY_SIZE_KEYS = {"audio": "num_audio_tokens", "image": "num_image_patches"}
|
||||
|
||||
|
||||
def _get_embed_token_id(replacement_ids: torch.Tensor) -> int:
|
||||
"""The token an expansion repeats is the one holding the embeddings."""
|
||||
return int(replacement_ids.mode().values)
|
||||
|
||||
|
||||
class MultiModalProcessingInfo(BaseProcessingInfo):
|
||||
@@ -94,21 +104,6 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
|
||||
)
|
||||
return modalities
|
||||
|
||||
def _get_audio_token_id(self) -> int:
|
||||
processor = self.get_hf_processor()
|
||||
if hasattr(processor, "audio_token_id"):
|
||||
return processor.audio_token_id
|
||||
config = self.get_hf_config()
|
||||
val = getattr_iter(config, ("audio_token_id", "audio_token_index"))
|
||||
if val is not None:
|
||||
return val
|
||||
if hasattr(processor, "audio_token"):
|
||||
tokenizer = self.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
if processor.audio_token in vocab:
|
||||
return vocab[processor.audio_token]
|
||||
raise ValueError("Cannot find audio_token_id on processor or model config")
|
||||
|
||||
def _get_audio_sampling_rate(self) -> float:
|
||||
sub = self._get_audio_processor()
|
||||
if sub is not None and hasattr(sub, "sampling_rate"):
|
||||
@@ -169,7 +164,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
||||
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
|
||||
processor = self.info.get_hf_processor()
|
||||
audio_token = getattr(processor, "audio_token", "")
|
||||
# Separated so that `_apply_audio` can tell the placeholders apart
|
||||
# Separated so that adjacent placeholders stay distinguishable
|
||||
text += " ".join([audio_token] * num_audios)
|
||||
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
|
||||
processor = self.info.get_hf_processor()
|
||||
@@ -210,23 +205,26 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
||||
return data
|
||||
|
||||
|
||||
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
def _get_prompt_updates(
|
||||
class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
"""Processing common to both Transformers backend processors: calling the HF
|
||||
processor, sizing images, and attributing its outputs to a modality.
|
||||
|
||||
Subclasses add the strategy for locating placeholders in the prompt.
|
||||
"""
|
||||
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""No updates: `apply` locates placeholders via `mm_token_type_ids` instead.
|
||||
|
||||
HF processors have no generic contract for the token sequence they insert,
|
||||
so it cannot be expressed as a `PromptUpdate`. Returning nothing is only
|
||||
safe because `apply` is overridden; the base class would reject it.
|
||||
"""
|
||||
return []
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""Rename the parser's `audios` key to the `audio` argument HF audio
|
||||
processors take."""
|
||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||
if self.info._is_audio_model() and "audios" in processor_data:
|
||||
processor_data["audio"] = processor_data.pop("audios")
|
||||
return processor_data, passthrough_data
|
||||
|
||||
def _get_modality_field_names(self, modality: str) -> set[str]:
|
||||
"""Field names the sub-processor for `modality` produces."""
|
||||
"""Names of the fields the sub-processor for `modality` produces."""
|
||||
# TODO: use else branch only once huggingface/transformers#44394 lands.
|
||||
if modality == "audio":
|
||||
sub_processor = self.info._get_audio_processor()
|
||||
@@ -273,6 +271,18 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
|
||||
return owned
|
||||
|
||||
def _get_slice_dim(self, data: Any, total_rows: int) -> int:
|
||||
"""Which dimension of a field holds the rows belonging to each item.
|
||||
|
||||
Some processors (e.g., Idefics3) return image fields with a leading batch
|
||||
dimension, putting the rows one dimension further in.
|
||||
"""
|
||||
if not isinstance(data, torch.Tensor) or data.ndim < 2:
|
||||
return 0
|
||||
if data.shape[0] != total_rows and data.shape[1] == total_rows:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: "BatchFeature",
|
||||
@@ -281,29 +291,50 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
# HF Processors always return a mask but vLLM doesn't need it
|
||||
hf_inputs.pop("attention_mask", None)
|
||||
|
||||
# Written by `_apply_audio`/`_apply_vision`; absent if the modality had no items
|
||||
# Absent if the modality had no items
|
||||
sizes = {
|
||||
"audio": hf_inputs.get("num_audio_tokens"),
|
||||
"image": hf_inputs.get("num_image_patches"),
|
||||
modality: hf_inputs.get(key)
|
||||
for modality, key in _MODALITY_SIZE_KEYS.items()
|
||||
}
|
||||
modalities = [m for m, size in sizes.items() if size is not None]
|
||||
|
||||
size_keys = {"num_audio_tokens", "num_image_patches"}
|
||||
keys = [key for key in hf_inputs if key not in size_keys]
|
||||
# Keys we wrote ourselves, rather than ones a sub-processor produced
|
||||
own_keys = set(_MODALITY_SIZE_KEYS.values()) | {
|
||||
f"{modality}_replacement_{suffix}"
|
||||
for modality in modalities
|
||||
for suffix in ("ids", "sizes")
|
||||
}
|
||||
keys = [key for key in hf_inputs if key not in own_keys]
|
||||
owned = self._partition_keys_by_modality(keys, modalities)
|
||||
|
||||
# Un-padded fields are already one entry per item, so index rather than slice
|
||||
mm_fields: dict[str, MultiModalFieldConfig] = {
|
||||
key: MultiModalFieldConfig.flat_from_sizes(modality, sizes[modality])
|
||||
key: MultiModalFieldConfig.batched(modality)
|
||||
if modality == "audio" or isinstance(hf_inputs[key], list)
|
||||
else MultiModalFieldConfig.flat_from_sizes(
|
||||
modality,
|
||||
sizes[modality],
|
||||
dim=self._get_slice_dim(hf_inputs[key], int(sizes[modality].sum())),
|
||||
)
|
||||
for modality in modalities
|
||||
for key in owned[modality]
|
||||
}
|
||||
|
||||
# Keep these as batched, as they always have batch size as first dim
|
||||
if "audio" in modalities:
|
||||
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched(
|
||||
"audio", keep_on_cpu=True
|
||||
for modality in modalities:
|
||||
# One row per item, and only ever read on the CPU
|
||||
mm_fields[_MODALITY_SIZE_KEYS[modality]] = MultiModalFieldConfig.batched(
|
||||
modality, keep_on_cpu=True
|
||||
)
|
||||
replacement_sizes = hf_inputs.get(f"{modality}_replacement_sizes")
|
||||
if replacement_sizes is not None:
|
||||
mm_fields[f"{modality}_replacement_ids"] = (
|
||||
MultiModalFieldConfig.flat_from_sizes(
|
||||
modality, replacement_sizes, keep_on_cpu=True
|
||||
)
|
||||
)
|
||||
|
||||
if "image" in modalities:
|
||||
# Always one row per item, whatever they describe
|
||||
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
@@ -311,25 +342,154 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
|
||||
return mm_fields
|
||||
|
||||
def _get_hf_mm_data(
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Run the HF processor and unpad inputs."""
|
||||
try:
|
||||
hf_inputs = super()._call_hf_processor(
|
||||
prompt, mm_data, mm_kwargs, tok_kwargs
|
||||
)
|
||||
except ValueError:
|
||||
if any(mm_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs)
|
||||
hf_inputs = transformers.BatchFeature(
|
||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(hf_inputs)
|
||||
self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs)
|
||||
return hf_inputs
|
||||
|
||||
def _unpad_audios(
|
||||
self,
|
||||
hf_inputs: "BatchFeature",
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> None:
|
||||
"""Replace the audio fields with each audio processed on its own.
|
||||
|
||||
Processors pad every audio up to the longest in the call, which would leave
|
||||
an item's data dependent on what it was processed with. Unlike images,
|
||||
nothing in the output states how long each one really is, and processors
|
||||
pad a lone audio too, so the only way to know what an audio produces by
|
||||
itself is to process it by itself.
|
||||
"""
|
||||
audios = mm_data.get("audio")
|
||||
if not audios or len({len(audio) for audio in audios}) == 1:
|
||||
return
|
||||
|
||||
alone = [
|
||||
self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(
|
||||
text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
|
||||
),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
)
|
||||
for audio in audios
|
||||
]
|
||||
|
||||
for key in self._get_modality_field_names("audio"):
|
||||
if isinstance(hf_inputs.get(key), torch.Tensor):
|
||||
hf_inputs[key] = [output[key][0] for output in alone]
|
||||
|
||||
def _unpad_images(self, hf_inputs: "BatchFeature") -> None:
|
||||
"""Trim each image back to its own size when the processor padded them all
|
||||
to the largest in the batch.
|
||||
|
||||
An image's data has to depend on nothing but that image, or the multi-modal
|
||||
processor cache would store it under that image's hash and later reuse it
|
||||
beside a different neighbour. Padding is re-applied when the encoder runs.
|
||||
"""
|
||||
pixel_values = hf_inputs.get("pixel_values")
|
||||
image_sizes = hf_inputs.get("image_sizes")
|
||||
if not isinstance(pixel_values, torch.Tensor):
|
||||
return
|
||||
if not isinstance(image_sizes, torch.Tensor):
|
||||
return
|
||||
if pixel_values.ndim != 4 or len(pixel_values) != len(image_sizes):
|
||||
return
|
||||
|
||||
# The sizes describe the trailing dimensions only if the largest of them is
|
||||
# what the batch was padded up to. Otherwise they mean something else, as
|
||||
# in llava-onevision, where they are the sizes before any processing.
|
||||
maxima = image_sizes.max(dim=0).values
|
||||
if maxima.tolist() != list(pixel_values.shape[-2:]):
|
||||
return
|
||||
|
||||
hf_inputs["pixel_values"] = [
|
||||
image[..., :height, :width]
|
||||
for image, (height, width) in zip(pixel_values, image_sizes.tolist())
|
||||
]
|
||||
|
||||
|
||||
class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"""Locates placeholders by searching the prompt the HF processor has already
|
||||
expanded for the tokens of each modality.
|
||||
|
||||
Serves transformers versions with no `return_text_replacement_offsets`.
|
||||
Placeholders found this way cannot be rebuilt from an unexpanded prompt, so
|
||||
this processor overrides `apply` and gets no multi-modal processor cache.
|
||||
Remove it once `requirements/common.txt` requires `transformers>=5.15.0`.
|
||||
"""
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Empty, because `apply` writes the placeholder ranges itself rather than
|
||||
deriving them from updates."""
|
||||
return []
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Ask which modality owns each token of the expanded prompt, which is the
|
||||
only marking of the tokens an expansion adds around an item."""
|
||||
if any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_mm_token_type_ids": True}
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||
"""Token ids marking where `modality` sits in the prompt, which for some
|
||||
processors differ from the placeholder written into it.
|
||||
|
||||
The expanded prompt is all this path has to go on, so it takes the
|
||||
processor at its word about which tokens belong to the modality.
|
||||
"""
|
||||
In contrast to the base class, this method requests
|
||||
`return_mm_token_type_ids` and remaps the `audios` key to `audio` for
|
||||
audio models.
|
||||
"""
|
||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||
if self.info._is_audio_model() and "audios" in processor_data:
|
||||
processor_data["audio"] = processor_data.pop("audios")
|
||||
processor_data["return_mm_token_type_ids"] = True
|
||||
return processor_data, passthrough_data
|
||||
info = self.info
|
||||
processor = info.get_hf_processor()
|
||||
declared = getattr(processor, f"{modality}_token_ids", None) or ()
|
||||
if ids := [token_id for token_id in declared if token_id is not None]:
|
||||
return ids
|
||||
config = info.get_hf_config()
|
||||
names = (f"{modality}_token_id", f"{modality}_token_index")
|
||||
token_id = getattr(processor, names[0], getattr_iter(config, names))
|
||||
if token_id is None:
|
||||
token = getattr(processor, f"{modality}_token", None)
|
||||
token_id = info.get_tokenizer().get_vocab().get(token)
|
||||
if token_id is None:
|
||||
raise ValueError(
|
||||
f"Cannot find {modality}_token_id on processor or model config"
|
||||
)
|
||||
return [token_id]
|
||||
|
||||
def _apply_audio(
|
||||
self,
|
||||
@@ -337,12 +497,17 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
processed_data: "BatchFeature",
|
||||
num_audios: int,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
audio_token_id = self.info._get_audio_token_id()
|
||||
"""Take each contiguous run of the audio token as one item's placeholder,
|
||||
and record how many tokens the run holds."""
|
||||
audio_token_ids = self._get_mm_token_ids("audio")
|
||||
prompt_tensor = torch.tensor(prompt_ids)
|
||||
is_audio = prompt_tensor == audio_token_id
|
||||
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
|
||||
|
||||
if not is_audio.any():
|
||||
return {}
|
||||
raise ValueError(
|
||||
f"{num_audios} audio item(s) were passed but the prompt "
|
||||
"contains no audio token. Add one placeholder per audio item."
|
||||
)
|
||||
|
||||
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
|
||||
transitions = padded.int().diff()
|
||||
@@ -365,6 +530,23 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
processed_data["num_audio_tokens"] = lengths
|
||||
return {"audio": ranges}
|
||||
|
||||
def _get_num_multimodal_tokens(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, list[int]]:
|
||||
"""Ask the HF processor how many tokens and patches each image expands to."""
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = [
|
||||
(size.height, size.width)
|
||||
for size in map(images.get_image_size, range(len(images)))
|
||||
]
|
||||
return processor._get_num_multimodal_tokens(
|
||||
image_sizes=image_sizes,
|
||||
**self.info.ctx.get_merged_mm_kwargs({}),
|
||||
)
|
||||
|
||||
def _apply_vision(
|
||||
self,
|
||||
prompt_ids: list[int],
|
||||
@@ -373,47 +555,46 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
mm_token_type_ids: torch.Tensor | None,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
# Placeholders can't be located without them, so give up rather than guess
|
||||
if mm_token_type_ids is None:
|
||||
return {}
|
||||
|
||||
"""Split the positions the processor marks as image into one placeholder per
|
||||
item, sized by the token count it reports for each image."""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
if mm_token_type_ids is None:
|
||||
raise ValueError(
|
||||
f"{type(hf_processor).__name__} returned no `mm_token_type_ids`, so "
|
||||
"the Transformers modeling backend cannot locate the placeholder of "
|
||||
"each image."
|
||||
)
|
||||
|
||||
# We can infer vLLM style placeholder from token type ids, if we split
|
||||
# it for each input `mm_data`.
|
||||
mm_positions = torch.where(mm_token_type_ids == 1)[1]
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = []
|
||||
for item_idx in range(len(images)):
|
||||
image_size = images.get_image_size(item_idx)
|
||||
image_sizes.append((image_size.height, image_size.width))
|
||||
|
||||
mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
|
||||
image_sizes=image_sizes,
|
||||
**self.info.ctx.get_merged_mm_kwargs({}),
|
||||
# These mark the tokens structuring an image as well as the image itself,
|
||||
# which is what the counts below are over
|
||||
mm_positions = torch.where(mm_token_type_ids[0] == 1)[0]
|
||||
mm_tokens_per_modality = self._get_num_multimodal_tokens(
|
||||
mm_items, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||
split_sizes = mm_tokens_per_modality["num_image_tokens"]
|
||||
if split_sizes:
|
||||
image_token_ids = getattr(hf_processor, "image_token_ids", None)
|
||||
if image_token_ids is None:
|
||||
# Transformers <5.10.0
|
||||
image_token_ids = [hf_processor.image_token_id]
|
||||
image_token_ids = torch.tensor(
|
||||
[i for i in image_token_ids if i is not None]
|
||||
if sum(split_sizes) != len(mm_positions):
|
||||
raise ValueError(
|
||||
f"The expanded prompt holds {len(mm_positions)} image token(s) but "
|
||||
f"{type(hf_processor).__name__} accounts for {sum(split_sizes)} "
|
||||
f"across {mm_items.get_count('image')} image item(s)."
|
||||
)
|
||||
|
||||
chunked_mm_positions = torch.split(mm_positions, split_sizes)
|
||||
mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
|
||||
chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
|
||||
if split_sizes:
|
||||
image_token_ids = torch.tensor(self._get_mm_token_ids("image"))
|
||||
mm_tokens = torch.tensor(prompt_ids)[mm_positions]
|
||||
ranges = [
|
||||
PlaceholderRange(
|
||||
offset=positions[0].item(),
|
||||
length=positions.shape[0],
|
||||
is_embed=torch.isin(mm_tokens, image_token_ids),
|
||||
# Only some of the span carries embeddings
|
||||
is_embed=torch.isin(tokens, image_token_ids),
|
||||
)
|
||||
for positions, tokens in zip(
|
||||
torch.split(mm_positions, split_sizes),
|
||||
torch.split(mm_tokens, split_sizes),
|
||||
)
|
||||
for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
|
||||
]
|
||||
mm_placeholders = {"image": ranges}
|
||||
|
||||
@@ -427,12 +608,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInput:
|
||||
"""
|
||||
Process multi-modal inputs to be used in vLLM.
|
||||
|
||||
Apply HF Processor on prompt text and multi-modal data together,
|
||||
outputting token IDs and processed tensors.
|
||||
"""
|
||||
"""Process the prompt and every multi-modal item in one HF processor call,
|
||||
then read the placeholder ranges out of the token ids it returns."""
|
||||
prompt = inputs.prompt
|
||||
mm_items = inputs.mm_data_items
|
||||
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
||||
@@ -468,7 +645,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
# For gemma3 we check `token_type_ids` as the key
|
||||
# Gemma3 reports them under the key the model uses for its own token types
|
||||
mm_token_type_ids = processed_data.pop("token_type_ids", None)
|
||||
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
|
||||
|
||||
@@ -516,6 +693,201 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
)
|
||||
|
||||
|
||||
class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"""Locates placeholders from the `text_replacement_offsets` the HF processor
|
||||
reports, expressing each one as a `PromptUpdate`.
|
||||
|
||||
Stating the expansion as an update is what lets it be rebuilt from an
|
||||
unexpanded prompt, so this processor takes the base class's processing path
|
||||
and with it the multi-modal processor cache.
|
||||
"""
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Replace each modality's placeholder token with the token ids that item's
|
||||
replacement text encodes to, marking which of them hold embeddings."""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
updates = []
|
||||
for modality, items in out_mm_kwargs.items():
|
||||
# Popped so they are neither cached nor sent to the model; the updates
|
||||
# they produce are cached alongside the item instead
|
||||
replacements = [
|
||||
PromptUpdateDetails.select_token_id(
|
||||
(ids := item.pop(f"{modality}_replacement_ids").data).tolist(),
|
||||
_get_embed_token_id(ids),
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
updates.append(
|
||||
PromptReplacement(
|
||||
modality=modality,
|
||||
target=getattr(hf_processor, f"{modality}_token"),
|
||||
replacement=replacements.__getitem__,
|
||||
)
|
||||
)
|
||||
return updates
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
*,
|
||||
enable_hf_prompt_update: bool,
|
||||
) -> tuple[list[int], "BatchFeature", bool]:
|
||||
"""Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice
|
||||
in, whatever `enable_hf_prompt_update` asks for.
|
||||
|
||||
This differs from `super()` only for a text prompt with
|
||||
`enable_hf_prompt_update`, where `super()` would keep the expanded token
|
||||
ids the HF processor returns. Both routes give the same ids, so forcing
|
||||
this one costs an extra processor call and buys the guarantee that a
|
||||
request bypassing the cache is identical to a cached one.
|
||||
|
||||
A text prompt only arrives with `enable_hf_prompt_update` when there is
|
||||
no multi-modal processor cache. Requests carrying pre-computed
|
||||
embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported
|
||||
here, so they fail validation shortly afterwards.
|
||||
"""
|
||||
if isinstance(prompt, str) and enable_hf_prompt_update:
|
||||
logger.warning_once(
|
||||
"Disabling the multi-modal processor cache is extra slow with the "
|
||||
"Transformers modeling backend: the prompt is still tokenized "
|
||||
"unexpanded and the expansion spliced in, to keep the token ids "
|
||||
"identical to what the cache produces, which costs an extra HF "
|
||||
"processor call per request."
|
||||
)
|
||||
return super()._apply_hf_processor_main(
|
||||
prompt,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
enable_hf_prompt_update=False,
|
||||
)
|
||||
|
||||
def _get_num_image_patches(
|
||||
self,
|
||||
hf_inputs: "BatchFeature",
|
||||
mm_data: Mapping[str, object],
|
||||
num_images: int,
|
||||
) -> torch.Tensor:
|
||||
"""How many rows of the image fields belong to each image.
|
||||
|
||||
Taken from whichever per-image count the processor reports,
|
||||
and checked against the data it has to slice.
|
||||
"""
|
||||
if (grid := hf_inputs.get("image_grid_thw")) is not None:
|
||||
num_patches = grid.prod(-1)
|
||||
elif (counts := self._get_num_patches_per_image(mm_data)) is not None:
|
||||
num_patches = torch.tensor(counts)
|
||||
else:
|
||||
num_patches = torch.ones(num_images, dtype=torch.long)
|
||||
|
||||
image_data = hf_inputs.get("pixel_values", hf_inputs.get("image_patches"))
|
||||
if isinstance(image_data, torch.Tensor):
|
||||
total = int(num_patches.sum())
|
||||
rows = image_data.shape[self._get_slice_dim(image_data, total)]
|
||||
if rows != total:
|
||||
raise ValueError(
|
||||
f"{type(self.info.get_hf_processor()).__name__} returned "
|
||||
f"{rows} row(s) of image data for {num_images} image(s), which "
|
||||
f"cannot be split into the {num_patches.tolist()} row(s) per "
|
||||
"image derived from its outputs, so the rows cannot be "
|
||||
"attributed to an image. Gemma3 does this when "
|
||||
"`do_pan_and_scan` crops an image."
|
||||
)
|
||||
return num_patches
|
||||
|
||||
def _get_num_patches_per_image(
|
||||
self, mm_data: Mapping[str, object]
|
||||
) -> list[int] | None:
|
||||
"""Ask the HF processor how many rows of image data each image produces."""
|
||||
images = mm_data.get("images")
|
||||
if not images:
|
||||
return None
|
||||
try:
|
||||
sizes = [(image.height, image.width) for image in images]
|
||||
mm_tokens = self.info.get_hf_processor()._get_num_multimodal_tokens(
|
||||
image_sizes=sizes, **self.info.ctx.get_merged_mm_kwargs({})
|
||||
)
|
||||
return list(mm_tokens["num_image_patches"])
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Ask for the replacement each placeholder expands to, and record it as
|
||||
per-item fields: its token ids, and the tokens or patches behind them."""
|
||||
if has_mm_data := any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_text_replacement_offsets": True}
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
# Drop the inputs the model would reject
|
||||
hf_inputs.pop("mm_token_type_ids", None)
|
||||
hf_inputs.pop("token_type_ids", None)
|
||||
|
||||
offsets = hf_inputs.pop("text_replacement_offsets", None)
|
||||
# Some processors return an empty batch as a tensor rather than a list
|
||||
if offsets is None or len(offsets) == 0 or len(offsets[0]) == 0:
|
||||
if has_mm_data:
|
||||
raise ValueError(
|
||||
f"{type(self.info.get_hf_processor()).__name__} returned no "
|
||||
"text replacement offsets, so the Transformers modeling backend "
|
||||
"cannot locate the placeholder of each item. Its `__call__` has "
|
||||
"to reach `ProcessorMixin.get_text_with_replacements` with one "
|
||||
"replacement per item, which usually means implementing "
|
||||
"`replace_<modality>_token`. Please report this to transformers "
|
||||
"so it can be fixed, and install `transformers<5.15.0` in the "
|
||||
"meantime to locate placeholders in the expanded prompt instead."
|
||||
)
|
||||
return hf_inputs
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
replacements = defaultdict[str, list[list[int]]](list)
|
||||
for entry in offsets[0]:
|
||||
replacements[entry["type"]].append(
|
||||
tokenizer.encode(entry["replacement"], add_special_tokens=False)
|
||||
)
|
||||
|
||||
for modality, seqs in replacements.items():
|
||||
hf_inputs[f"{modality}_replacement_ids"] = torch.tensor(
|
||||
[token_id for seq in seqs for token_id in seq]
|
||||
)
|
||||
hf_inputs[f"{modality}_replacement_sizes"] = torch.tensor(
|
||||
[len(seq) for seq in seqs]
|
||||
)
|
||||
if modality == "image":
|
||||
hf_inputs["num_image_patches"] = self._get_num_image_patches(
|
||||
hf_inputs, mm_data, len(seqs)
|
||||
)
|
||||
elif modality == "audio":
|
||||
counts = []
|
||||
for seq in seqs:
|
||||
ids = torch.tensor(seq)
|
||||
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
|
||||
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
|
||||
|
||||
return hf_inputs
|
||||
|
||||
|
||||
# From this version on, a processor reporting no offsets is an error rather than a
|
||||
# fallback to searching the expanded prompt
|
||||
MultiModalProcessor = (
|
||||
LegacyMultiModalProcessor
|
||||
if Version(transformers.__version__) < Version("5.15.0")
|
||||
else OffsetsMultiModalProcessor
|
||||
)
|
||||
|
||||
|
||||
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
# Skip SupportsMRoPE.__init__ and call the next class in MRO
|
||||
@@ -733,18 +1105,32 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
kwargs.pop("token_type_ids", None)
|
||||
kwargs.pop("mm_token_type_ids", None)
|
||||
|
||||
# HuggingFace's `get_audio_features` implementations branch on
|
||||
# per-sample feature lengths internally.
|
||||
with gpu_sync_allowed():
|
||||
audio_output = self.model.get_audio_features(
|
||||
input_features, return_dict=True, **kwargs
|
||||
)
|
||||
audio_embeddings = audio_output.pooler_output
|
||||
|
||||
# Per-audio token counts are needed as Python ints to split.
|
||||
with gpu_sync_allowed():
|
||||
split_sizes = num_audio_tokens.flatten().tolist()
|
||||
return self._split_embeddings(audio_embeddings, split_sizes)
|
||||
if isinstance(input_features, torch.Tensor):
|
||||
# HuggingFace's `get_audio_features` implementations branch on
|
||||
# per-sample feature lengths internally.
|
||||
with gpu_sync_allowed():
|
||||
audio_output = self.model.get_audio_features(
|
||||
input_features, return_dict=True, **kwargs
|
||||
)
|
||||
return self._split_embeddings(audio_output.pooler_output, split_sizes)
|
||||
|
||||
# Audios the processor left un-padded arrive as a list once their
|
||||
# lengths differ. Encode them one at a time so that none of them is
|
||||
# padded to match another.
|
||||
embeddings: list[torch.Tensor] = []
|
||||
for index, features in enumerate(input_features):
|
||||
audio_output = self.model.get_audio_features(
|
||||
features.unsqueeze(0),
|
||||
return_dict=True,
|
||||
**self._select_item_kwargs(kwargs, index, len(input_features)),
|
||||
)
|
||||
embeddings.extend(
|
||||
self._split_embeddings(audio_output.pooler_output, [split_sizes[index]])
|
||||
)
|
||||
return embeddings
|
||||
|
||||
def _process_image_input(self, **kwargs) -> list[torch.Tensor] | None:
|
||||
pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
|
||||
@@ -761,6 +1147,42 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
|
||||
num_image_patches = kwargs.pop("num_image_patches")
|
||||
|
||||
split_sizes = num_image_patches.flatten().tolist()
|
||||
if isinstance(pixel_values, torch.Tensor):
|
||||
vision_embeddings = self._get_image_features(pixel_values, **kwargs)
|
||||
if isinstance(vision_embeddings, torch.Tensor):
|
||||
return self._split_embeddings(vision_embeddings, split_sizes)
|
||||
return list(vision_embeddings)
|
||||
|
||||
# Images the processor left un-padded arrive as a list once their
|
||||
# shapes differ. Encode them one at a time so that none of them is
|
||||
# padded to match another.
|
||||
embeddings: list[torch.Tensor] = []
|
||||
for index, image in enumerate(pixel_values):
|
||||
features = self._get_image_features(
|
||||
image.unsqueeze(0),
|
||||
**self._select_item_kwargs(kwargs, index, len(pixel_values)),
|
||||
)
|
||||
# Encoders which return one entry per image return a single entry
|
||||
if not isinstance(features, torch.Tensor):
|
||||
features = torch.cat(list(features))
|
||||
embeddings.extend(self._split_embeddings(features, [split_sizes[index]]))
|
||||
return embeddings
|
||||
|
||||
def _select_item_kwargs(
|
||||
self, kwargs: dict[str, Any], index: int, num_items: int
|
||||
) -> dict[str, Any]:
|
||||
"""Narrow the entries of `kwargs` that hold one row per item down to the item
|
||||
at `index`. Length is all there is to match on, so an unrelated entry of the
|
||||
same length is narrowed too."""
|
||||
return {
|
||||
key: value[index : index + 1]
|
||||
if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
|
||||
else value
|
||||
for key, value in kwargs.items()
|
||||
}
|
||||
|
||||
def _get_image_features(self, pixel_values: torch.Tensor, **kwargs) -> Any:
|
||||
# grid_thw fields are registered keep_on_cpu; restore the on-device
|
||||
# placement that HF get_image_features implementations expect.
|
||||
for key, value in kwargs.items():
|
||||
@@ -772,22 +1194,17 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
# padding images via boolean-mask indexing, LlavaOnevision
|
||||
# branches on per-sample batch counts).
|
||||
with gpu_sync_allowed():
|
||||
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
|
||||
features = self.model.get_image_features(pixel_values, **kwargs)
|
||||
|
||||
# Transformers `v5`, `self.get_image_features` returns a tuple
|
||||
# containing the features and optionally attentions/hidden_states
|
||||
# After v5 is settled, we can enable qwen3-vl with several outputs
|
||||
# from `self.get_image_features`
|
||||
if isinstance(vision_embeddings, tuple):
|
||||
vision_embeddings = vision_embeddings[0]
|
||||
elif isinstance(vision_embeddings, dict):
|
||||
vision_embeddings = vision_embeddings.pooler_output
|
||||
|
||||
if isinstance(vision_embeddings, torch.Tensor):
|
||||
split_sizes = num_image_patches.flatten().tolist()
|
||||
return self._split_embeddings(vision_embeddings, split_sizes)
|
||||
|
||||
return list(vision_embeddings)
|
||||
if isinstance(features, tuple):
|
||||
return features[0]
|
||||
if isinstance(features, dict):
|
||||
return features.pooler_output
|
||||
return features
|
||||
|
||||
def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings:
|
||||
# Each helper detects its own inputs. We are called once per modality, so the
|
||||
|
||||
Reference in New Issue
Block a user