diff --git a/tests/models/multimodal/processing/test_transformers_audio.py b/tests/models/multimodal/processing/test_transformers_audio.py index bc0dedcd1d..61517ea0af 100644 --- a/tests/models/multimodal/processing/test_transformers_audio.py +++ b/tests/models/multimodal/processing/test_transformers_audio.py @@ -4,8 +4,11 @@ import numpy as np import pytest from vllm.config import ModelConfig +from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor from vllm.multimodal import MULTIMODAL_REGISTRY +from .transformers_backend import PROCESSOR_CLASSES, create_processor + AUDIO_MODEL_SETTINGS = { "ibm-granite/granite-speech-3.3-2b": { "prompt": ( @@ -53,6 +56,7 @@ AUDIO_MODEL_SETTINGS = { } +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) @pytest.mark.parametrize( "model_id", [ @@ -61,12 +65,12 @@ AUDIO_MODEL_SETTINGS = { pytest.param( "mistralai/Voxtral-Mini-3B-2507", marks=pytest.mark.xfail( - reason="MistralCommonBackend.encode does not produce the audio " - "placeholder token (ID 24) from raw text. apply_chat_template " - "yields token IDs with placeholders, but MultiModalProcessor." - "apply() decodes the prompt back to text and re-tokenizes, at " - "which point the placeholders are lost. Fix belongs in " - "mistral_common or in the Voxtral-specific path.", + reason="Voxtral's mistral_common processor does not compose with " + "the Transformers modelling backend. Loading it currently fails " + "outright, because MistralCommonBackend.from_pretrained rejects " + "the kwargs vLLM passes, and it implements no " + "`replace_audio_token`, so it would report no replacement " + "offsets. Both fixes belong in mistral_common or transformers.", strict=False, ), ), @@ -74,15 +78,10 @@ AUDIO_MODEL_SETTINGS = { "zai-org/GLM-ASR-Nano-2512", ], ) -def test_audio_multimodal_processor(model_id): +def test_audio_multimodal_processor(model_id, processor_cls): settings = AUDIO_MODEL_SETTINGS[model_id] - model_config = ModelConfig( - model=model_id, - model_impl="transformers", - ) - - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + mm_processor = create_processor(model_id, processor_cls) audio = np.zeros(16000, dtype=np.float32) mm_data = {"audio": (audio, 16000)} @@ -113,10 +112,13 @@ def test_audio_multimodal_processor(model_id): ) -def _process_granite_speech(separator: str): +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) +@pytest.mark.parametrize("separator", [" and ", ""]) +def test_audio_multiple_inputs(separator, processor_cls): + """Multiple audios per prompt are each detected as a separate placeholder + and multi-modal item by the Transformers modelling backend.""" model_id = "ibm-granite/granite-speech-3.3-2b" - model_config = ModelConfig(model=model_id, model_impl="transformers") - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + mm_processor = create_processor(model_id, processor_cls) audio_token = mm_processor.info.get_hf_processor().audio_token # One token per audio; the processor expands each to its placeholder run. @@ -126,17 +128,21 @@ def _process_granite_speech(separator: str): ) audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)] - return mm_processor( - prompt=prompt, - mm_items=mm_processor.info.parse_mm_data({"audio": audios}), - hf_processor_mm_kwargs={}, - ) + def process(): + return mm_processor( + prompt=prompt, + mm_items=mm_processor.info.parse_mm_data({"audio": audios}), + hf_processor_mm_kwargs={}, + ) + # The legacy path reads placeholders off contiguous runs of the audio token, so + # it cannot tell adjacent ones apart and says so instead of merging them + if processor_cls is LegacyMultiModalProcessor and not separator: + with pytest.raises(ValueError, match="Separate them in the prompt"): + process() + return -def test_audio_multiple_inputs(): - """Multiple audios per prompt are each detected as a separate placeholder - and multi-modal item by the Transformers modelling backend.""" - result = _process_granite_speech(separator=" and ") + result = process() assert len(result["mm_placeholders"]["audio"]) == 2 assert len(result["mm_kwargs"]["audio"]) == 2 @@ -167,9 +173,3 @@ def test_unclaimed_fields_warn_rather_than_raise(): assert owned["audio"] == ["input_features"] assert owned["image"] == [] - - -def test_audio_adjacent_inputs(): - """Adjacent audios are rejected rather than silently merged into one placeholder.""" - with pytest.raises(ValueError, match="told apart"): - _process_granite_speech(separator="") diff --git a/tests/models/multimodal/processing/test_transformers_image.py b/tests/models/multimodal/processing/test_transformers_image.py index 6aa42fc71c..f5de7d66f4 100644 --- a/tests/models/multimodal/processing/test_transformers_image.py +++ b/tests/models/multimodal/processing/test_transformers_image.py @@ -1,20 +1,27 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from unittest.mock import patch + import pytest from vllm.assets.image import ImageAsset -from vllm.config import ModelConfig -from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.model_executor.models.transformers.multimodal import ( + LegacyMultiModalProcessor, + OffsetsMultiModalProcessor, +) + +from .transformers_backend import ( + PROCESSOR_CLASSES, + create_cached_processor, + create_processor, + offsets_only, +) +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) @pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]) -def test_multimodal_processor(model_id): - model_config = ModelConfig( - model=model_id, - model_impl="transformers", - ) - - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) +def test_multimodal_processor(model_id, processor_cls): + mm_processor = create_processor(model_id, processor_cls) image_pil = ImageAsset("cherry_blossom").pil_image mm_data = {"image": image_pil} @@ -56,10 +63,9 @@ def test_multimodal_processor(model_id): ) -def _process_two_images(separator: str): +def _process_two_images(processor_cls, separator: str): model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" - model_config = ModelConfig(model=model_id, model_impl="transformers") - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + mm_processor = create_processor(model_id, processor_cls) image = ImageAsset("cherry_blossom").pil_image prompt = ( @@ -74,28 +80,147 @@ def _process_two_images(separator: str): ) -def test_image_multiple_inputs(): +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) +def test_image_multiple_inputs(processor_cls): """Multiple images per prompt are each detected as a separate placeholder and multi-modal item by the Transformers modelling backend.""" - result = _process_two_images(separator="\n and ") + result = _process_two_images(processor_cls, separator="\n and ") assert len(result["mm_placeholders"]["image"]) == 2 assert len(result["mm_kwargs"]["image"]) == 2 -def test_image_adjacent_inputs(): +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) +def test_image_adjacent_inputs(processor_cls): """Adjacent images stay separate placeholders rather than merging into one.""" - result = _process_two_images(separator="") + result = _process_two_images(processor_cls, separator="") assert len(result["mm_placeholders"]["image"]) == 2 assert len(result["mm_kwargs"]["image"]) == 2 -def test_text_only_prompt(): +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) +def test_batch_padding_removed_from_image_items(processor_cls): + """Emu3 pads every image up to the largest in the batch, which would leave an + item's data dependent on what it was processed with and so uncacheable.""" + mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls) + image_token = mm_processor.info.get_hf_processor().image_token + + images = [ + ImageAsset("cherry_blossom").pil_image, + ImageAsset("cherry_blossom").pil_image.resize((256, 1024)), + ] + result = mm_processor( + prompt=f"{image_token} and {image_token}", + mm_items=mm_processor.info.parse_mm_data({"image": images}), + hf_processor_mm_kwargs={}, + ) + + items = result["mm_kwargs"]["image"] + shapes = set() + for item in items: + height, width = item["image_sizes"].data.flatten().tolist() + pixel_values = item["pixel_values"].data + assert tuple(pixel_values.shape[-2:]) == (height, width) + shapes.add(tuple(pixel_values.shape)) + + # Both images would have been padded to a common shape had they been kept + assert len(shapes) == 2 + + +def _process_one_gemma3_image(processor_cls): + mm_processor = create_processor("google/gemma-3-4b-it", processor_cls) + hf_processor = mm_processor.info.get_hf_processor() + result = mm_processor( + prompt=f"{hf_processor.boi_token} What is this?", + mm_items=mm_processor.info.parse_mm_data( + {"image": ImageAsset("cherry_blossom").pil_image} + ), + hf_processor_mm_kwargs={}, + ) + return hf_processor, result + + +@offsets_only +def test_non_embedding_tokens_excluded_from_placeholders(): + """Gemma3 wraps each image in text that carries no embeddings, which must be + inside the placeholder range but masked out of it.""" + _, result = _process_one_gemma3_image(OffsetsMultiModalProcessor) + + (placeholder,) = result["mm_placeholders"]["image"] + assert placeholder.is_embed is not None + assert 0 < int(placeholder.is_embed.sum()) < placeholder.length + + +def test_legacy_placeholders_hold_only_image_tokens(): + """The legacy path spans whatever `mm_token_type_ids` attributes to the image, + which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets + path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the + marker in the unexpanded prompt, not the token the expansion repeats.""" + hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor) + + (placeholder,) = result["mm_placeholders"]["image"] + assert placeholder.length == hf_processor.image_seq_length + prompt_ids = result["prompt_token_ids"] + covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length] + assert set(covered) == {hf_processor.tokenizer.image_token_id} + + +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) +def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls): + """SmolVLM splits each image into tiles introduced by tokens carrying no + embeddings. Those belong inside the placeholder and masked out, because the token + count the processor reports is over the whole span. Idefics3 also refuses a prompt + holding `` when no images are passed, which is how the offsets path has to + tokenize it before splicing in the expansion.""" + mm_processor = create_processor( + "HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls + ) + result = mm_processor( + prompt="What is this?", + mm_items=mm_processor.info.parse_mm_data( + {"image": ImageAsset("cherry_blossom").pil_image} + ), + hf_processor_mm_kwargs={}, + ) + + (placeholder,) = result["mm_placeholders"]["image"] + assert placeholder.is_embed is not None + assert 0 < int(placeholder.is_embed.sum()) < placeholder.length + + +@offsets_only +def test_missing_replacement_offsets_names_the_processor(): + """A processor that reports no replacement offsets cannot be served, which must + be said plainly rather than surfacing later as a field config mismatch.""" + model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" + mm_processor = create_processor(model_id, OffsetsMultiModalProcessor) + hf_processor_cls = type(mm_processor.info.get_hf_processor()) + hf_call = hf_processor_cls.__call__ + + def without_offsets(self, *args, **kwargs): + hf_inputs = hf_call(self, *args, **kwargs) + hf_inputs.pop("text_replacement_offsets", None) + return hf_inputs + + with ( + patch.object(hf_processor_cls, "__call__", without_offsets), + pytest.raises(ValueError, match="LlavaOnevisionProcessor returned no"), + ): + mm_processor( + prompt="\nWhat is the content of this image?", + mm_items=mm_processor.info.parse_mm_data( + {"image": ImageAsset("cherry_blossom").pil_image} + ), + hf_processor_mm_kwargs={}, + ) + + +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) +def test_text_only_prompt(processor_cls): """An image model still accepts a prompt with no images.""" model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" - model_config = ModelConfig(model=model_id, model_impl="transformers") - mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + mm_processor = create_processor(model_id, processor_cls) result = mm_processor( prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n", @@ -105,3 +230,87 @@ def test_text_only_prompt(): assert len(result["prompt_token_ids"]) > 0 assert not result["mm_placeholders"] + + +@offsets_only +def test_repeated_image_hits_the_processor_cache(): + """Check that mm caching is actually working.""" + mm_processor, cache = create_cached_processor( + "llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor + ) + image = ImageAsset("cherry_blossom").pil_image + + def process(): + return mm_processor( + prompt="\nWhat is this?", + mm_items=mm_processor.info.parse_mm_data({"image": image}), + hf_processor_mm_kwargs={}, + ) + + first, second = process(), process() + + assert cache.make_stats().hits > 0 + assert first["prompt_token_ids"] == second["prompt_token_ids"] + assert first["mm_hashes"] == second["mm_hashes"] + + +@offsets_only +@pytest.mark.parametrize( + ("model_id", "prompt"), + [ + ("llava-hf/llava-onevision-qwen2-0.5b-ov-hf", "\nWhat is this?"), + ("google/gemma-3-4b-it", " What is this?"), + ("HuggingFaceTB/SmolVLM-256M-Instruct", "What is this?"), + pytest.param( + "BAAI/Emu3-Chat-hf", + " and more text", + marks=pytest.mark.xfail( + reason="Emu3Processor prepends its BOS token only when images are " + "passed, so the unexpanded prompt the offsets path tokenizes never " + "gets one. Fixed by huggingface/transformers#47924, unreleased.", + strict=False, + ), + ), + ], +) +def test_spliced_prompt_matches_hf_expansion(model_id, prompt): + """The offsets path splices the expansion into a prompt tokenized without any + multi-modal data, so its token ids have to come out the same as the ones the HF + processor produces itself, which is what the legacy path returns.""" + prompt_ids = [] + for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor): + mm_processor = create_processor(model_id, processor_cls) + prompt_ids.append( + mm_processor( + prompt=prompt, + mm_items=mm_processor.info.parse_mm_data( + {"image": ImageAsset("cherry_blossom").pil_image} + ), + hf_processor_mm_kwargs={}, + )["prompt_token_ids"] + ) + + legacy_ids, offsets_ids = prompt_ids + assert legacy_ids == offsets_ids + + +@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES) +def test_nested_image_fields_split_per_image(processor_cls): + """Idefics3 returns image fields with a leading batch dimension, putting the rows + belonging to each image one dimension further in. Slicing the batch dimension + instead handed the first image every row and the second an empty tensor.""" + mm_processor = create_processor( + "HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls + ) + image = ImageAsset("cherry_blossom").pil_image + result = mm_processor( + prompt=" and ", + mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}), + hf_processor_mm_kwargs={}, + ) + + items = result["mm_kwargs"]["image"] + assert len(items) == 2 + for item in items: + pixel_values = item["pixel_values"].data + assert pixel_values.shape[1] == int(item["num_image_patches"].data) diff --git a/tests/models/multimodal/processing/transformers_backend.py b/tests/models/multimodal/processing/transformers_backend.py new file mode 100644 index 0000000000..dc49036e83 --- /dev/null +++ b/tests/models/multimodal/processing/transformers_backend.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Helpers for testing both processors of the Transformers modelling backend.""" + +import pytest + +from vllm.config import ModelConfig +from vllm.model_executor.models.transformers.multimodal import ( + LegacyMultiModalProcessor, + MultiModalDummyInputsBuilder, + MultiModalProcessingInfo, + MultiModalProcessor, + OffsetsMultiModalProcessor, +) +from vllm.multimodal.cache import MultiModalProcessorOnlyCache +from vllm.multimodal.processing import InputProcessingContext +from vllm.tokenizers.registry import cached_tokenizer_from_config + +offsets_only = pytest.mark.skipif( + MultiModalProcessor is not OffsetsMultiModalProcessor, + reason="Replacement offsets are only used from transformers 5.15.0 onwards", +) + +PROCESSOR_CLASSES = [ + pytest.param(LegacyMultiModalProcessor, id="legacy"), + pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only), +] + + +def create_processor(model_id: str, processor_cls): + """Build a processor directly, because the registry only ever builds the one the + installed transformers version selects, leaving the other path untested.""" + model_config = ModelConfig(model=model_id, model_impl="transformers") + ctx = InputProcessingContext( + model_config, cached_tokenizer_from_config(model_config) + ) + info = MultiModalProcessingInfo(ctx) + return processor_cls(info, MultiModalDummyInputsBuilder(info)) + + +def create_cached_processor(model_id: str, processor_cls): + """Build a processor backed by a real multi-modal processor cache, and hand the + cache back so a test can check it was actually used.""" + model_config = ModelConfig(model=model_id, model_impl="transformers") + model_config.multimodal_config.mm_processor_cache_gb = 4 + ctx = InputProcessingContext( + model_config, cached_tokenizer_from_config(model_config) + ) + info = MultiModalProcessingInfo(ctx) + cache = MultiModalProcessorOnlyCache(model_config) + return processor_cls(info, MultiModalDummyInputsBuilder(info), cache=cache), cache diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index e94490232c..8b6568bf3c 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -22,6 +22,8 @@ from contextlib import ExitStack, contextmanager from typing import TYPE_CHECKING, Any import torch +import transformers +from packaging.version import Version from vllm.compilation.decorators import should_torch_compile_mm_encoder from vllm.config.utils import getattr_iter @@ -50,7 +52,9 @@ from vllm.multimodal.processing import ( BaseMultiModalProcessor, BaseProcessingInfo, ProcessorInputs, + PromptReplacement, PromptUpdate, + PromptUpdateDetails, TimingContext, ) from vllm.sequence import IntermediateTensors @@ -65,6 +69,12 @@ if TYPE_CHECKING: logger = init_logger(__name__) _MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3} +_MODALITY_SIZE_KEYS = {"audio": "num_audio_tokens", "image": "num_image_patches"} + + +def _get_embed_token_id(replacement_ids: torch.Tensor) -> int: + """The token an expansion repeats is the one holding the embeddings.""" + return int(replacement_ids.mode().values) class MultiModalProcessingInfo(BaseProcessingInfo): @@ -94,21 +104,6 @@ class MultiModalProcessingInfo(BaseProcessingInfo): ) return modalities - def _get_audio_token_id(self) -> int: - processor = self.get_hf_processor() - if hasattr(processor, "audio_token_id"): - return processor.audio_token_id - config = self.get_hf_config() - val = getattr_iter(config, ("audio_token_id", "audio_token_index")) - if val is not None: - return val - if hasattr(processor, "audio_token"): - tokenizer = self.get_tokenizer() - vocab = tokenizer.get_vocab() - if processor.audio_token in vocab: - return vocab[processor.audio_token] - raise ValueError("Cannot find audio_token_id on processor or model config") - def _get_audio_sampling_rate(self) -> float: sub = self._get_audio_processor() if sub is not None and hasattr(sub, "sampling_rate"): @@ -169,7 +164,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)): processor = self.info.get_hf_processor() audio_token = getattr(processor, "audio_token", "") - # Separated so that `_apply_audio` can tell the placeholders apart + # Separated so that adjacent placeholders stay distinguishable text += " ".join([audio_token] * num_audios) if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)): processor = self.info.get_hf_processor() @@ -210,23 +205,26 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn return data -class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): - def _get_prompt_updates( +class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]): + """Processing common to both Transformers backend processors: calling the HF + processor, sizing images, and attributing its outputs to a modality. + + Subclasses add the strategy for locating placeholders in the prompt. + """ + + def _get_hf_mm_data( self, mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - """No updates: `apply` locates placeholders via `mm_token_type_ids` instead. - - HF processors have no generic contract for the token sequence they insert, - so it cannot be expressed as a `PromptUpdate`. Returning nothing is only - safe because `apply` is overridden; the base class would reject it. - """ - return [] + ) -> tuple[Mapping[str, object], Mapping[str, object]]: + """Rename the parser's `audios` key to the `audio` argument HF audio + processors take.""" + processor_data, passthrough_data = super()._get_hf_mm_data(mm_items) + if self.info._is_audio_model() and "audios" in processor_data: + processor_data["audio"] = processor_data.pop("audios") + return processor_data, passthrough_data def _get_modality_field_names(self, modality: str) -> set[str]: - """Field names the sub-processor for `modality` produces.""" + """Names of the fields the sub-processor for `modality` produces.""" # TODO: use else branch only once huggingface/transformers#44394 lands. if modality == "audio": sub_processor = self.info._get_audio_processor() @@ -273,6 +271,18 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): return owned + def _get_slice_dim(self, data: Any, total_rows: int) -> int: + """Which dimension of a field holds the rows belonging to each item. + + Some processors (e.g., Idefics3) return image fields with a leading batch + dimension, putting the rows one dimension further in. + """ + if not isinstance(data, torch.Tensor) or data.ndim < 2: + return 0 + if data.shape[0] != total_rows and data.shape[1] == total_rows: + return 1 + return 0 + def _get_mm_fields_config( self, hf_inputs: "BatchFeature", @@ -281,29 +291,50 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): # HF Processors always return a mask but vLLM doesn't need it hf_inputs.pop("attention_mask", None) - # Written by `_apply_audio`/`_apply_vision`; absent if the modality had no items + # Absent if the modality had no items sizes = { - "audio": hf_inputs.get("num_audio_tokens"), - "image": hf_inputs.get("num_image_patches"), + modality: hf_inputs.get(key) + for modality, key in _MODALITY_SIZE_KEYS.items() } modalities = [m for m, size in sizes.items() if size is not None] - size_keys = {"num_audio_tokens", "num_image_patches"} - keys = [key for key in hf_inputs if key not in size_keys] + # Keys we wrote ourselves, rather than ones a sub-processor produced + own_keys = set(_MODALITY_SIZE_KEYS.values()) | { + f"{modality}_replacement_{suffix}" + for modality in modalities + for suffix in ("ids", "sizes") + } + keys = [key for key in hf_inputs if key not in own_keys] owned = self._partition_keys_by_modality(keys, modalities) + # Un-padded fields are already one entry per item, so index rather than slice mm_fields: dict[str, MultiModalFieldConfig] = { - key: MultiModalFieldConfig.flat_from_sizes(modality, sizes[modality]) + key: MultiModalFieldConfig.batched(modality) + if modality == "audio" or isinstance(hf_inputs[key], list) + else MultiModalFieldConfig.flat_from_sizes( + modality, + sizes[modality], + dim=self._get_slice_dim(hf_inputs[key], int(sizes[modality].sum())), + ) for modality in modalities for key in owned[modality] } - # Keep these as batched, as they always have batch size as first dim - if "audio" in modalities: - mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched( - "audio", keep_on_cpu=True + for modality in modalities: + # One row per item, and only ever read on the CPU + mm_fields[_MODALITY_SIZE_KEYS[modality]] = MultiModalFieldConfig.batched( + modality, keep_on_cpu=True ) + replacement_sizes = hf_inputs.get(f"{modality}_replacement_sizes") + if replacement_sizes is not None: + mm_fields[f"{modality}_replacement_ids"] = ( + MultiModalFieldConfig.flat_from_sizes( + modality, replacement_sizes, keep_on_cpu=True + ) + ) + if "image" in modalities: + # Always one row per item, whatever they describe mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched( "image", keep_on_cpu=True ) @@ -311,25 +342,154 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched( "image", keep_on_cpu=True ) - mm_fields["num_image_patches"] = MultiModalFieldConfig.batched( - "image", keep_on_cpu=True - ) + return mm_fields - def _get_hf_mm_data( + def _call_hf_processor( + self, + prompt: str, + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], + tok_kwargs: Mapping[str, object], + ) -> "BatchFeature": + """Run the HF processor and unpad inputs.""" + try: + hf_inputs = super()._call_hf_processor( + prompt, mm_data, mm_kwargs, tok_kwargs + ) + except ValueError: + if any(mm_data.values()): + raise + # Some processors reject a prompt holding placeholders with + # no data to go with them, so tokenize it without them + prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs) + hf_inputs = transformers.BatchFeature( + dict(input_ids=[prompt_ids]), tensor_type="pt" + ) + self._unpad_images(hf_inputs) + self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs) + return hf_inputs + + def _unpad_audios( + self, + hf_inputs: "BatchFeature", + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], + tok_kwargs: Mapping[str, object], + ) -> None: + """Replace the audio fields with each audio processed on its own. + + Processors pad every audio up to the longest in the call, which would leave + an item's data dependent on what it was processed with. Unlike images, + nothing in the output states how long each one really is, and processors + pad a lone audio too, so the only way to know what an audio produces by + itself is to process it by itself. + """ + audios = mm_data.get("audio") + if not audios or len({len(audio) for audio in audios}) == 1: + return + + alone = [ + self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**mm_kwargs), + dict( + text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio] + ), + dict(**mm_kwargs, **tok_kwargs), + ) + for audio in audios + ] + + for key in self._get_modality_field_names("audio"): + if isinstance(hf_inputs.get(key), torch.Tensor): + hf_inputs[key] = [output[key][0] for output in alone] + + def _unpad_images(self, hf_inputs: "BatchFeature") -> None: + """Trim each image back to its own size when the processor padded them all + to the largest in the batch. + + An image's data has to depend on nothing but that image, or the multi-modal + processor cache would store it under that image's hash and later reuse it + beside a different neighbour. Padding is re-applied when the encoder runs. + """ + pixel_values = hf_inputs.get("pixel_values") + image_sizes = hf_inputs.get("image_sizes") + if not isinstance(pixel_values, torch.Tensor): + return + if not isinstance(image_sizes, torch.Tensor): + return + if pixel_values.ndim != 4 or len(pixel_values) != len(image_sizes): + return + + # The sizes describe the trailing dimensions only if the largest of them is + # what the batch was padded up to. Otherwise they mean something else, as + # in llava-onevision, where they are the sizes before any processing. + maxima = image_sizes.max(dim=0).values + if maxima.tolist() != list(pixel_values.shape[-2:]): + return + + hf_inputs["pixel_values"] = [ + image[..., :height, :width] + for image, (height, width) in zip(pixel_values, image_sizes.tolist()) + ] + + +class LegacyMultiModalProcessor(_MultiModalProcessorBase): + """Locates placeholders by searching the prompt the HF processor has already + expanded for the tokens of each modality. + + Serves transformers versions with no `return_text_replacement_offsets`. + Placeholders found this way cannot be rebuilt from an unexpanded prompt, so + this processor overrides `apply` and gets no multi-modal processor cache. + Remove it once `requirements/common.txt` requires `transformers>=5.15.0`. + """ + + def _get_prompt_updates( self, mm_items: MultiModalDataItems, - ) -> tuple[Mapping[str, object], Mapping[str, object]]: + hf_processor_mm_kwargs: Mapping[str, object], + out_mm_kwargs: MultiModalKwargsItems, + ) -> Sequence[PromptUpdate]: + """Empty, because `apply` writes the placeholder ranges itself rather than + deriving them from updates.""" + return [] + + def _call_hf_processor( + self, + prompt: str, + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], + tok_kwargs: Mapping[str, object], + ) -> "BatchFeature": + """Ask which modality owns each token of the expanded prompt, which is the + only marking of the tokens an expansion adds around an item.""" + if any(mm_data.values()): + mm_data = {**mm_data, "return_mm_token_type_ids": True} + return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) + + def _get_mm_token_ids(self, modality: str) -> list[int]: + """Token ids marking where `modality` sits in the prompt, which for some + processors differ from the placeholder written into it. + + The expanded prompt is all this path has to go on, so it takes the + processor at its word about which tokens belong to the modality. """ - In contrast to the base class, this method requests - `return_mm_token_type_ids` and remaps the `audios` key to `audio` for - audio models. - """ - processor_data, passthrough_data = super()._get_hf_mm_data(mm_items) - if self.info._is_audio_model() and "audios" in processor_data: - processor_data["audio"] = processor_data.pop("audios") - processor_data["return_mm_token_type_ids"] = True - return processor_data, passthrough_data + info = self.info + processor = info.get_hf_processor() + declared = getattr(processor, f"{modality}_token_ids", None) or () + if ids := [token_id for token_id in declared if token_id is not None]: + return ids + config = info.get_hf_config() + names = (f"{modality}_token_id", f"{modality}_token_index") + token_id = getattr(processor, names[0], getattr_iter(config, names)) + if token_id is None: + token = getattr(processor, f"{modality}_token", None) + token_id = info.get_tokenizer().get_vocab().get(token) + if token_id is None: + raise ValueError( + f"Cannot find {modality}_token_id on processor or model config" + ) + return [token_id] def _apply_audio( self, @@ -337,12 +497,17 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): processed_data: "BatchFeature", num_audios: int, ) -> dict[str, list[PlaceholderRange]]: - audio_token_id = self.info._get_audio_token_id() + """Take each contiguous run of the audio token as one item's placeholder, + and record how many tokens the run holds.""" + audio_token_ids = self._get_mm_token_ids("audio") prompt_tensor = torch.tensor(prompt_ids) - is_audio = prompt_tensor == audio_token_id + is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids)) if not is_audio.any(): - return {} + raise ValueError( + f"{num_audios} audio item(s) were passed but the prompt " + "contains no audio token. Add one placeholder per audio item." + ) padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])]) transitions = padded.int().diff() @@ -365,6 +530,23 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): processed_data["num_audio_tokens"] = lengths return {"audio": ranges} + def _get_num_multimodal_tokens( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> Mapping[str, list[int]]: + """Ask the HF processor how many tokens and patches each image expands to.""" + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + images = mm_items.get_items("image", ImageProcessorItems) + image_sizes = [ + (size.height, size.width) + for size in map(images.get_image_size, range(len(images))) + ] + return processor._get_num_multimodal_tokens( + image_sizes=image_sizes, + **self.info.ctx.get_merged_mm_kwargs({}), + ) + def _apply_vision( self, prompt_ids: list[int], @@ -373,47 +555,46 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): hf_processor_mm_kwargs: Mapping[str, object], mm_token_type_ids: torch.Tensor | None, ) -> dict[str, list[PlaceholderRange]]: - # Placeholders can't be located without them, so give up rather than guess - if mm_token_type_ids is None: - return {} - + """Split the positions the processor marks as image into one placeholder per + item, sized by the token count it reports for each image.""" hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + if mm_token_type_ids is None: + raise ValueError( + f"{type(hf_processor).__name__} returned no `mm_token_type_ids`, so " + "the Transformers modeling backend cannot locate the placeholder of " + "each image." + ) - # We can infer vLLM style placeholder from token type ids, if we split - # it for each input `mm_data`. - mm_positions = torch.where(mm_token_type_ids == 1)[1] - images = mm_items.get_items("image", ImageProcessorItems) - image_sizes = [] - for item_idx in range(len(images)): - image_size = images.get_image_size(item_idx) - image_sizes.append((image_size.height, image_size.width)) - - mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens( - image_sizes=image_sizes, - **self.info.ctx.get_merged_mm_kwargs({}), + # These mark the tokens structuring an image as well as the image itself, + # which is what the counts below are over + mm_positions = torch.where(mm_token_type_ids[0] == 1)[0] + mm_tokens_per_modality = self._get_num_multimodal_tokens( + mm_items, hf_processor_mm_kwargs ) mm_placeholders: dict[str, list[PlaceholderRange]] = {} split_sizes = mm_tokens_per_modality["num_image_tokens"] - if split_sizes: - image_token_ids = getattr(hf_processor, "image_token_ids", None) - if image_token_ids is None: - # Transformers <5.10.0 - image_token_ids = [hf_processor.image_token_id] - image_token_ids = torch.tensor( - [i for i in image_token_ids if i is not None] + if sum(split_sizes) != len(mm_positions): + raise ValueError( + f"The expanded prompt holds {len(mm_positions)} image token(s) but " + f"{type(hf_processor).__name__} accounts for {sum(split_sizes)} " + f"across {mm_items.get_count('image')} image item(s)." ) - chunked_mm_positions = torch.split(mm_positions, split_sizes) - mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()] - chunked_mm_tokens = torch.split(mm_tokens, split_sizes) + if split_sizes: + image_token_ids = torch.tensor(self._get_mm_token_ids("image")) + mm_tokens = torch.tensor(prompt_ids)[mm_positions] ranges = [ PlaceholderRange( offset=positions[0].item(), length=positions.shape[0], - is_embed=torch.isin(mm_tokens, image_token_ids), + # Only some of the span carries embeddings + is_embed=torch.isin(tokens, image_token_ids), + ) + for positions, tokens in zip( + torch.split(mm_positions, split_sizes), + torch.split(mm_tokens, split_sizes), ) - for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens) ] mm_placeholders = {"image": ranges} @@ -427,12 +608,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): inputs: ProcessorInputs, timing_ctx: TimingContext, ) -> MultiModalInput: - """ - Process multi-modal inputs to be used in vLLM. - - Apply HF Processor on prompt text and multi-modal data together, - outputting token IDs and processed tensors. - """ + """Process the prompt and every multi-modal item in one HF processor call, + then read the placeholder ranges out of the token ids it returns.""" prompt = inputs.prompt mm_items = inputs.mm_data_items hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs @@ -468,7 +645,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): self.info.ctx.get_mm_config().mm_hasher_algorithm, ) - # For gemma3 we check `token_type_ids` as the key + # Gemma3 reports them under the key the model uses for its own token types mm_token_type_ids = processed_data.pop("token_type_ids", None) mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids) @@ -516,6 +693,201 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): ) +class OffsetsMultiModalProcessor(_MultiModalProcessorBase): + """Locates placeholders from the `text_replacement_offsets` the HF processor + reports, expressing each one as a `PromptUpdate`. + + Stating the expansion as an update is what lets it be rebuilt from an + unexpanded prompt, so this processor takes the base class's processing path + and with it the multi-modal processor cache. + """ + + def _get_prompt_updates( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + out_mm_kwargs: MultiModalKwargsItems, + ) -> Sequence[PromptUpdate]: + """Replace each modality's placeholder token with the token ids that item's + replacement text encodes to, marking which of them hold embeddings.""" + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + updates = [] + for modality, items in out_mm_kwargs.items(): + # Popped so they are neither cached nor sent to the model; the updates + # they produce are cached alongside the item instead + replacements = [ + PromptUpdateDetails.select_token_id( + (ids := item.pop(f"{modality}_replacement_ids").data).tolist(), + _get_embed_token_id(ids), + ) + for item in items + ] + updates.append( + PromptReplacement( + modality=modality, + target=getattr(hf_processor, f"{modality}_token"), + replacement=replacements.__getitem__, + ) + ) + return updates + + def _apply_hf_processor_main( + self, + prompt: str | list[int], + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + tokenization_kwargs: Mapping[str, object], + *, + enable_hf_prompt_update: bool, + ) -> tuple[list[int], "BatchFeature", bool]: + """Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice + in, whatever `enable_hf_prompt_update` asks for. + + This differs from `super()` only for a text prompt with + `enable_hf_prompt_update`, where `super()` would keep the expanded token + ids the HF processor returns. Both routes give the same ids, so forcing + this one costs an extra processor call and buys the guarantee that a + request bypassing the cache is identical to a cached one. + + A text prompt only arrives with `enable_hf_prompt_update` when there is + no multi-modal processor cache. Requests carrying pre-computed + embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported + here, so they fail validation shortly afterwards. + """ + if isinstance(prompt, str) and enable_hf_prompt_update: + logger.warning_once( + "Disabling the multi-modal processor cache is extra slow with the " + "Transformers modeling backend: the prompt is still tokenized " + "unexpanded and the expansion spliced in, to keep the token ids " + "identical to what the cache produces, which costs an extra HF " + "processor call per request." + ) + return super()._apply_hf_processor_main( + prompt, + mm_items, + hf_processor_mm_kwargs, + tokenization_kwargs, + enable_hf_prompt_update=False, + ) + + def _get_num_image_patches( + self, + hf_inputs: "BatchFeature", + mm_data: Mapping[str, object], + num_images: int, + ) -> torch.Tensor: + """How many rows of the image fields belong to each image. + + Taken from whichever per-image count the processor reports, + and checked against the data it has to slice. + """ + if (grid := hf_inputs.get("image_grid_thw")) is not None: + num_patches = grid.prod(-1) + elif (counts := self._get_num_patches_per_image(mm_data)) is not None: + num_patches = torch.tensor(counts) + else: + num_patches = torch.ones(num_images, dtype=torch.long) + + image_data = hf_inputs.get("pixel_values", hf_inputs.get("image_patches")) + if isinstance(image_data, torch.Tensor): + total = int(num_patches.sum()) + rows = image_data.shape[self._get_slice_dim(image_data, total)] + if rows != total: + raise ValueError( + f"{type(self.info.get_hf_processor()).__name__} returned " + f"{rows} row(s) of image data for {num_images} image(s), which " + f"cannot be split into the {num_patches.tolist()} row(s) per " + "image derived from its outputs, so the rows cannot be " + "attributed to an image. Gemma3 does this when " + "`do_pan_and_scan` crops an image." + ) + return num_patches + + def _get_num_patches_per_image( + self, mm_data: Mapping[str, object] + ) -> list[int] | None: + """Ask the HF processor how many rows of image data each image produces.""" + images = mm_data.get("images") + if not images: + return None + try: + sizes = [(image.height, image.width) for image in images] + mm_tokens = self.info.get_hf_processor()._get_num_multimodal_tokens( + image_sizes=sizes, **self.info.ctx.get_merged_mm_kwargs({}) + ) + return list(mm_tokens["num_image_patches"]) + except (AttributeError, KeyError, TypeError): + return None + + def _call_hf_processor( + self, + prompt: str, + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], + tok_kwargs: Mapping[str, object], + ) -> "BatchFeature": + """Ask for the replacement each placeholder expands to, and record it as + per-item fields: its token ids, and the tokens or patches behind them.""" + if has_mm_data := any(mm_data.values()): + mm_data = {**mm_data, "return_text_replacement_offsets": True} + hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs) + # Drop the inputs the model would reject + hf_inputs.pop("mm_token_type_ids", None) + hf_inputs.pop("token_type_ids", None) + + offsets = hf_inputs.pop("text_replacement_offsets", None) + # Some processors return an empty batch as a tensor rather than a list + if offsets is None or len(offsets) == 0 or len(offsets[0]) == 0: + if has_mm_data: + raise ValueError( + f"{type(self.info.get_hf_processor()).__name__} returned no " + "text replacement offsets, so the Transformers modeling backend " + "cannot locate the placeholder of each item. Its `__call__` has " + "to reach `ProcessorMixin.get_text_with_replacements` with one " + "replacement per item, which usually means implementing " + "`replace__token`. Please report this to transformers " + "so it can be fixed, and install `transformers<5.15.0` in the " + "meantime to locate placeholders in the expanded prompt instead." + ) + return hf_inputs + + tokenizer = self.info.get_tokenizer() + replacements = defaultdict[str, list[list[int]]](list) + for entry in offsets[0]: + replacements[entry["type"]].append( + tokenizer.encode(entry["replacement"], add_special_tokens=False) + ) + + for modality, seqs in replacements.items(): + hf_inputs[f"{modality}_replacement_ids"] = torch.tensor( + [token_id for seq in seqs for token_id in seq] + ) + hf_inputs[f"{modality}_replacement_sizes"] = torch.tensor( + [len(seq) for seq in seqs] + ) + if modality == "image": + hf_inputs["num_image_patches"] = self._get_num_image_patches( + hf_inputs, mm_data, len(seqs) + ) + elif modality == "audio": + counts = [] + for seq in seqs: + ids = torch.tensor(seq) + counts.append(int(ids.eq(_get_embed_token_id(ids)).sum())) + hf_inputs["num_audio_tokens"] = torch.tensor(counts) + + return hf_inputs + + +# From this version on, a processor reporting no offsets is an error rather than a +# fallback to searching the expanded prompt +MultiModalProcessor = ( + LegacyMultiModalProcessor + if Version(transformers.__version__) < Version("5.15.0") + else OffsetsMultiModalProcessor +) + + class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): # Skip SupportsMRoPE.__init__ and call the next class in MRO @@ -733,18 +1105,32 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): kwargs.pop("token_type_ids", None) kwargs.pop("mm_token_type_ids", None) - # HuggingFace's `get_audio_features` implementations branch on - # per-sample feature lengths internally. - with gpu_sync_allowed(): - audio_output = self.model.get_audio_features( - input_features, return_dict=True, **kwargs - ) - audio_embeddings = audio_output.pooler_output - # Per-audio token counts are needed as Python ints to split. with gpu_sync_allowed(): split_sizes = num_audio_tokens.flatten().tolist() - return self._split_embeddings(audio_embeddings, split_sizes) + if isinstance(input_features, torch.Tensor): + # HuggingFace's `get_audio_features` implementations branch on + # per-sample feature lengths internally. + with gpu_sync_allowed(): + audio_output = self.model.get_audio_features( + input_features, return_dict=True, **kwargs + ) + return self._split_embeddings(audio_output.pooler_output, split_sizes) + + # Audios the processor left un-padded arrive as a list once their + # lengths differ. Encode them one at a time so that none of them is + # padded to match another. + embeddings: list[torch.Tensor] = [] + for index, features in enumerate(input_features): + audio_output = self.model.get_audio_features( + features.unsqueeze(0), + return_dict=True, + **self._select_item_kwargs(kwargs, index, len(input_features)), + ) + embeddings.extend( + self._split_embeddings(audio_output.pooler_output, [split_sizes[index]]) + ) + return embeddings def _process_image_input(self, **kwargs) -> list[torch.Tensor] | None: pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None) @@ -761,6 +1147,42 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): num_image_patches = kwargs.pop("num_image_patches") + split_sizes = num_image_patches.flatten().tolist() + if isinstance(pixel_values, torch.Tensor): + vision_embeddings = self._get_image_features(pixel_values, **kwargs) + if isinstance(vision_embeddings, torch.Tensor): + return self._split_embeddings(vision_embeddings, split_sizes) + return list(vision_embeddings) + + # Images the processor left un-padded arrive as a list once their + # shapes differ. Encode them one at a time so that none of them is + # padded to match another. + embeddings: list[torch.Tensor] = [] + for index, image in enumerate(pixel_values): + features = self._get_image_features( + image.unsqueeze(0), + **self._select_item_kwargs(kwargs, index, len(pixel_values)), + ) + # Encoders which return one entry per image return a single entry + if not isinstance(features, torch.Tensor): + features = torch.cat(list(features)) + embeddings.extend(self._split_embeddings(features, [split_sizes[index]])) + return embeddings + + def _select_item_kwargs( + self, kwargs: dict[str, Any], index: int, num_items: int + ) -> dict[str, Any]: + """Narrow the entries of `kwargs` that hold one row per item down to the item + at `index`. Length is all there is to match on, so an unrelated entry of the + same length is narrowed too.""" + return { + key: value[index : index + 1] + if isinstance(value, (torch.Tensor, list)) and len(value) == num_items + else value + for key, value in kwargs.items() + } + + def _get_image_features(self, pixel_values: torch.Tensor, **kwargs) -> Any: # grid_thw fields are registered keep_on_cpu; restore the on-device # placement that HF get_image_features implementations expect. for key, value in kwargs.items(): @@ -772,22 +1194,17 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): # padding images via boolean-mask indexing, LlavaOnevision # branches on per-sample batch counts). with gpu_sync_allowed(): - vision_embeddings = self.model.get_image_features(pixel_values, **kwargs) + features = self.model.get_image_features(pixel_values, **kwargs) # Transformers `v5`, `self.get_image_features` returns a tuple # containing the features and optionally attentions/hidden_states # After v5 is settled, we can enable qwen3-vl with several outputs # from `self.get_image_features` - if isinstance(vision_embeddings, tuple): - vision_embeddings = vision_embeddings[0] - elif isinstance(vision_embeddings, dict): - vision_embeddings = vision_embeddings.pooler_output - - if isinstance(vision_embeddings, torch.Tensor): - split_sizes = num_image_patches.flatten().tolist() - return self._split_embeddings(vision_embeddings, split_sizes) - - return list(vision_embeddings) + if isinstance(features, tuple): + return features[0] + if isinstance(features, dict): + return features.pooler_output + return features def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings: # Each helper detects its own inputs. We are called once per modality, so the