[3/N] Harden Transformers modelling backend multi-modal path (#51827)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -4,8 +4,11 @@ import numpy as np
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.config import ModelConfig
|
from vllm.config import ModelConfig
|
||||||
|
from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
|
|
||||||
|
from .transformers_backend import PROCESSOR_CLASSES, create_processor
|
||||||
|
|
||||||
AUDIO_MODEL_SETTINGS = {
|
AUDIO_MODEL_SETTINGS = {
|
||||||
"ibm-granite/granite-speech-3.3-2b": {
|
"ibm-granite/granite-speech-3.3-2b": {
|
||||||
"prompt": (
|
"prompt": (
|
||||||
@@ -53,6 +56,7 @@ AUDIO_MODEL_SETTINGS = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"model_id",
|
"model_id",
|
||||||
[
|
[
|
||||||
@@ -61,12 +65,12 @@ AUDIO_MODEL_SETTINGS = {
|
|||||||
pytest.param(
|
pytest.param(
|
||||||
"mistralai/Voxtral-Mini-3B-2507",
|
"mistralai/Voxtral-Mini-3B-2507",
|
||||||
marks=pytest.mark.xfail(
|
marks=pytest.mark.xfail(
|
||||||
reason="MistralCommonBackend.encode does not produce the audio "
|
reason="Voxtral's mistral_common processor does not compose with "
|
||||||
"placeholder token (ID 24) from raw text. apply_chat_template "
|
"the Transformers modelling backend. Loading it currently fails "
|
||||||
"yields token IDs with placeholders, but MultiModalProcessor."
|
"outright, because MistralCommonBackend.from_pretrained rejects "
|
||||||
"apply() decodes the prompt back to text and re-tokenizes, at "
|
"the kwargs vLLM passes, and it implements no "
|
||||||
"which point the placeholders are lost. Fix belongs in "
|
"`replace_audio_token`, so it would report no replacement "
|
||||||
"mistral_common or in the Voxtral-specific path.",
|
"offsets. Both fixes belong in mistral_common or transformers.",
|
||||||
strict=False,
|
strict=False,
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
@@ -74,15 +78,10 @@ AUDIO_MODEL_SETTINGS = {
|
|||||||
"zai-org/GLM-ASR-Nano-2512",
|
"zai-org/GLM-ASR-Nano-2512",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_audio_multimodal_processor(model_id):
|
def test_audio_multimodal_processor(model_id, processor_cls):
|
||||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||||
|
|
||||||
model_config = ModelConfig(
|
mm_processor = create_processor(model_id, processor_cls)
|
||||||
model=model_id,
|
|
||||||
model_impl="transformers",
|
|
||||||
)
|
|
||||||
|
|
||||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
|
||||||
|
|
||||||
audio = np.zeros(16000, dtype=np.float32)
|
audio = np.zeros(16000, dtype=np.float32)
|
||||||
mm_data = {"audio": (audio, 16000)}
|
mm_data = {"audio": (audio, 16000)}
|
||||||
@@ -113,10 +112,13 @@ def test_audio_multimodal_processor(model_id):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _process_granite_speech(separator: str):
|
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||||
|
@pytest.mark.parametrize("separator", [" and ", ""])
|
||||||
|
def test_audio_multiple_inputs(separator, processor_cls):
|
||||||
|
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||||
|
and multi-modal item by the Transformers modelling backend."""
|
||||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
mm_processor = create_processor(model_id, processor_cls)
|
||||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
|
||||||
|
|
||||||
audio_token = mm_processor.info.get_hf_processor().audio_token
|
audio_token = mm_processor.info.get_hf_processor().audio_token
|
||||||
# One token per audio; the processor expands each to its placeholder run.
|
# One token per audio; the processor expands each to its placeholder run.
|
||||||
@@ -126,17 +128,21 @@ def _process_granite_speech(separator: str):
|
|||||||
)
|
)
|
||||||
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
||||||
|
|
||||||
return mm_processor(
|
def process():
|
||||||
prompt=prompt,
|
return mm_processor(
|
||||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
prompt=prompt,
|
||||||
hf_processor_mm_kwargs={},
|
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||||
)
|
hf_processor_mm_kwargs={},
|
||||||
|
)
|
||||||
|
|
||||||
|
# The legacy path reads placeholders off contiguous runs of the audio token, so
|
||||||
|
# it cannot tell adjacent ones apart and says so instead of merging them
|
||||||
|
if processor_cls is LegacyMultiModalProcessor and not separator:
|
||||||
|
with pytest.raises(ValueError, match="Separate them in the prompt"):
|
||||||
|
process()
|
||||||
|
return
|
||||||
|
|
||||||
def test_audio_multiple_inputs():
|
result = process()
|
||||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
|
||||||
and multi-modal item by the Transformers modelling backend."""
|
|
||||||
result = _process_granite_speech(separator=" and ")
|
|
||||||
|
|
||||||
assert len(result["mm_placeholders"]["audio"]) == 2
|
assert len(result["mm_placeholders"]["audio"]) == 2
|
||||||
assert len(result["mm_kwargs"]["audio"]) == 2
|
assert len(result["mm_kwargs"]["audio"]) == 2
|
||||||
@@ -167,9 +173,3 @@ def test_unclaimed_fields_warn_rather_than_raise():
|
|||||||
|
|
||||||
assert owned["audio"] == ["input_features"]
|
assert owned["audio"] == ["input_features"]
|
||||||
assert owned["image"] == []
|
assert owned["image"] == []
|
||||||
|
|
||||||
|
|
||||||
def test_audio_adjacent_inputs():
|
|
||||||
"""Adjacent audios are rejected rather than silently merged into one placeholder."""
|
|
||||||
with pytest.raises(ValueError, match="told apart"):
|
|
||||||
_process_granite_speech(separator="")
|
|
||||||
|
|||||||
@@ -1,20 +1,27 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.assets.image import ImageAsset
|
from vllm.assets.image import ImageAsset
|
||||||
from vllm.config import ModelConfig
|
from vllm.model_executor.models.transformers.multimodal import (
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
LegacyMultiModalProcessor,
|
||||||
|
OffsetsMultiModalProcessor,
|
||||||
|
)
|
||||||
|
|
||||||
|
from .transformers_backend import (
|
||||||
|
PROCESSOR_CLASSES,
|
||||||
|
create_cached_processor,
|
||||||
|
create_processor,
|
||||||
|
offsets_only,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||||
def test_multimodal_processor(model_id):
|
def test_multimodal_processor(model_id, processor_cls):
|
||||||
model_config = ModelConfig(
|
mm_processor = create_processor(model_id, processor_cls)
|
||||||
model=model_id,
|
|
||||||
model_impl="transformers",
|
|
||||||
)
|
|
||||||
|
|
||||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
|
||||||
|
|
||||||
image_pil = ImageAsset("cherry_blossom").pil_image
|
image_pil = ImageAsset("cherry_blossom").pil_image
|
||||||
mm_data = {"image": image_pil}
|
mm_data = {"image": image_pil}
|
||||||
@@ -56,10 +63,9 @@ def test_multimodal_processor(model_id):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _process_two_images(separator: str):
|
def _process_two_images(processor_cls, separator: str):
|
||||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
mm_processor = create_processor(model_id, processor_cls)
|
||||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
|
||||||
|
|
||||||
image = ImageAsset("cherry_blossom").pil_image
|
image = ImageAsset("cherry_blossom").pil_image
|
||||||
prompt = (
|
prompt = (
|
||||||
@@ -74,28 +80,147 @@ def _process_two_images(separator: str):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_image_multiple_inputs():
|
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||||
|
def test_image_multiple_inputs(processor_cls):
|
||||||
"""Multiple images per prompt are each detected as a separate placeholder
|
"""Multiple images per prompt are each detected as a separate placeholder
|
||||||
and multi-modal item by the Transformers modelling backend."""
|
and multi-modal item by the Transformers modelling backend."""
|
||||||
result = _process_two_images(separator="\n and ")
|
result = _process_two_images(processor_cls, separator="\n and ")
|
||||||
|
|
||||||
assert len(result["mm_placeholders"]["image"]) == 2
|
assert len(result["mm_placeholders"]["image"]) == 2
|
||||||
assert len(result["mm_kwargs"]["image"]) == 2
|
assert len(result["mm_kwargs"]["image"]) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_image_adjacent_inputs():
|
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||||
|
def test_image_adjacent_inputs(processor_cls):
|
||||||
"""Adjacent images stay separate placeholders rather than merging into one."""
|
"""Adjacent images stay separate placeholders rather than merging into one."""
|
||||||
result = _process_two_images(separator="")
|
result = _process_two_images(processor_cls, separator="")
|
||||||
|
|
||||||
assert len(result["mm_placeholders"]["image"]) == 2
|
assert len(result["mm_placeholders"]["image"]) == 2
|
||||||
assert len(result["mm_kwargs"]["image"]) == 2
|
assert len(result["mm_kwargs"]["image"]) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_text_only_prompt():
|
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||||
|
def test_batch_padding_removed_from_image_items(processor_cls):
|
||||||
|
"""Emu3 pads every image up to the largest in the batch, which would leave an
|
||||||
|
item's data dependent on what it was processed with and so uncacheable."""
|
||||||
|
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
|
||||||
|
image_token = mm_processor.info.get_hf_processor().image_token
|
||||||
|
|
||||||
|
images = [
|
||||||
|
ImageAsset("cherry_blossom").pil_image,
|
||||||
|
ImageAsset("cherry_blossom").pil_image.resize((256, 1024)),
|
||||||
|
]
|
||||||
|
result = mm_processor(
|
||||||
|
prompt=f"{image_token} and {image_token}",
|
||||||
|
mm_items=mm_processor.info.parse_mm_data({"image": images}),
|
||||||
|
hf_processor_mm_kwargs={},
|
||||||
|
)
|
||||||
|
|
||||||
|
items = result["mm_kwargs"]["image"]
|
||||||
|
shapes = set()
|
||||||
|
for item in items:
|
||||||
|
height, width = item["image_sizes"].data.flatten().tolist()
|
||||||
|
pixel_values = item["pixel_values"].data
|
||||||
|
assert tuple(pixel_values.shape[-2:]) == (height, width)
|
||||||
|
shapes.add(tuple(pixel_values.shape))
|
||||||
|
|
||||||
|
# Both images would have been padded to a common shape had they been kept
|
||||||
|
assert len(shapes) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def _process_one_gemma3_image(processor_cls):
|
||||||
|
mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
|
||||||
|
hf_processor = mm_processor.info.get_hf_processor()
|
||||||
|
result = mm_processor(
|
||||||
|
prompt=f"{hf_processor.boi_token} What is this?",
|
||||||
|
mm_items=mm_processor.info.parse_mm_data(
|
||||||
|
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||||
|
),
|
||||||
|
hf_processor_mm_kwargs={},
|
||||||
|
)
|
||||||
|
return hf_processor, result
|
||||||
|
|
||||||
|
|
||||||
|
@offsets_only
|
||||||
|
def test_non_embedding_tokens_excluded_from_placeholders():
|
||||||
|
"""Gemma3 wraps each image in text that carries no embeddings, which must be
|
||||||
|
inside the placeholder range but masked out of it."""
|
||||||
|
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
|
||||||
|
|
||||||
|
(placeholder,) = result["mm_placeholders"]["image"]
|
||||||
|
assert placeholder.is_embed is not None
|
||||||
|
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_placeholders_hold_only_image_tokens():
|
||||||
|
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
|
||||||
|
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
|
||||||
|
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
|
||||||
|
marker in the unexpanded prompt, not the token the expansion repeats."""
|
||||||
|
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
|
||||||
|
|
||||||
|
(placeholder,) = result["mm_placeholders"]["image"]
|
||||||
|
assert placeholder.length == hf_processor.image_seq_length
|
||||||
|
prompt_ids = result["prompt_token_ids"]
|
||||||
|
covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
|
||||||
|
assert set(covered) == {hf_processor.tokenizer.image_token_id}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||||
|
def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||||
|
"""SmolVLM splits each image into tiles introduced by tokens carrying no
|
||||||
|
embeddings. Those belong inside the placeholder and masked out, because the token
|
||||||
|
count the processor reports is over the whole span. Idefics3 also refuses a prompt
|
||||||
|
holding `<image>` when no images are passed, which is how the offsets path has to
|
||||||
|
tokenize it before splicing in the expansion."""
|
||||||
|
mm_processor = create_processor(
|
||||||
|
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||||
|
)
|
||||||
|
result = mm_processor(
|
||||||
|
prompt="<image>What is this?",
|
||||||
|
mm_items=mm_processor.info.parse_mm_data(
|
||||||
|
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||||
|
),
|
||||||
|
hf_processor_mm_kwargs={},
|
||||||
|
)
|
||||||
|
|
||||||
|
(placeholder,) = result["mm_placeholders"]["image"]
|
||||||
|
assert placeholder.is_embed is not None
|
||||||
|
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
|
||||||
|
|
||||||
|
|
||||||
|
@offsets_only
|
||||||
|
def test_missing_replacement_offsets_names_the_processor():
|
||||||
|
"""A processor that reports no replacement offsets cannot be served, which must
|
||||||
|
be said plainly rather than surfacing later as a field config mismatch."""
|
||||||
|
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||||
|
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
|
||||||
|
hf_processor_cls = type(mm_processor.info.get_hf_processor())
|
||||||
|
hf_call = hf_processor_cls.__call__
|
||||||
|
|
||||||
|
def without_offsets(self, *args, **kwargs):
|
||||||
|
hf_inputs = hf_call(self, *args, **kwargs)
|
||||||
|
hf_inputs.pop("text_replacement_offsets", None)
|
||||||
|
return hf_inputs
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(hf_processor_cls, "__call__", without_offsets),
|
||||||
|
pytest.raises(ValueError, match="LlavaOnevisionProcessor returned no"),
|
||||||
|
):
|
||||||
|
mm_processor(
|
||||||
|
prompt="<image>\nWhat is the content of this image?",
|
||||||
|
mm_items=mm_processor.info.parse_mm_data(
|
||||||
|
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||||
|
),
|
||||||
|
hf_processor_mm_kwargs={},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||||
|
def test_text_only_prompt(processor_cls):
|
||||||
"""An image model still accepts a prompt with no images."""
|
"""An image model still accepts a prompt with no images."""
|
||||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
mm_processor = create_processor(model_id, processor_cls)
|
||||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
|
||||||
|
|
||||||
result = mm_processor(
|
result = mm_processor(
|
||||||
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
|
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
|
||||||
@@ -105,3 +230,87 @@ def test_text_only_prompt():
|
|||||||
|
|
||||||
assert len(result["prompt_token_ids"]) > 0
|
assert len(result["prompt_token_ids"]) > 0
|
||||||
assert not result["mm_placeholders"]
|
assert not result["mm_placeholders"]
|
||||||
|
|
||||||
|
|
||||||
|
@offsets_only
|
||||||
|
def test_repeated_image_hits_the_processor_cache():
|
||||||
|
"""Check that mm caching is actually working."""
|
||||||
|
mm_processor, cache = create_cached_processor(
|
||||||
|
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
|
||||||
|
)
|
||||||
|
image = ImageAsset("cherry_blossom").pil_image
|
||||||
|
|
||||||
|
def process():
|
||||||
|
return mm_processor(
|
||||||
|
prompt="<image>\nWhat is this?",
|
||||||
|
mm_items=mm_processor.info.parse_mm_data({"image": image}),
|
||||||
|
hf_processor_mm_kwargs={},
|
||||||
|
)
|
||||||
|
|
||||||
|
first, second = process(), process()
|
||||||
|
|
||||||
|
assert cache.make_stats().hits > 0
|
||||||
|
assert first["prompt_token_ids"] == second["prompt_token_ids"]
|
||||||
|
assert first["mm_hashes"] == second["mm_hashes"]
|
||||||
|
|
||||||
|
|
||||||
|
@offsets_only
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("model_id", "prompt"),
|
||||||
|
[
|
||||||
|
("llava-hf/llava-onevision-qwen2-0.5b-ov-hf", "<image>\nWhat is this?"),
|
||||||
|
("google/gemma-3-4b-it", "<start_of_image> What is this?"),
|
||||||
|
("HuggingFaceTB/SmolVLM-256M-Instruct", "<image>What is this?"),
|
||||||
|
pytest.param(
|
||||||
|
"BAAI/Emu3-Chat-hf",
|
||||||
|
"<image> and more text",
|
||||||
|
marks=pytest.mark.xfail(
|
||||||
|
reason="Emu3Processor prepends its BOS token only when images are "
|
||||||
|
"passed, so the unexpanded prompt the offsets path tokenizes never "
|
||||||
|
"gets one. Fixed by huggingface/transformers#47924, unreleased.",
|
||||||
|
strict=False,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
|
||||||
|
"""The offsets path splices the expansion into a prompt tokenized without any
|
||||||
|
multi-modal data, so its token ids have to come out the same as the ones the HF
|
||||||
|
processor produces itself, which is what the legacy path returns."""
|
||||||
|
prompt_ids = []
|
||||||
|
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
|
||||||
|
mm_processor = create_processor(model_id, processor_cls)
|
||||||
|
prompt_ids.append(
|
||||||
|
mm_processor(
|
||||||
|
prompt=prompt,
|
||||||
|
mm_items=mm_processor.info.parse_mm_data(
|
||||||
|
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||||
|
),
|
||||||
|
hf_processor_mm_kwargs={},
|
||||||
|
)["prompt_token_ids"]
|
||||||
|
)
|
||||||
|
|
||||||
|
legacy_ids, offsets_ids = prompt_ids
|
||||||
|
assert legacy_ids == offsets_ids
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||||
|
def test_nested_image_fields_split_per_image(processor_cls):
|
||||||
|
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
|
||||||
|
belonging to each image one dimension further in. Slicing the batch dimension
|
||||||
|
instead handed the first image every row and the second an empty tensor."""
|
||||||
|
mm_processor = create_processor(
|
||||||
|
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||||
|
)
|
||||||
|
image = ImageAsset("cherry_blossom").pil_image
|
||||||
|
result = mm_processor(
|
||||||
|
prompt="<image> and <image>",
|
||||||
|
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
|
||||||
|
hf_processor_mm_kwargs={},
|
||||||
|
)
|
||||||
|
|
||||||
|
items = result["mm_kwargs"]["image"]
|
||||||
|
assert len(items) == 2
|
||||||
|
for item in items:
|
||||||
|
pixel_values = item["pixel_values"].data
|
||||||
|
assert pixel_values.shape[1] == int(item["num_image_patches"].data)
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Helpers for testing both processors of the Transformers modelling backend."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.config import ModelConfig
|
||||||
|
from vllm.model_executor.models.transformers.multimodal import (
|
||||||
|
LegacyMultiModalProcessor,
|
||||||
|
MultiModalDummyInputsBuilder,
|
||||||
|
MultiModalProcessingInfo,
|
||||||
|
MultiModalProcessor,
|
||||||
|
OffsetsMultiModalProcessor,
|
||||||
|
)
|
||||||
|
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||||
|
from vllm.multimodal.processing import InputProcessingContext
|
||||||
|
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||||
|
|
||||||
|
offsets_only = pytest.mark.skipif(
|
||||||
|
MultiModalProcessor is not OffsetsMultiModalProcessor,
|
||||||
|
reason="Replacement offsets are only used from transformers 5.15.0 onwards",
|
||||||
|
)
|
||||||
|
|
||||||
|
PROCESSOR_CLASSES = [
|
||||||
|
pytest.param(LegacyMultiModalProcessor, id="legacy"),
|
||||||
|
pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def create_processor(model_id: str, processor_cls):
|
||||||
|
"""Build a processor directly, because the registry only ever builds the one the
|
||||||
|
installed transformers version selects, leaving the other path untested."""
|
||||||
|
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||||
|
ctx = InputProcessingContext(
|
||||||
|
model_config, cached_tokenizer_from_config(model_config)
|
||||||
|
)
|
||||||
|
info = MultiModalProcessingInfo(ctx)
|
||||||
|
return processor_cls(info, MultiModalDummyInputsBuilder(info))
|
||||||
|
|
||||||
|
|
||||||
|
def create_cached_processor(model_id: str, processor_cls):
|
||||||
|
"""Build a processor backed by a real multi-modal processor cache, and hand the
|
||||||
|
cache back so a test can check it was actually used."""
|
||||||
|
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||||
|
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||||
|
ctx = InputProcessingContext(
|
||||||
|
model_config, cached_tokenizer_from_config(model_config)
|
||||||
|
)
|
||||||
|
info = MultiModalProcessingInfo(ctx)
|
||||||
|
cache = MultiModalProcessorOnlyCache(model_config)
|
||||||
|
return processor_cls(info, MultiModalDummyInputsBuilder(info), cache=cache), cache
|
||||||
@@ -22,6 +22,8 @@ from contextlib import ExitStack, contextmanager
|
|||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
import transformers
|
||||||
|
from packaging.version import Version
|
||||||
|
|
||||||
from vllm.compilation.decorators import should_torch_compile_mm_encoder
|
from vllm.compilation.decorators import should_torch_compile_mm_encoder
|
||||||
from vllm.config.utils import getattr_iter
|
from vllm.config.utils import getattr_iter
|
||||||
@@ -50,7 +52,9 @@ from vllm.multimodal.processing import (
|
|||||||
BaseMultiModalProcessor,
|
BaseMultiModalProcessor,
|
||||||
BaseProcessingInfo,
|
BaseProcessingInfo,
|
||||||
ProcessorInputs,
|
ProcessorInputs,
|
||||||
|
PromptReplacement,
|
||||||
PromptUpdate,
|
PromptUpdate,
|
||||||
|
PromptUpdateDetails,
|
||||||
TimingContext,
|
TimingContext,
|
||||||
)
|
)
|
||||||
from vllm.sequence import IntermediateTensors
|
from vllm.sequence import IntermediateTensors
|
||||||
@@ -65,6 +69,12 @@ if TYPE_CHECKING:
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
_MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3}
|
_MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3}
|
||||||
|
_MODALITY_SIZE_KEYS = {"audio": "num_audio_tokens", "image": "num_image_patches"}
|
||||||
|
|
||||||
|
|
||||||
|
def _get_embed_token_id(replacement_ids: torch.Tensor) -> int:
|
||||||
|
"""The token an expansion repeats is the one holding the embeddings."""
|
||||||
|
return int(replacement_ids.mode().values)
|
||||||
|
|
||||||
|
|
||||||
class MultiModalProcessingInfo(BaseProcessingInfo):
|
class MultiModalProcessingInfo(BaseProcessingInfo):
|
||||||
@@ -94,21 +104,6 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
|
|||||||
)
|
)
|
||||||
return modalities
|
return modalities
|
||||||
|
|
||||||
def _get_audio_token_id(self) -> int:
|
|
||||||
processor = self.get_hf_processor()
|
|
||||||
if hasattr(processor, "audio_token_id"):
|
|
||||||
return processor.audio_token_id
|
|
||||||
config = self.get_hf_config()
|
|
||||||
val = getattr_iter(config, ("audio_token_id", "audio_token_index"))
|
|
||||||
if val is not None:
|
|
||||||
return val
|
|
||||||
if hasattr(processor, "audio_token"):
|
|
||||||
tokenizer = self.get_tokenizer()
|
|
||||||
vocab = tokenizer.get_vocab()
|
|
||||||
if processor.audio_token in vocab:
|
|
||||||
return vocab[processor.audio_token]
|
|
||||||
raise ValueError("Cannot find audio_token_id on processor or model config")
|
|
||||||
|
|
||||||
def _get_audio_sampling_rate(self) -> float:
|
def _get_audio_sampling_rate(self) -> float:
|
||||||
sub = self._get_audio_processor()
|
sub = self._get_audio_processor()
|
||||||
if sub is not None and hasattr(sub, "sampling_rate"):
|
if sub is not None and hasattr(sub, "sampling_rate"):
|
||||||
@@ -169,7 +164,7 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
|||||||
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
|
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
|
||||||
processor = self.info.get_hf_processor()
|
processor = self.info.get_hf_processor()
|
||||||
audio_token = getattr(processor, "audio_token", "")
|
audio_token = getattr(processor, "audio_token", "")
|
||||||
# Separated so that `_apply_audio` can tell the placeholders apart
|
# Separated so that adjacent placeholders stay distinguishable
|
||||||
text += " ".join([audio_token] * num_audios)
|
text += " ".join([audio_token] * num_audios)
|
||||||
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
|
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
|
||||||
processor = self.info.get_hf_processor()
|
processor = self.info.get_hf_processor()
|
||||||
@@ -210,23 +205,26 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
|||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||||
def _get_prompt_updates(
|
"""Processing common to both Transformers backend processors: calling the HF
|
||||||
|
processor, sizing images, and attributing its outputs to a modality.
|
||||||
|
|
||||||
|
Subclasses add the strategy for locating placeholders in the prompt.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _get_hf_mm_data(
|
||||||
self,
|
self,
|
||||||
mm_items: MultiModalDataItems,
|
mm_items: MultiModalDataItems,
|
||||||
hf_processor_mm_kwargs: Mapping[str, object],
|
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||||
out_mm_kwargs: MultiModalKwargsItems,
|
"""Rename the parser's `audios` key to the `audio` argument HF audio
|
||||||
) -> Sequence[PromptUpdate]:
|
processors take."""
|
||||||
"""No updates: `apply` locates placeholders via `mm_token_type_ids` instead.
|
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||||
|
if self.info._is_audio_model() and "audios" in processor_data:
|
||||||
HF processors have no generic contract for the token sequence they insert,
|
processor_data["audio"] = processor_data.pop("audios")
|
||||||
so it cannot be expressed as a `PromptUpdate`. Returning nothing is only
|
return processor_data, passthrough_data
|
||||||
safe because `apply` is overridden; the base class would reject it.
|
|
||||||
"""
|
|
||||||
return []
|
|
||||||
|
|
||||||
def _get_modality_field_names(self, modality: str) -> set[str]:
|
def _get_modality_field_names(self, modality: str) -> set[str]:
|
||||||
"""Field names the sub-processor for `modality` produces."""
|
"""Names of the fields the sub-processor for `modality` produces."""
|
||||||
# TODO: use else branch only once huggingface/transformers#44394 lands.
|
# TODO: use else branch only once huggingface/transformers#44394 lands.
|
||||||
if modality == "audio":
|
if modality == "audio":
|
||||||
sub_processor = self.info._get_audio_processor()
|
sub_processor = self.info._get_audio_processor()
|
||||||
@@ -273,6 +271,18 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
|
|
||||||
return owned
|
return owned
|
||||||
|
|
||||||
|
def _get_slice_dim(self, data: Any, total_rows: int) -> int:
|
||||||
|
"""Which dimension of a field holds the rows belonging to each item.
|
||||||
|
|
||||||
|
Some processors (e.g., Idefics3) return image fields with a leading batch
|
||||||
|
dimension, putting the rows one dimension further in.
|
||||||
|
"""
|
||||||
|
if not isinstance(data, torch.Tensor) or data.ndim < 2:
|
||||||
|
return 0
|
||||||
|
if data.shape[0] != total_rows and data.shape[1] == total_rows:
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
def _get_mm_fields_config(
|
def _get_mm_fields_config(
|
||||||
self,
|
self,
|
||||||
hf_inputs: "BatchFeature",
|
hf_inputs: "BatchFeature",
|
||||||
@@ -281,29 +291,50 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
# HF Processors always return a mask but vLLM doesn't need it
|
# HF Processors always return a mask but vLLM doesn't need it
|
||||||
hf_inputs.pop("attention_mask", None)
|
hf_inputs.pop("attention_mask", None)
|
||||||
|
|
||||||
# Written by `_apply_audio`/`_apply_vision`; absent if the modality had no items
|
# Absent if the modality had no items
|
||||||
sizes = {
|
sizes = {
|
||||||
"audio": hf_inputs.get("num_audio_tokens"),
|
modality: hf_inputs.get(key)
|
||||||
"image": hf_inputs.get("num_image_patches"),
|
for modality, key in _MODALITY_SIZE_KEYS.items()
|
||||||
}
|
}
|
||||||
modalities = [m for m, size in sizes.items() if size is not None]
|
modalities = [m for m, size in sizes.items() if size is not None]
|
||||||
|
|
||||||
size_keys = {"num_audio_tokens", "num_image_patches"}
|
# Keys we wrote ourselves, rather than ones a sub-processor produced
|
||||||
keys = [key for key in hf_inputs if key not in size_keys]
|
own_keys = set(_MODALITY_SIZE_KEYS.values()) | {
|
||||||
|
f"{modality}_replacement_{suffix}"
|
||||||
|
for modality in modalities
|
||||||
|
for suffix in ("ids", "sizes")
|
||||||
|
}
|
||||||
|
keys = [key for key in hf_inputs if key not in own_keys]
|
||||||
owned = self._partition_keys_by_modality(keys, modalities)
|
owned = self._partition_keys_by_modality(keys, modalities)
|
||||||
|
|
||||||
|
# Un-padded fields are already one entry per item, so index rather than slice
|
||||||
mm_fields: dict[str, MultiModalFieldConfig] = {
|
mm_fields: dict[str, MultiModalFieldConfig] = {
|
||||||
key: MultiModalFieldConfig.flat_from_sizes(modality, sizes[modality])
|
key: MultiModalFieldConfig.batched(modality)
|
||||||
|
if modality == "audio" or isinstance(hf_inputs[key], list)
|
||||||
|
else MultiModalFieldConfig.flat_from_sizes(
|
||||||
|
modality,
|
||||||
|
sizes[modality],
|
||||||
|
dim=self._get_slice_dim(hf_inputs[key], int(sizes[modality].sum())),
|
||||||
|
)
|
||||||
for modality in modalities
|
for modality in modalities
|
||||||
for key in owned[modality]
|
for key in owned[modality]
|
||||||
}
|
}
|
||||||
|
|
||||||
# Keep these as batched, as they always have batch size as first dim
|
for modality in modalities:
|
||||||
if "audio" in modalities:
|
# One row per item, and only ever read on the CPU
|
||||||
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched(
|
mm_fields[_MODALITY_SIZE_KEYS[modality]] = MultiModalFieldConfig.batched(
|
||||||
"audio", keep_on_cpu=True
|
modality, keep_on_cpu=True
|
||||||
)
|
)
|
||||||
|
replacement_sizes = hf_inputs.get(f"{modality}_replacement_sizes")
|
||||||
|
if replacement_sizes is not None:
|
||||||
|
mm_fields[f"{modality}_replacement_ids"] = (
|
||||||
|
MultiModalFieldConfig.flat_from_sizes(
|
||||||
|
modality, replacement_sizes, keep_on_cpu=True
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
if "image" in modalities:
|
if "image" in modalities:
|
||||||
|
# Always one row per item, whatever they describe
|
||||||
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched(
|
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched(
|
||||||
"image", keep_on_cpu=True
|
"image", keep_on_cpu=True
|
||||||
)
|
)
|
||||||
@@ -311,25 +342,154 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched(
|
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched(
|
||||||
"image", keep_on_cpu=True
|
"image", keep_on_cpu=True
|
||||||
)
|
)
|
||||||
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
|
|
||||||
"image", keep_on_cpu=True
|
|
||||||
)
|
|
||||||
return mm_fields
|
return mm_fields
|
||||||
|
|
||||||
def _get_hf_mm_data(
|
def _call_hf_processor(
|
||||||
|
self,
|
||||||
|
prompt: str,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
mm_kwargs: Mapping[str, object],
|
||||||
|
tok_kwargs: Mapping[str, object],
|
||||||
|
) -> "BatchFeature":
|
||||||
|
"""Run the HF processor and unpad inputs."""
|
||||||
|
try:
|
||||||
|
hf_inputs = super()._call_hf_processor(
|
||||||
|
prompt, mm_data, mm_kwargs, tok_kwargs
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
if any(mm_data.values()):
|
||||||
|
raise
|
||||||
|
# Some processors reject a prompt holding placeholders with
|
||||||
|
# no data to go with them, so tokenize it without them
|
||||||
|
prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs)
|
||||||
|
hf_inputs = transformers.BatchFeature(
|
||||||
|
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||||
|
)
|
||||||
|
self._unpad_images(hf_inputs)
|
||||||
|
self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs)
|
||||||
|
return hf_inputs
|
||||||
|
|
||||||
|
def _unpad_audios(
|
||||||
|
self,
|
||||||
|
hf_inputs: "BatchFeature",
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
mm_kwargs: Mapping[str, object],
|
||||||
|
tok_kwargs: Mapping[str, object],
|
||||||
|
) -> None:
|
||||||
|
"""Replace the audio fields with each audio processed on its own.
|
||||||
|
|
||||||
|
Processors pad every audio up to the longest in the call, which would leave
|
||||||
|
an item's data dependent on what it was processed with. Unlike images,
|
||||||
|
nothing in the output states how long each one really is, and processors
|
||||||
|
pad a lone audio too, so the only way to know what an audio produces by
|
||||||
|
itself is to process it by itself.
|
||||||
|
"""
|
||||||
|
audios = mm_data.get("audio")
|
||||||
|
if not audios or len({len(audio) for audio in audios}) == 1:
|
||||||
|
return
|
||||||
|
|
||||||
|
alone = [
|
||||||
|
self.info.ctx.call_hf_processor(
|
||||||
|
self.info.get_hf_processor(**mm_kwargs),
|
||||||
|
dict(
|
||||||
|
text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
|
||||||
|
),
|
||||||
|
dict(**mm_kwargs, **tok_kwargs),
|
||||||
|
)
|
||||||
|
for audio in audios
|
||||||
|
]
|
||||||
|
|
||||||
|
for key in self._get_modality_field_names("audio"):
|
||||||
|
if isinstance(hf_inputs.get(key), torch.Tensor):
|
||||||
|
hf_inputs[key] = [output[key][0] for output in alone]
|
||||||
|
|
||||||
|
def _unpad_images(self, hf_inputs: "BatchFeature") -> None:
|
||||||
|
"""Trim each image back to its own size when the processor padded them all
|
||||||
|
to the largest in the batch.
|
||||||
|
|
||||||
|
An image's data has to depend on nothing but that image, or the multi-modal
|
||||||
|
processor cache would store it under that image's hash and later reuse it
|
||||||
|
beside a different neighbour. Padding is re-applied when the encoder runs.
|
||||||
|
"""
|
||||||
|
pixel_values = hf_inputs.get("pixel_values")
|
||||||
|
image_sizes = hf_inputs.get("image_sizes")
|
||||||
|
if not isinstance(pixel_values, torch.Tensor):
|
||||||
|
return
|
||||||
|
if not isinstance(image_sizes, torch.Tensor):
|
||||||
|
return
|
||||||
|
if pixel_values.ndim != 4 or len(pixel_values) != len(image_sizes):
|
||||||
|
return
|
||||||
|
|
||||||
|
# The sizes describe the trailing dimensions only if the largest of them is
|
||||||
|
# what the batch was padded up to. Otherwise they mean something else, as
|
||||||
|
# in llava-onevision, where they are the sizes before any processing.
|
||||||
|
maxima = image_sizes.max(dim=0).values
|
||||||
|
if maxima.tolist() != list(pixel_values.shape[-2:]):
|
||||||
|
return
|
||||||
|
|
||||||
|
hf_inputs["pixel_values"] = [
|
||||||
|
image[..., :height, :width]
|
||||||
|
for image, (height, width) in zip(pixel_values, image_sizes.tolist())
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||||
|
"""Locates placeholders by searching the prompt the HF processor has already
|
||||||
|
expanded for the tokens of each modality.
|
||||||
|
|
||||||
|
Serves transformers versions with no `return_text_replacement_offsets`.
|
||||||
|
Placeholders found this way cannot be rebuilt from an unexpanded prompt, so
|
||||||
|
this processor overrides `apply` and gets no multi-modal processor cache.
|
||||||
|
Remove it once `requirements/common.txt` requires `transformers>=5.15.0`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _get_prompt_updates(
|
||||||
self,
|
self,
|
||||||
mm_items: MultiModalDataItems,
|
mm_items: MultiModalDataItems,
|
||||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
out_mm_kwargs: MultiModalKwargsItems,
|
||||||
|
) -> Sequence[PromptUpdate]:
|
||||||
|
"""Empty, because `apply` writes the placeholder ranges itself rather than
|
||||||
|
deriving them from updates."""
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _call_hf_processor(
|
||||||
|
self,
|
||||||
|
prompt: str,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
mm_kwargs: Mapping[str, object],
|
||||||
|
tok_kwargs: Mapping[str, object],
|
||||||
|
) -> "BatchFeature":
|
||||||
|
"""Ask which modality owns each token of the expanded prompt, which is the
|
||||||
|
only marking of the tokens an expansion adds around an item."""
|
||||||
|
if any(mm_data.values()):
|
||||||
|
mm_data = {**mm_data, "return_mm_token_type_ids": True}
|
||||||
|
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||||
|
|
||||||
|
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||||
|
"""Token ids marking where `modality` sits in the prompt, which for some
|
||||||
|
processors differ from the placeholder written into it.
|
||||||
|
|
||||||
|
The expanded prompt is all this path has to go on, so it takes the
|
||||||
|
processor at its word about which tokens belong to the modality.
|
||||||
"""
|
"""
|
||||||
In contrast to the base class, this method requests
|
info = self.info
|
||||||
`return_mm_token_type_ids` and remaps the `audios` key to `audio` for
|
processor = info.get_hf_processor()
|
||||||
audio models.
|
declared = getattr(processor, f"{modality}_token_ids", None) or ()
|
||||||
"""
|
if ids := [token_id for token_id in declared if token_id is not None]:
|
||||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
return ids
|
||||||
if self.info._is_audio_model() and "audios" in processor_data:
|
config = info.get_hf_config()
|
||||||
processor_data["audio"] = processor_data.pop("audios")
|
names = (f"{modality}_token_id", f"{modality}_token_index")
|
||||||
processor_data["return_mm_token_type_ids"] = True
|
token_id = getattr(processor, names[0], getattr_iter(config, names))
|
||||||
return processor_data, passthrough_data
|
if token_id is None:
|
||||||
|
token = getattr(processor, f"{modality}_token", None)
|
||||||
|
token_id = info.get_tokenizer().get_vocab().get(token)
|
||||||
|
if token_id is None:
|
||||||
|
raise ValueError(
|
||||||
|
f"Cannot find {modality}_token_id on processor or model config"
|
||||||
|
)
|
||||||
|
return [token_id]
|
||||||
|
|
||||||
def _apply_audio(
|
def _apply_audio(
|
||||||
self,
|
self,
|
||||||
@@ -337,12 +497,17 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
processed_data: "BatchFeature",
|
processed_data: "BatchFeature",
|
||||||
num_audios: int,
|
num_audios: int,
|
||||||
) -> dict[str, list[PlaceholderRange]]:
|
) -> dict[str, list[PlaceholderRange]]:
|
||||||
audio_token_id = self.info._get_audio_token_id()
|
"""Take each contiguous run of the audio token as one item's placeholder,
|
||||||
|
and record how many tokens the run holds."""
|
||||||
|
audio_token_ids = self._get_mm_token_ids("audio")
|
||||||
prompt_tensor = torch.tensor(prompt_ids)
|
prompt_tensor = torch.tensor(prompt_ids)
|
||||||
is_audio = prompt_tensor == audio_token_id
|
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
|
||||||
|
|
||||||
if not is_audio.any():
|
if not is_audio.any():
|
||||||
return {}
|
raise ValueError(
|
||||||
|
f"{num_audios} audio item(s) were passed but the prompt "
|
||||||
|
"contains no audio token. Add one placeholder per audio item."
|
||||||
|
)
|
||||||
|
|
||||||
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
|
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
|
||||||
transitions = padded.int().diff()
|
transitions = padded.int().diff()
|
||||||
@@ -365,6 +530,23 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
processed_data["num_audio_tokens"] = lengths
|
processed_data["num_audio_tokens"] = lengths
|
||||||
return {"audio": ranges}
|
return {"audio": ranges}
|
||||||
|
|
||||||
|
def _get_num_multimodal_tokens(
|
||||||
|
self,
|
||||||
|
mm_items: MultiModalDataItems,
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
) -> Mapping[str, list[int]]:
|
||||||
|
"""Ask the HF processor how many tokens and patches each image expands to."""
|
||||||
|
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
images = mm_items.get_items("image", ImageProcessorItems)
|
||||||
|
image_sizes = [
|
||||||
|
(size.height, size.width)
|
||||||
|
for size in map(images.get_image_size, range(len(images)))
|
||||||
|
]
|
||||||
|
return processor._get_num_multimodal_tokens(
|
||||||
|
image_sizes=image_sizes,
|
||||||
|
**self.info.ctx.get_merged_mm_kwargs({}),
|
||||||
|
)
|
||||||
|
|
||||||
def _apply_vision(
|
def _apply_vision(
|
||||||
self,
|
self,
|
||||||
prompt_ids: list[int],
|
prompt_ids: list[int],
|
||||||
@@ -373,47 +555,46 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
hf_processor_mm_kwargs: Mapping[str, object],
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
mm_token_type_ids: torch.Tensor | None,
|
mm_token_type_ids: torch.Tensor | None,
|
||||||
) -> dict[str, list[PlaceholderRange]]:
|
) -> dict[str, list[PlaceholderRange]]:
|
||||||
# Placeholders can't be located without them, so give up rather than guess
|
"""Split the positions the processor marks as image into one placeholder per
|
||||||
if mm_token_type_ids is None:
|
item, sized by the token count it reports for each image."""
|
||||||
return {}
|
|
||||||
|
|
||||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
if mm_token_type_ids is None:
|
||||||
|
raise ValueError(
|
||||||
|
f"{type(hf_processor).__name__} returned no `mm_token_type_ids`, so "
|
||||||
|
"the Transformers modeling backend cannot locate the placeholder of "
|
||||||
|
"each image."
|
||||||
|
)
|
||||||
|
|
||||||
# We can infer vLLM style placeholder from token type ids, if we split
|
# These mark the tokens structuring an image as well as the image itself,
|
||||||
# it for each input `mm_data`.
|
# which is what the counts below are over
|
||||||
mm_positions = torch.where(mm_token_type_ids == 1)[1]
|
mm_positions = torch.where(mm_token_type_ids[0] == 1)[0]
|
||||||
images = mm_items.get_items("image", ImageProcessorItems)
|
mm_tokens_per_modality = self._get_num_multimodal_tokens(
|
||||||
image_sizes = []
|
mm_items, hf_processor_mm_kwargs
|
||||||
for item_idx in range(len(images)):
|
|
||||||
image_size = images.get_image_size(item_idx)
|
|
||||||
image_sizes.append((image_size.height, image_size.width))
|
|
||||||
|
|
||||||
mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
|
|
||||||
image_sizes=image_sizes,
|
|
||||||
**self.info.ctx.get_merged_mm_kwargs({}),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||||
split_sizes = mm_tokens_per_modality["num_image_tokens"]
|
split_sizes = mm_tokens_per_modality["num_image_tokens"]
|
||||||
if split_sizes:
|
if sum(split_sizes) != len(mm_positions):
|
||||||
image_token_ids = getattr(hf_processor, "image_token_ids", None)
|
raise ValueError(
|
||||||
if image_token_ids is None:
|
f"The expanded prompt holds {len(mm_positions)} image token(s) but "
|
||||||
# Transformers <5.10.0
|
f"{type(hf_processor).__name__} accounts for {sum(split_sizes)} "
|
||||||
image_token_ids = [hf_processor.image_token_id]
|
f"across {mm_items.get_count('image')} image item(s)."
|
||||||
image_token_ids = torch.tensor(
|
|
||||||
[i for i in image_token_ids if i is not None]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
chunked_mm_positions = torch.split(mm_positions, split_sizes)
|
if split_sizes:
|
||||||
mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
|
image_token_ids = torch.tensor(self._get_mm_token_ids("image"))
|
||||||
chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
|
mm_tokens = torch.tensor(prompt_ids)[mm_positions]
|
||||||
ranges = [
|
ranges = [
|
||||||
PlaceholderRange(
|
PlaceholderRange(
|
||||||
offset=positions[0].item(),
|
offset=positions[0].item(),
|
||||||
length=positions.shape[0],
|
length=positions.shape[0],
|
||||||
is_embed=torch.isin(mm_tokens, image_token_ids),
|
# Only some of the span carries embeddings
|
||||||
|
is_embed=torch.isin(tokens, image_token_ids),
|
||||||
|
)
|
||||||
|
for positions, tokens in zip(
|
||||||
|
torch.split(mm_positions, split_sizes),
|
||||||
|
torch.split(mm_tokens, split_sizes),
|
||||||
)
|
)
|
||||||
for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
|
|
||||||
]
|
]
|
||||||
mm_placeholders = {"image": ranges}
|
mm_placeholders = {"image": ranges}
|
||||||
|
|
||||||
@@ -427,12 +608,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
inputs: ProcessorInputs,
|
inputs: ProcessorInputs,
|
||||||
timing_ctx: TimingContext,
|
timing_ctx: TimingContext,
|
||||||
) -> MultiModalInput:
|
) -> MultiModalInput:
|
||||||
"""
|
"""Process the prompt and every multi-modal item in one HF processor call,
|
||||||
Process multi-modal inputs to be used in vLLM.
|
then read the placeholder ranges out of the token ids it returns."""
|
||||||
|
|
||||||
Apply HF Processor on prompt text and multi-modal data together,
|
|
||||||
outputting token IDs and processed tensors.
|
|
||||||
"""
|
|
||||||
prompt = inputs.prompt
|
prompt = inputs.prompt
|
||||||
mm_items = inputs.mm_data_items
|
mm_items = inputs.mm_data_items
|
||||||
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
||||||
@@ -468,7 +645,7 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||||
)
|
)
|
||||||
|
|
||||||
# For gemma3 we check `token_type_ids` as the key
|
# Gemma3 reports them under the key the model uses for its own token types
|
||||||
mm_token_type_ids = processed_data.pop("token_type_ids", None)
|
mm_token_type_ids = processed_data.pop("token_type_ids", None)
|
||||||
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
|
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
|
||||||
|
|
||||||
@@ -516,6 +693,201 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||||
|
"""Locates placeholders from the `text_replacement_offsets` the HF processor
|
||||||
|
reports, expressing each one as a `PromptUpdate`.
|
||||||
|
|
||||||
|
Stating the expansion as an update is what lets it be rebuilt from an
|
||||||
|
unexpanded prompt, so this processor takes the base class's processing path
|
||||||
|
and with it the multi-modal processor cache.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _get_prompt_updates(
|
||||||
|
self,
|
||||||
|
mm_items: MultiModalDataItems,
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
out_mm_kwargs: MultiModalKwargsItems,
|
||||||
|
) -> Sequence[PromptUpdate]:
|
||||||
|
"""Replace each modality's placeholder token with the token ids that item's
|
||||||
|
replacement text encodes to, marking which of them hold embeddings."""
|
||||||
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||||
|
updates = []
|
||||||
|
for modality, items in out_mm_kwargs.items():
|
||||||
|
# Popped so they are neither cached nor sent to the model; the updates
|
||||||
|
# they produce are cached alongside the item instead
|
||||||
|
replacements = [
|
||||||
|
PromptUpdateDetails.select_token_id(
|
||||||
|
(ids := item.pop(f"{modality}_replacement_ids").data).tolist(),
|
||||||
|
_get_embed_token_id(ids),
|
||||||
|
)
|
||||||
|
for item in items
|
||||||
|
]
|
||||||
|
updates.append(
|
||||||
|
PromptReplacement(
|
||||||
|
modality=modality,
|
||||||
|
target=getattr(hf_processor, f"{modality}_token"),
|
||||||
|
replacement=replacements.__getitem__,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return updates
|
||||||
|
|
||||||
|
def _apply_hf_processor_main(
|
||||||
|
self,
|
||||||
|
prompt: str | list[int],
|
||||||
|
mm_items: MultiModalDataItems,
|
||||||
|
hf_processor_mm_kwargs: Mapping[str, object],
|
||||||
|
tokenization_kwargs: Mapping[str, object],
|
||||||
|
*,
|
||||||
|
enable_hf_prompt_update: bool,
|
||||||
|
) -> tuple[list[int], "BatchFeature", bool]:
|
||||||
|
"""Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice
|
||||||
|
in, whatever `enable_hf_prompt_update` asks for.
|
||||||
|
|
||||||
|
This differs from `super()` only for a text prompt with
|
||||||
|
`enable_hf_prompt_update`, where `super()` would keep the expanded token
|
||||||
|
ids the HF processor returns. Both routes give the same ids, so forcing
|
||||||
|
this one costs an extra processor call and buys the guarantee that a
|
||||||
|
request bypassing the cache is identical to a cached one.
|
||||||
|
|
||||||
|
A text prompt only arrives with `enable_hf_prompt_update` when there is
|
||||||
|
no multi-modal processor cache. Requests carrying pre-computed
|
||||||
|
embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported
|
||||||
|
here, so they fail validation shortly afterwards.
|
||||||
|
"""
|
||||||
|
if isinstance(prompt, str) and enable_hf_prompt_update:
|
||||||
|
logger.warning_once(
|
||||||
|
"Disabling the multi-modal processor cache is extra slow with the "
|
||||||
|
"Transformers modeling backend: the prompt is still tokenized "
|
||||||
|
"unexpanded and the expansion spliced in, to keep the token ids "
|
||||||
|
"identical to what the cache produces, which costs an extra HF "
|
||||||
|
"processor call per request."
|
||||||
|
)
|
||||||
|
return super()._apply_hf_processor_main(
|
||||||
|
prompt,
|
||||||
|
mm_items,
|
||||||
|
hf_processor_mm_kwargs,
|
||||||
|
tokenization_kwargs,
|
||||||
|
enable_hf_prompt_update=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _get_num_image_patches(
|
||||||
|
self,
|
||||||
|
hf_inputs: "BatchFeature",
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
num_images: int,
|
||||||
|
) -> torch.Tensor:
|
||||||
|
"""How many rows of the image fields belong to each image.
|
||||||
|
|
||||||
|
Taken from whichever per-image count the processor reports,
|
||||||
|
and checked against the data it has to slice.
|
||||||
|
"""
|
||||||
|
if (grid := hf_inputs.get("image_grid_thw")) is not None:
|
||||||
|
num_patches = grid.prod(-1)
|
||||||
|
elif (counts := self._get_num_patches_per_image(mm_data)) is not None:
|
||||||
|
num_patches = torch.tensor(counts)
|
||||||
|
else:
|
||||||
|
num_patches = torch.ones(num_images, dtype=torch.long)
|
||||||
|
|
||||||
|
image_data = hf_inputs.get("pixel_values", hf_inputs.get("image_patches"))
|
||||||
|
if isinstance(image_data, torch.Tensor):
|
||||||
|
total = int(num_patches.sum())
|
||||||
|
rows = image_data.shape[self._get_slice_dim(image_data, total)]
|
||||||
|
if rows != total:
|
||||||
|
raise ValueError(
|
||||||
|
f"{type(self.info.get_hf_processor()).__name__} returned "
|
||||||
|
f"{rows} row(s) of image data for {num_images} image(s), which "
|
||||||
|
f"cannot be split into the {num_patches.tolist()} row(s) per "
|
||||||
|
"image derived from its outputs, so the rows cannot be "
|
||||||
|
"attributed to an image. Gemma3 does this when "
|
||||||
|
"`do_pan_and_scan` crops an image."
|
||||||
|
)
|
||||||
|
return num_patches
|
||||||
|
|
||||||
|
def _get_num_patches_per_image(
|
||||||
|
self, mm_data: Mapping[str, object]
|
||||||
|
) -> list[int] | None:
|
||||||
|
"""Ask the HF processor how many rows of image data each image produces."""
|
||||||
|
images = mm_data.get("images")
|
||||||
|
if not images:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
sizes = [(image.height, image.width) for image in images]
|
||||||
|
mm_tokens = self.info.get_hf_processor()._get_num_multimodal_tokens(
|
||||||
|
image_sizes=sizes, **self.info.ctx.get_merged_mm_kwargs({})
|
||||||
|
)
|
||||||
|
return list(mm_tokens["num_image_patches"])
|
||||||
|
except (AttributeError, KeyError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _call_hf_processor(
|
||||||
|
self,
|
||||||
|
prompt: str,
|
||||||
|
mm_data: Mapping[str, object],
|
||||||
|
mm_kwargs: Mapping[str, object],
|
||||||
|
tok_kwargs: Mapping[str, object],
|
||||||
|
) -> "BatchFeature":
|
||||||
|
"""Ask for the replacement each placeholder expands to, and record it as
|
||||||
|
per-item fields: its token ids, and the tokens or patches behind them."""
|
||||||
|
if has_mm_data := any(mm_data.values()):
|
||||||
|
mm_data = {**mm_data, "return_text_replacement_offsets": True}
|
||||||
|
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||||
|
# Drop the inputs the model would reject
|
||||||
|
hf_inputs.pop("mm_token_type_ids", None)
|
||||||
|
hf_inputs.pop("token_type_ids", None)
|
||||||
|
|
||||||
|
offsets = hf_inputs.pop("text_replacement_offsets", None)
|
||||||
|
# Some processors return an empty batch as a tensor rather than a list
|
||||||
|
if offsets is None or len(offsets) == 0 or len(offsets[0]) == 0:
|
||||||
|
if has_mm_data:
|
||||||
|
raise ValueError(
|
||||||
|
f"{type(self.info.get_hf_processor()).__name__} returned no "
|
||||||
|
"text replacement offsets, so the Transformers modeling backend "
|
||||||
|
"cannot locate the placeholder of each item. Its `__call__` has "
|
||||||
|
"to reach `ProcessorMixin.get_text_with_replacements` with one "
|
||||||
|
"replacement per item, which usually means implementing "
|
||||||
|
"`replace_<modality>_token`. Please report this to transformers "
|
||||||
|
"so it can be fixed, and install `transformers<5.15.0` in the "
|
||||||
|
"meantime to locate placeholders in the expanded prompt instead."
|
||||||
|
)
|
||||||
|
return hf_inputs
|
||||||
|
|
||||||
|
tokenizer = self.info.get_tokenizer()
|
||||||
|
replacements = defaultdict[str, list[list[int]]](list)
|
||||||
|
for entry in offsets[0]:
|
||||||
|
replacements[entry["type"]].append(
|
||||||
|
tokenizer.encode(entry["replacement"], add_special_tokens=False)
|
||||||
|
)
|
||||||
|
|
||||||
|
for modality, seqs in replacements.items():
|
||||||
|
hf_inputs[f"{modality}_replacement_ids"] = torch.tensor(
|
||||||
|
[token_id for seq in seqs for token_id in seq]
|
||||||
|
)
|
||||||
|
hf_inputs[f"{modality}_replacement_sizes"] = torch.tensor(
|
||||||
|
[len(seq) for seq in seqs]
|
||||||
|
)
|
||||||
|
if modality == "image":
|
||||||
|
hf_inputs["num_image_patches"] = self._get_num_image_patches(
|
||||||
|
hf_inputs, mm_data, len(seqs)
|
||||||
|
)
|
||||||
|
elif modality == "audio":
|
||||||
|
counts = []
|
||||||
|
for seq in seqs:
|
||||||
|
ids = torch.tensor(seq)
|
||||||
|
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
|
||||||
|
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
|
||||||
|
|
||||||
|
return hf_inputs
|
||||||
|
|
||||||
|
|
||||||
|
# From this version on, a processor reporting no offsets is an error rather than a
|
||||||
|
# fallback to searching the expanded prompt
|
||||||
|
MultiModalProcessor = (
|
||||||
|
LegacyMultiModalProcessor
|
||||||
|
if Version(transformers.__version__) < Version("5.15.0")
|
||||||
|
else OffsetsMultiModalProcessor
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||||
# Skip SupportsMRoPE.__init__ and call the next class in MRO
|
# Skip SupportsMRoPE.__init__ and call the next class in MRO
|
||||||
@@ -733,18 +1105,32 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
|||||||
kwargs.pop("token_type_ids", None)
|
kwargs.pop("token_type_ids", None)
|
||||||
kwargs.pop("mm_token_type_ids", None)
|
kwargs.pop("mm_token_type_ids", None)
|
||||||
|
|
||||||
# HuggingFace's `get_audio_features` implementations branch on
|
|
||||||
# per-sample feature lengths internally.
|
|
||||||
with gpu_sync_allowed():
|
|
||||||
audio_output = self.model.get_audio_features(
|
|
||||||
input_features, return_dict=True, **kwargs
|
|
||||||
)
|
|
||||||
audio_embeddings = audio_output.pooler_output
|
|
||||||
|
|
||||||
# Per-audio token counts are needed as Python ints to split.
|
# Per-audio token counts are needed as Python ints to split.
|
||||||
with gpu_sync_allowed():
|
with gpu_sync_allowed():
|
||||||
split_sizes = num_audio_tokens.flatten().tolist()
|
split_sizes = num_audio_tokens.flatten().tolist()
|
||||||
return self._split_embeddings(audio_embeddings, split_sizes)
|
if isinstance(input_features, torch.Tensor):
|
||||||
|
# HuggingFace's `get_audio_features` implementations branch on
|
||||||
|
# per-sample feature lengths internally.
|
||||||
|
with gpu_sync_allowed():
|
||||||
|
audio_output = self.model.get_audio_features(
|
||||||
|
input_features, return_dict=True, **kwargs
|
||||||
|
)
|
||||||
|
return self._split_embeddings(audio_output.pooler_output, split_sizes)
|
||||||
|
|
||||||
|
# Audios the processor left un-padded arrive as a list once their
|
||||||
|
# lengths differ. Encode them one at a time so that none of them is
|
||||||
|
# padded to match another.
|
||||||
|
embeddings: list[torch.Tensor] = []
|
||||||
|
for index, features in enumerate(input_features):
|
||||||
|
audio_output = self.model.get_audio_features(
|
||||||
|
features.unsqueeze(0),
|
||||||
|
return_dict=True,
|
||||||
|
**self._select_item_kwargs(kwargs, index, len(input_features)),
|
||||||
|
)
|
||||||
|
embeddings.extend(
|
||||||
|
self._split_embeddings(audio_output.pooler_output, [split_sizes[index]])
|
||||||
|
)
|
||||||
|
return embeddings
|
||||||
|
|
||||||
def _process_image_input(self, **kwargs) -> list[torch.Tensor] | None:
|
def _process_image_input(self, **kwargs) -> list[torch.Tensor] | None:
|
||||||
pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
|
pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
|
||||||
@@ -761,6 +1147,42 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
|||||||
|
|
||||||
num_image_patches = kwargs.pop("num_image_patches")
|
num_image_patches = kwargs.pop("num_image_patches")
|
||||||
|
|
||||||
|
split_sizes = num_image_patches.flatten().tolist()
|
||||||
|
if isinstance(pixel_values, torch.Tensor):
|
||||||
|
vision_embeddings = self._get_image_features(pixel_values, **kwargs)
|
||||||
|
if isinstance(vision_embeddings, torch.Tensor):
|
||||||
|
return self._split_embeddings(vision_embeddings, split_sizes)
|
||||||
|
return list(vision_embeddings)
|
||||||
|
|
||||||
|
# Images the processor left un-padded arrive as a list once their
|
||||||
|
# shapes differ. Encode them one at a time so that none of them is
|
||||||
|
# padded to match another.
|
||||||
|
embeddings: list[torch.Tensor] = []
|
||||||
|
for index, image in enumerate(pixel_values):
|
||||||
|
features = self._get_image_features(
|
||||||
|
image.unsqueeze(0),
|
||||||
|
**self._select_item_kwargs(kwargs, index, len(pixel_values)),
|
||||||
|
)
|
||||||
|
# Encoders which return one entry per image return a single entry
|
||||||
|
if not isinstance(features, torch.Tensor):
|
||||||
|
features = torch.cat(list(features))
|
||||||
|
embeddings.extend(self._split_embeddings(features, [split_sizes[index]]))
|
||||||
|
return embeddings
|
||||||
|
|
||||||
|
def _select_item_kwargs(
|
||||||
|
self, kwargs: dict[str, Any], index: int, num_items: int
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Narrow the entries of `kwargs` that hold one row per item down to the item
|
||||||
|
at `index`. Length is all there is to match on, so an unrelated entry of the
|
||||||
|
same length is narrowed too."""
|
||||||
|
return {
|
||||||
|
key: value[index : index + 1]
|
||||||
|
if isinstance(value, (torch.Tensor, list)) and len(value) == num_items
|
||||||
|
else value
|
||||||
|
for key, value in kwargs.items()
|
||||||
|
}
|
||||||
|
|
||||||
|
def _get_image_features(self, pixel_values: torch.Tensor, **kwargs) -> Any:
|
||||||
# grid_thw fields are registered keep_on_cpu; restore the on-device
|
# grid_thw fields are registered keep_on_cpu; restore the on-device
|
||||||
# placement that HF get_image_features implementations expect.
|
# placement that HF get_image_features implementations expect.
|
||||||
for key, value in kwargs.items():
|
for key, value in kwargs.items():
|
||||||
@@ -772,22 +1194,17 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
|||||||
# padding images via boolean-mask indexing, LlavaOnevision
|
# padding images via boolean-mask indexing, LlavaOnevision
|
||||||
# branches on per-sample batch counts).
|
# branches on per-sample batch counts).
|
||||||
with gpu_sync_allowed():
|
with gpu_sync_allowed():
|
||||||
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
|
features = self.model.get_image_features(pixel_values, **kwargs)
|
||||||
|
|
||||||
# Transformers `v5`, `self.get_image_features` returns a tuple
|
# Transformers `v5`, `self.get_image_features` returns a tuple
|
||||||
# containing the features and optionally attentions/hidden_states
|
# containing the features and optionally attentions/hidden_states
|
||||||
# After v5 is settled, we can enable qwen3-vl with several outputs
|
# After v5 is settled, we can enable qwen3-vl with several outputs
|
||||||
# from `self.get_image_features`
|
# from `self.get_image_features`
|
||||||
if isinstance(vision_embeddings, tuple):
|
if isinstance(features, tuple):
|
||||||
vision_embeddings = vision_embeddings[0]
|
return features[0]
|
||||||
elif isinstance(vision_embeddings, dict):
|
if isinstance(features, dict):
|
||||||
vision_embeddings = vision_embeddings.pooler_output
|
return features.pooler_output
|
||||||
|
return features
|
||||||
if isinstance(vision_embeddings, torch.Tensor):
|
|
||||||
split_sizes = num_image_patches.flatten().tolist()
|
|
||||||
return self._split_embeddings(vision_embeddings, split_sizes)
|
|
||||||
|
|
||||||
return list(vision_embeddings)
|
|
||||||
|
|
||||||
def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings:
|
def embed_multimodal(self, **kwargs) -> MultiModalEmbeddings:
|
||||||
# Each helper detects its own inputs. We are called once per modality, so the
|
# Each helper detects its own inputs. We are called once per modality, so the
|
||||||
|
|||||||
Reference in New Issue
Block a user