[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
@@ -1,154 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 The vLLM team.
# Copyright 2025 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
# reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from transformers import PreTrainedConfig
from tests.models.registry import HF_EXAMPLE_MODELS
from vllm.config.multimodal import MultiModalDummyOptions
class MockAudioFlamingo3Config(PreTrainedConfig):
model_type = "audioflamingo3"
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.audio_config = PreTrainedConfig()
self.text_config = PreTrainedConfig()
class MockAudioFlamingo3Processor:
def __init__(self):
self.audio_token = "<sound>"
self.audio_token_id = 12345
self.max_audio_len = 60
self.feature_extractor = MockFeatureExtractor()
self.tokenizer = self._tokenize
def __call__(self, text=None, audio=None, **kwargs):
return {
"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
"input_features": torch.zeros((3, 80, 3000)),
"input_features_mask": torch.ones((3, 3000), dtype=torch.long),
}
def _tokenize(self, text, **kwargs):
return {"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long)}
class MockFeatureExtractor:
def __init__(self):
self.sampling_rate = 16000
self.chunk_length = 30
self.hop_length = 160
def __call__(self, audios, **kwargs):
return {
"input_features": torch.zeros((len(audios), 80, 3000)),
"attention_mask": torch.ones((len(audios), 3000), dtype=torch.long),
}
@pytest.fixture
def mock_ctx():
config = MockAudioFlamingo3Config()
ctx = MagicMock()
ctx.get_hf_config.return_value = config
ctx.get_hf_processor.return_value = MockAudioFlamingo3Processor()
ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
**data, **kwargs
)
ctx.model_config.hf_config = config
return ctx
@pytest.fixture(autouse=True)
def check_transformers_version():
model_info = HF_EXAMPLE_MODELS.get_hf_info("AudioFlamingo3ForConditionalGeneration")
model_info.check_transformers_version(on_fail="skip")
def test_audio_chunk_counting(mock_ctx):
from vllm.model_executor.models.audioflamingo3 import (
AudioFlamingo3DummyInputsBuilder,
AudioFlamingo3MultiModalProcessor,
AudioFlamingo3ProcessingInfo,
)
info = AudioFlamingo3ProcessingInfo(mock_ctx)
processor = AudioFlamingo3MultiModalProcessor(
info, AudioFlamingo3DummyInputsBuilder(info)
)
sr = 16000
audio_1 = np.zeros(30 * sr)
audio_2 = np.zeros(75 * sr)
mm_data = {"audio": [audio_1, audio_2]}
mm_items = processor.info.parse_mm_data(mm_data, validate=False)
processed = processor._apply_hf_processor_main(mm_items, {})
chunk_counts = processed["chunk_counts"]
assert chunk_counts[0].item() == 1
assert chunk_counts[1].item() == 2
assert len(chunk_counts) == 2
assert processed["feature_attention_mask"].shape == (3, 3000)
def test_dummy_data_generation(mock_ctx):
from vllm.model_executor.models.audioflamingo3 import (
AudioFlamingo3DummyInputsBuilder,
AudioFlamingo3ProcessingInfo,
)
info = AudioFlamingo3ProcessingInfo(mock_ctx)
builder = AudioFlamingo3DummyInputsBuilder(info)
mm_counts = {"audio": 2}
dummy_data = builder.get_dummy_mm_data(100, mm_counts, MultiModalDummyOptions())
assert "audio" in dummy_data
assert len(dummy_data["audio"]) == 2
expected_len = 60 * 16000
assert len(dummy_data["audio"][0]) == expected_len
def test_audio_token_count_matches_hf_processor_math():
from vllm.model_executor.models.audioflamingo3 import (
_count_audio_tokens_from_mask,
)
feature_attention_mask = torch.zeros((3, 3000), dtype=torch.long)
feature_attention_mask[0, :2999] = 1
feature_attention_mask[1, :2999] = 1
feature_attention_mask[2, :1500] = 1
chunk_counts = torch.tensor([2, 1], dtype=torch.long)
assert (
_count_audio_tokens_from_mask(feature_attention_mask, chunk_counts, 0) == 1499
)
assert _count_audio_tokens_from_mask(feature_attention_mask, chunk_counts, 1) == 375
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Bailing prompt expansion tests without downloading a checkpoint."""
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from tokenizers import Tokenizer, decoders, models, pre_tokenizers
from transformers import TokenizersBackend
from vllm.model_executor.models.bailing_moe_v3_vl import (
IMAGE_PLACEHOLDER,
IMAGE_TOKEN,
BailingMoeV3VLMultiModalProcessor,
)
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
pytestmark = pytest.mark.cpu_test
@pytest.fixture
def processor():
vocab = {
token: index
for index, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
}
# A newline added by the processor must merge with an existing newline.
vocab["ĊĊ"] = len(vocab)
backend = Tokenizer(models.BPE(vocab=vocab, merges=[("Ċ", "Ċ")]))
backend.pre_tokenizer = pre_tokenizers.ByteLevel(
add_prefix_space=False, use_regex=False
)
backend.decoder = decoders.ByteLevel()
tokenizer = TokenizersBackend(
tokenizer_object=backend,
additional_special_tokens=[
"<|vision_start|>",
IMAGE_TOKEN,
"<|vision_end|>",
],
clean_up_tokenization_spaces=False,
)
processor = object.__new__(BailingMoeV3VLMultiModalProcessor)
processor.info = SimpleNamespace(
get_tokenizer=lambda: tokenizer,
get_hf_config=lambda: SimpleNamespace(
vision_config=SimpleNamespace(spatial_merge_size=2)
),
)
return processor
@pytest.mark.parametrize(
("prompt", "expected_text", "token_counts"),
[
(
f"before{IMAGE_PLACEHOLDER}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
[2],
),
(
f"before{IMAGE_TOKEN}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
[2],
),
(
f"before{IMAGE_PLACEHOLDER}\nafter",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n\nafter",
[2],
),
(
f"before{IMAGE_PLACEHOLDER}{IMAGE_PLACEHOLDER}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n"
f"<|vision_start|>{IMAGE_TOKEN * 3}<|vision_end|>\nafter",
[2, 3],
),
],
ids=["wrapped", "bare", "existing_newline", "adjacent_images"],
)
def test_image_expansion_matches_wrappers_and_newline_tokenization(
processor, prompt, expected_text, token_counts
):
tokenizer = processor.info.get_tokenizer()
image_id = tokenizer.get_vocab()[IMAGE_TOKEN]
grids = torch.tensor([[1, 2, 2 * count] for count in token_counts])
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
{"image_grid_thw": grids},
{"image_grid_thw": MultiModalFieldConfig.batched("image")},
)
mm_items = MultiModalDataItems(
{"image": ImageProcessorItems([Image.new("RGB", (1, 1))] * len(token_counts))}
)
updates = processor._get_mm_prompt_updates(mm_items, {}, mm_kwargs)
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
expected_ids = tokenizer.encode(expected_text, add_special_tokens=False)
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, updates)
assert actual_ids == expected_ids
assert len(placeholders["image"]) == len(token_counts)
found = processor._find_mm_placeholders(actual_ids, updates)
search_offset = 0
for index, count in enumerate(token_counts):
placeholder = placeholders["image"][index]
expected_offset = expected_ids.index(image_id, search_offset)
assert placeholder.start_idx == expected_offset
assert placeholder.tokens == [image_id] * count
assert placeholder.to_range().get_num_embeds() == count
assert placeholder.to_range() == found["image"][index].to_range()
search_offset = expected_offset + count
def test_text_only_prompt_preserves_original_token_ids(processor):
tokenizer = processor.info.get_tokenizer()
prompt_ids = tokenizer.encode("before\n", add_special_tokens=False)
prompt_ids += tokenizer.encode("\nafter", add_special_tokens=False)
# Deliberately use two newline tokens instead of their merged BPE token.
assert prompt_ids != tokenizer.encode("before\n\nafter", add_special_tokens=False)
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, {})
assert actual_ids == prompt_ids
assert placeholders == {}
@@ -1,244 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for DeepSeek-OCR TensorSchema validation with empty images_crop.
When using the Gundam preset (BASE_SIZE=1024, IMAGE_SIZE=640, CROP_MODE=True),
images that are small enough to not require cropping produce an empty
images_crop tensor with shape (0, 3, 640, 640). The _parse_and_validate_image_input
method must correctly read image_size from this tensor's shape rather than
falling back to base_size, which would cause a TensorSchema mismatch.
Run with:
pytest tests/models/multimodal/processing/test_deepseek_ocr.py -v
"""
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from transformers import AutoTokenizer
from vllm.model_executor.models.deepseek_ocr import (
DeepseekOCRForCausalLM,
DeepseekOCRImagePixelInputs,
)
from vllm.model_executor.models.deepseek_ocr2 import DeepseekOCR2ForCausalLM
from vllm.transformers_utils.processors.deepseek_ocr import (
DeepseekOCRProcessor,
ImageTransform,
)
MODEL_ID = "deepseek-ai/DeepSeek-OCR"
@pytest.fixture(scope="module")
def processor():
"""Load the DeepseekOCRProcessor with tokenizer from HuggingFace."""
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
return DeepseekOCRProcessor(tokenizer=tokenizer)
class TestDeepseekOCREmptyImagesCrop:
"""Verify TensorSchema validation handles empty images_crop correctly."""
def test_empty_images_crop_small_image(self, processor):
"""A small image (<=640px) produces empty images_crop and should
not crash the TensorSchema validation.
Previously, the code used ``numel() > 0`` to decide whether to read
image_size from the tensor shape. When numel()==0, it fell back to
base_size=1024, mismatching the actual tensor dim of 640.
"""
# Small image: both dims <= IMAGE_SIZE (640) → no crops
small_image = Image.new("RGB", (100, 100), color="red")
result = processor(
prompt="<image>\nDescribe this image.",
images=[small_image],
)
pixel_values = result["pixel_values"]
images_crop = result["images_crop"]
images_spatial_crop = result["images_spatial_crop"]
# Processor must produce an empty crop tensor for a small image
assert images_crop.shape[0] == 0
base_size = pixel_values.shape[-1]
image_size = images_crop.shape[-1] if images_crop is not None else base_size
# This should NOT raise ValueError
schema = DeepseekOCRImagePixelInputs(
type="pixel_values",
data=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
resolve_bindings={
"base_size": base_size,
"image_size": image_size,
},
)
assert schema.data.shape == (1, 3, 1024, 1024)
assert schema.images_crop.shape == (0, 3, 640, 640)
def test_populated_images_crop_large_image(self, processor):
"""A large image (>640px) produces populated images_crop."""
# Large image: exceeds IMAGE_SIZE (640) → dynamic crop tiles
large_image = Image.new("RGB", (1200, 800), color="blue")
result = processor(
prompt="<image>\nDescribe this image.",
images=[large_image],
)
pixel_values = result["pixel_values"]
images_crop = result["images_crop"]
images_spatial_crop = result["images_spatial_crop"]
assert images_crop.shape[0] > 0
base_size = pixel_values.shape[-1]
image_size = images_crop.shape[-1]
schema = DeepseekOCRImagePixelInputs(
type="pixel_values",
data=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
resolve_bindings={
"base_size": base_size,
"image_size": image_size,
},
)
assert schema.data.shape == (1, 3, 1024, 1024)
assert schema.images_crop.shape[-1] == 640
def test_mismatched_image_size_raises(self, processor):
"""Deliberately wrong image_size binding should still be caught
by TensorSchema validation."""
small_image = Image.new("RGB", (100, 100), color="green")
result = processor(
prompt="<image>\nDescribe this image.",
images=[small_image],
)
pixel_values = result["pixel_values"]
images_crop = result["images_crop"]
images_spatial_crop = result["images_spatial_crop"]
with pytest.raises(ValueError, match="images_crop"):
DeepseekOCRImagePixelInputs(
type="pixel_values",
data=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
resolve_bindings={
"base_size": 1024,
"image_size": 1024, # Wrong! Tensor has 640
},
)
class TestDeepseekOCRInputValidation:
"""Bare ``assert`` used to validate user input here returned HTTP 500
(AssertionError) instead of HTTP 400 (ValueError) and disappeared under
``python -O``, silently corrupting the tokenized sequence. See #53850.
"""
def test_mismatched_image_count_raises_value_error(self, processor):
"""Two ``<image>`` tokens with one image must raise ValueError, not
AssertionError (which would yield HTTP 500 and, under ``-O``, silently
drop the middle text segment)."""
image = Image.new("RGB", (100, 100), color="red")
with pytest.raises(ValueError, match="does not match number of images"):
processor(prompt="<image> and <image>", images=[image])
def test_missing_prompt_raises_value_error(self, processor):
image = Image.new("RGB", (100, 100), color="red")
with pytest.raises(ValueError, match="prompt and images"):
processor(prompt=None, images=[image])
def test_missing_images_raises_value_error(self, processor):
with pytest.raises(ValueError, match="prompt and images"):
processor(prompt="<image>\nDescribe this image.", images=None)
def _balanced_normalized_pixels(
batch: int = 1, channels: int = 3, height: int = 64, width: int = 64
) -> torch.Tensor:
"""Build a pixel tensor whose values cancel to an exact sum of 0.
Matches the normalized half-black / half-white case from the DeepSeek-OCR
processor (Normalize(0.5, 0.5) maps 0→-1 and 255→+1).
"""
assert width % 2 == 0
pixel_values = torch.ones(batch, channels, height, width)
pixel_values[..., : width // 2] = -1.0
assert pixel_values.sum().item() == 0.0
return pixel_values
class TestDeepseekOCRZeroSumPixelsAccepted:
"""Zero-sum normalized pixels must still produce validated image inputs.
Returning None from ``_parse_and_validate_image_input`` makes
``embed_multimodal`` return None and trips the worker encoder-output
sanity check, killing EngineCore. The zero-sum shortcut is therefore
removed; balanced images must parse like any other valid tensor.
"""
def test_ocr_accepts_balanced_normalized_pixels(self):
pixel_values = _balanced_normalized_pixels()
images_crop = torch.zeros(0, 3, 40, 40)
images_spatial_crop = torch.tensor([[1, 1]])
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
None,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert image_input.data is pixel_values
def test_ocr2_accepts_balanced_normalized_pixels(self):
pixel_values = _balanced_normalized_pixels()
images_crop = torch.zeros(0, 3, 40, 40)
images_spatial_crop = torch.tensor([[1, 1]])
model = SimpleNamespace(vision_config=SimpleNamespace(image_size=64))
image_input = DeepseekOCR2ForCausalLM._parse_and_validate_image_input(
model,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert image_input.data is pixel_values
def test_image_transform_half_black_half_white_sums_to_zero(self):
"""A 1024×1024 half-black/half-white PNG through ImageTransform
(ToTensor + Normalize(0.5, 0.5)) yields an exact zero sum, and parse
must still accept it.
"""
image = Image.new("RGB", (1024, 1024), (0, 0, 0))
image.paste(Image.new("RGB", (512, 1024), (255, 255, 255)), (512, 0))
pixel_values = ImageTransform()(image).unsqueeze(0)
assert pixel_values.sum().item() == 0.0
images_crop = torch.zeros(0, 3, 640, 640)
images_spatial_crop = torch.tensor([[1, 1]])
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
None,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert torch.equal(image_input.data, pixel_values)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4 VL processing helpers."""
from PIL import Image
from transformers import DeepseekV4Config
from vllm.models.deepseek_v4.common.mm_preprocess import (
DeepseekV4VLProcessingInfo,
DeepseekV4VLProcessor,
build_image_block_pad_free,
)
# Vision fields of deepseek-ai/DeepSeek-V4-Flash-Vision-Exp
VISION_CONFIG = dict(
vision_n_layers=24,
vision_dim=1024,
vision_n_heads=16,
vision_inter_dim=2816,
vision_patch_size=14,
vision_rope_theta=10000.0,
vision_downsample_ratio=3,
vision_max_n_token=384,
vision_min_pixels=147456,
vision_max_wh_ratio=8,
)
class _ConfigCtx:
def __init__(self, config: DeepseekV4Config) -> None:
self._config = config
def get_hf_config(self, *args, **kwargs):
return self._config
def _feature_counts(
processor: DeepseekV4VLProcessor, width: int, height: int
) -> tuple[int, int]:
out = processor(images=[Image.new("RGB", (width, height))])
n_llm_h, n_llm_w = out["llm_grid"][0].tolist()
types, _ = build_image_block_pad_free(n_llm_h, n_llm_w)
return len(types), out["patches"].shape[0]
def test_get_image_size_with_most_features_beats_wide_real_image():
"""Dummy size must maximize tokens/patches within the budget (#59271).
A square under-counts because each aligner row pays an IMAGE_NEW_LINE.
"""
config = DeepseekV4Config(**VISION_CONFIG)
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
processor = DeepseekV4VLProcessor(config)
dummy = info.get_image_size_with_most_features()
dummy_features = _feature_counts(processor, dummy.width, dummy.height)
wide_features = _feature_counts(processor, 1932, 336)
assert dummy == (1932, 336)
assert wide_features <= dummy_features
def test_get_image_size_with_most_features_respects_max_wh_ratio():
config = DeepseekV4Config(**VISION_CONFIG)
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
dummy = info.get_image_size_with_most_features()
assert dummy.width / dummy.height <= config.vision_max_wh_ratio
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.models.gemma3n_audio_utils import (
adjust_audio_features_to_expected_length,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
# Gemma3 (image) model
GEMMA3_MODEL_ID = "google/gemma-3-4b-it"
# Gemma3n (multimodal with audio) model
GEMMA3N_MODEL_ID = "google/gemma-3n-E2B-it"
# Expected audio tokens for Gemma3n (audio_soft_tokens_per_image)
GEMMA3N_EXPECTED_AUDIO_TOKENS = 188
class TestGemma3nAudioTensorLogic:
"""CPU-based tests for Gemma3n audio feature tensor manipulation.
These tests validate the padding/truncation logic in
adjust_audio_features_to_expected_length() which fixes the
integer overflow in _process_audio_input when audio_seq_len > 188.
"""
def test_padding_when_audio_short(self):
"""Test that short audio is padded to expected length."""
batch_size, seq_len, embed_dim = 1, 100, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == (batch_size, expected_tokens, embed_dim)
assert tokens_truncated == 0
# First 100 tokens should be original, rest should be padding (zeros)
assert torch.allclose(result[:, :seq_len, :], audio_features)
assert torch.allclose(
result[:, seq_len:, :],
torch.zeros(batch_size, expected_tokens - seq_len, embed_dim),
)
def test_truncation_when_audio_long(self):
"""Test that long audio is truncated to expected length.
This is the key test for the overflow fix. Previously, when
audio_seq_len > expected_tokens, the code would compute a negative
padding value causing: RuntimeError: numel: integer multiplication overflow
"""
batch_size, seq_len, embed_dim = 1, 192, 256 # 192 > 188
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == (batch_size, expected_tokens, embed_dim)
assert tokens_truncated == seq_len - expected_tokens # 192 - 188 = 4
# Result should be first 188 tokens of original
assert torch.allclose(result, audio_features[:, :expected_tokens, :])
def test_no_change_when_exact_length(self):
"""Test that exact-length audio passes through unchanged."""
batch_size, embed_dim = 1, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, expected_tokens, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == audio_features.shape
assert tokens_truncated == 0
assert torch.allclose(result, audio_features)
def test_original_bug_would_fail(self):
"""Verify the original buggy implementation would cause overflow.
The original code always tried to pad, which fails when
audio_seq_len > expected_tokens because expand() gets negative size.
"""
batch_size, seq_len, embed_dim = 1, 192, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
padding_embs = torch.zeros(1, 1, embed_dim)
# Original buggy logic (always pads, never truncates)
extra_padding_tokens = expected_tokens - seq_len # = -4 (negative!)
with pytest.raises(RuntimeError):
# This should fail with negative size error
padding_embs.expand(batch_size, extra_padding_tokens, embed_dim)
@pytest.mark.parametrize(
"seq_len",
[50, 100, 150, 187, 188, 189, 192, 200, 300],
)
def test_various_audio_lengths(self, seq_len: int):
"""Test padding/truncation with various audio lengths."""
batch_size, embed_dim = 1, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
# Should not raise any errors
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
# Output should always be expected_tokens length
assert result.shape == (batch_size, expected_tokens, embed_dim)
# Verify truncation count is correct
if seq_len > expected_tokens:
assert tokens_truncated == seq_len - expected_tokens
else:
assert tokens_truncated == 0
def test_batch_processing(self):
"""Test that batch processing works correctly."""
batch_size, seq_len, embed_dim = 4, 192, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == (batch_size, expected_tokens, embed_dim)
assert tokens_truncated == seq_len - expected_tokens
@pytest.mark.parametrize("model_id", [GEMMA3_MODEL_ID])
@pytest.mark.parametrize("mm_processor_kwargs", [{}])
def test_get_image_size_with_most_features(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
):
ctx = build_model_context(
model_id,
mm_processor_kwargs={"do_pan_and_scan": True},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
max_image_size = processor.info.get_image_size_with_most_features()
max_tokens = processor.info.get_num_image_tokens(
image_width=max_image_size.width,
image_height=max_image_size.height,
processor=hf_processor,
mm_kwargs=mm_processor_kwargs,
)
prompt = "<start_of_image>"
image_seq_length = hf_processor.image_seq_length
for asset in image_assets:
mm_data = {"image": [asset.pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
mm_kwargs_data = processed_inputs["mm_kwargs"].get_data()
num_patches_tensor = mm_kwargs_data["num_patches"]
tokens = int(num_patches_tensor.item()) * image_seq_length
assert tokens <= max_tokens
@@ -1,405 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from PIL import Image as PILImage
from vllm.exceptions import VLLMValidationError
from vllm.model_executor.models.gemma4_mm import (
Gemma4ForConditionalGeneration,
Gemma4ImagePixelInputs,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import MultiModalFieldConfig
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.worker.encoder_cudagraph_defs import ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG
from ....conftest import ImageTestAssets
from ...utils import build_model_context
# TODO: to be updated to "google/gemma-4-e2b-it" once the models are available
GEMMA4_MODEL_ID = "google/gemma-4-E2B-it"
def test_gemma4_image_schema_accepts_variable_patch_counts():
Gemma4ImagePixelInputs(
pixel_values=[
torch.randn(10080, 768),
torch.randn(2520, 768),
],
pixel_position_ids=[
torch.zeros(10080, 2, dtype=torch.long),
torch.zeros(2520, 2, dtype=torch.long),
],
)
def test_gemma4_image_batching_keeps_variable_patch_counts_unstacked():
field = MultiModalFieldConfig.batched("image").field
elems = field.build_elems(
"image",
"pixel_values",
[torch.randn(10080, 768), torch.randn(2520, 768)],
)
reduced = field.reduce_data(list(elems))
assert isinstance(reduced, list)
assert [tensor.shape for tensor in reduced] == [
torch.Size([10080, 768]),
torch.Size([2520, 768]),
]
@pytest.mark.parametrize(
"image_width,image_height,max_soft_tokens",
[
# Production repro: a 3x900 image (extreme aspect ratio) made the
# prompt-side estimator return 289 while the HF Gemma 4 image
# processor's vision tower output capped at 280, producing the
# "Attempted to assign 280 multimodal tokens to 289 placeholders"
# mismatch that crashed EngineCore.
(900, 3, 280),
(3, 900, 280),
# Same pathology should hold for the video-frame budget (70 tokens).
(900, 3, 70),
# And for any other supported budget.
(4000, 2, 1120),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
model_id: str,
image_width: int,
image_height: int,
max_soft_tokens: int,
):
"""Regression for the Gemma 3/4 multimodal crash.
`_compute_num_soft_tokens` must never return a value larger than
`max_soft_tokens`. The HF Gemma 4 image processor clamps its vision
tower output to that value; if the prompt-side estimator returns more,
the prompt has more `image` placeholder tokens than the encoder will
fill, and `_merge_multimodal_embeddings` raises `ValueError` deep in
the model forward.
"""
ctx = build_model_context(
model_id,
mm_processor_kwargs={"do_pan_and_scan": True},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
num_soft_tokens = processor.info._compute_num_soft_tokens(
image_width=image_width,
image_height=image_height,
max_soft_tokens=max_soft_tokens,
)
assert num_soft_tokens <= max_soft_tokens, (
f"_compute_num_soft_tokens returned {num_soft_tokens} for "
f"image_width={image_width}, image_height={image_height}, "
f"max_soft_tokens={max_soft_tokens} — exceeds the cap that the HF "
f"image processor enforces on its vision tower output. This is "
f"the placeholder/encoder count mismatch that crashes EngineCore."
)
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_image_tokens"),
[
({}, 280),
({"max_soft_tokens": 70}, 70),
({"max_soft_tokens": 280}, 280),
({"max_soft_tokens": 1120}, 1120),
({"images_kwargs": {"max_soft_tokens": 560}}, 560),
({"images_kwargs": None}, 280),
({"images_kwargs": "not-a-dict"}, 280),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_image_tokens: int,
):
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"image": 1, "video": 1},
)
assert tokens is not None
assert tokens["image"] == expected_image_tokens
assert tokens["video"] == 32 * (70 + 2 + 6)
@pytest.mark.parametrize(
("limit_mm_per_prompt", "expected_video_tokens"),
[
({"video": 1}, 32 * (70 + 2 + 6)),
({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
model_id: str,
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
expected_video_tokens: int,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt=limit_mm_per_prompt,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"video": 1},
)
assert tokens is not None
assert tokens["image"] == 280
assert tokens["video"] == expected_video_tokens
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
ctx = build_model_context(
model_id,
mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = PILImage.new("RGB", (1000, 1000), color="white")
image_size = image.size
mm_items = processor.info.parse_mm_data({"image": image})
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
replacement = prompt_update.resolve(0).content.full
expected = processor.info.get_image_repl(
image_width=image_size[0],
image_height=image_size[1],
processor=processor.info.get_hf_processor(),
max_soft_tokens=560,
).full
assert replacement == expected
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
@pytest.mark.parametrize(
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
)
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
def test_video_cache_is_independent_of_image_kwargs(
model_id: str,
image_kwargs: dict[str, object],
video_uuid: str | None,
):
"""Request image overrides must not change video frames or make results
depend on cache warmth.
"""
request_kwargs = {"images_kwargs": image_kwargs}
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 1},
mm_processor_cache_gb=1,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor()
image = PILImage.new("RGB", (48, 48), color=(128, 128, 128))
frames = np.stack([np.asarray(image)] * 2)
metadata = {"fps": 2.0, "frames_indices": [0, 1]}
mm_items = processor.info.parse_mm_data(
{"image": image, "video": [(frames, metadata)]}
)
def process(mm_kwargs, cache):
return processor(
hf_processor.image_token + hf_processor.video_token,
mm_items,
mm_uuid_items={"video": [video_uuid]},
hf_processor_mm_kwargs=mm_kwargs,
cache=cache,
)
# Populate the video cache without image overrides, then change only
# images_kwargs on the next request. Since images_kwargs are excluded from
# the video hash, and must not affect the processed video, the cached result
# must match a fresh recomputation.
baseline = process({}, cache=cache)
cached = process(request_kwargs, cache=cache)
cache.clear_cache()
fresh = process(request_kwargs, cache=cache)
def pixels(result, modality, field):
return result["mm_kwargs"][modality][0][field].data
video_pixels = pixels(fresh, "video", "pixel_values_videos")
assert torch.equal(pixels(cached, "video", "pixel_values_videos"), video_pixels)
assert torch.equal(pixels(baseline, "video", "pixel_values_videos"), video_pixels)
assert baseline["mm_hashes"]["video"] == fresh["mm_hashes"]["video"]
assert not torch.equal(
pixels(baseline, "image", "pixel_values"),
pixels(fresh, "image", "pixel_values"),
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_limit_mm_per_prompt(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that limit_mm_per_prompt accurately restricts multiple images."""
# We only allow 1 image
ctx = build_model_context(
model_id,
mm_processor_kwargs={},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Provide 2 images in the prompt
prompt = "<image><image>"
# image_assets usually has multiple images
images = [asset.pil_image for asset in image_assets][:2]
if len(images) < 2:
images = [images[0], images[0]]
mm_data = {"image": images}
# Expect ValueError when exceeding limit
with pytest.raises(VLLMValidationError, match="At most 1 image"):
processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
# Regression guard for PR #43169 follow-up: the batched Gemma4 vision encoder
# admitted ``chunk ~= 53`` on a 22 GiB L4 with a 26B AWQ model loaded,
# allocating 2.43 GiB int64 inside
# ``F.one_hot(num_classes=position_embedding_size)`` and OOMing because only
# 2.41 GiB was actually free. The fix sizes ``chunk`` from currently-free GPU
# memory and counts the ``F.one_hot`` transient as the dominant cost.
_encoder_chunk = Gemma4ForConditionalGeneration._encoder_chunk
# Gemma4 vision config default (HF: configuration_gemma4.py).
_POSITION_EMBEDDING_SIZE = 10240
# Video frame: max_soft_tokens=70, pooling_kernel_size=2 -> 70 * 4 patches.
_VIDEO_PATCHES_PER_FRAME = 280
def test_encoder_chunk_tight_budget_fits_in_free():
free = 3 * GiB_bytes # L4 22 GiB after 26B AWQ load.
total = 22 * GiB_bytes
chunk = _encoder_chunk(
_VIDEO_PATCHES_PER_FRAME, free, total, _POSITION_EMBEDDING_SIZE
)
one_hot_bytes = chunk * _VIDEO_PATCHES_PER_FRAME * 2 * _POSITION_EMBEDDING_SIZE * 8
assert one_hot_bytes <= free // 2
def test_encoder_chunk_roomy_gpu_keeps_batching():
chunk = _encoder_chunk(
_VIDEO_PATCHES_PER_FRAME,
60 * GiB_bytes,
80 * GiB_bytes,
_POSITION_EMBEDDING_SIZE,
)
assert chunk > 8
def test_encoder_chunk_zero_patches_is_safe():
assert (
_encoder_chunk(0, 60 * GiB_bytes, 80 * GiB_bytes, _POSITION_EMBEDDING_SIZE) == 1
)
def test_encoder_chunk_zero_position_embedding_size_is_safe():
# Degenerate config: must not raise ZeroDivisionError.
assert (
_encoder_chunk(_VIDEO_PATCHES_PER_FRAME, 60 * GiB_bytes, 80 * GiB_bytes, 0) == 1
)
def test_encoder_chunk_no_free_memory_falls_back_to_one():
assert (
_encoder_chunk(
_VIDEO_PATCHES_PER_FRAME, 0, 22 * GiB_bytes, _POSITION_EMBEDDING_SIZE
)
== 1
)
def _gemma4_for_encoder_cudagraph() -> Gemma4ForConditionalGeneration:
model = object.__new__(Gemma4ForConditionalGeneration)
vision_config = SimpleNamespace(
patch_size=16, pooling_kernel_size=3, default_output_length=280
)
model.vision_tower = SimpleNamespace(config=vision_config)
model.config = SimpleNamespace(vision_config=vision_config)
return model
@pytest.mark.parametrize(
("token_budget", "row_tokens"), [(4096, 70), (4096, 280), (70, 280)]
)
def test_encoder_cudagraph_capture_rows_fit_token_budget(
token_budget: int, row_tokens: int
):
# Regression for #61045: rows were max_frames_per_batch (58 * 32) at
# 280 tokens each, so the 4096-token buffer alone was 6.69 GiB.
model = _gemma4_for_encoder_cudagraph()
inputs = model.prepare_encoder_cudagraph_capture_inputs(
token_budget=token_budget,
max_batch_size=58,
max_frames_per_batch=58 * 32,
device="meta",
dtype=torch.bfloat16,
axis_keys=(row_tokens,),
)
num_rows, row_patches, _ = inputs.values["pixel_values"].shape
assert row_patches == row_tokens * 9
assert num_rows == max(1, token_budget // row_tokens)
def test_encoder_cudagraph_mixed_width_images_pack_by_padded_width():
model = _gemma4_for_encoder_cudagraph()
mm_kwargs = {
"pixel_values": [torch.zeros(2520, 768), torch.zeros(630, 768)],
"pixel_position_ids": [
torch.zeros(2520, 2, dtype=torch.long),
torch.zeros(630, 2, dtype=torch.long),
],
}
specs = model.get_encoder_cudagraph_item_specs(mm_kwargs)
assert [spec.output_tokens for spec in specs] == [280, 70]
assert [spec.get_path_output_tokens("default") for spec in specs] == [280, 280]
selected = model.select_encoder_cudagraph_items(mm_kwargs, [1])
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (70,)
selected = model.select_encoder_cudagraph_items(mm_kwargs, [0, 1])
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (280,)
@@ -1,248 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
import pytest
import torch
from PIL import Image as PILImage
from transformers.models.gemma4 import Gemma4AudioFeatureExtractor
from vllm.exceptions import VLLMValidationError
from vllm.model_executor.models.gemma4_mm import (
Gemma4ImagePixelInputs,
_dummy_audio_num_samples,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig
from ....conftest import ImageTestAssets
from ...utils import build_model_context
# The Unified model ID for testing purposes
GEMMA4_UNIFIED_MODEL_ID = "google/gemma-4-12B-it"
def test_gemma4_unified_image_schema_accepts_variable_patch_counts():
Gemma4ImagePixelInputs(
pixel_values=[
torch.randn(10080, 768),
torch.randn(2520, 768),
],
pixel_position_ids=[
torch.zeros(10080, 2, dtype=torch.long),
torch.zeros(2520, 2, dtype=torch.long),
],
)
def test_gemma4_unified_image_batching_keeps_variable_patch_counts_unstacked():
field = MultiModalFieldConfig.batched("image").field
elems = field.build_elems(
"image",
"pixel_values",
[torch.randn(10080, 768), torch.randn(2520, 768)],
)
reduced = field.reduce_data(list(elems))
assert isinstance(reduced, list)
assert [tensor.shape for tensor in reduced] == [
torch.Size([10080, 768]),
torch.Size([2520, 768]),
]
@pytest.mark.parametrize(
"image_width,image_height,max_soft_tokens",
[
(900, 3, 280),
(3, 900, 280),
(900, 3, 70),
(4000, 2, 1120),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
model_id: str,
image_width: int,
image_height: int,
max_soft_tokens: int,
):
"""Verify ``_compute_num_soft_tokens`` caps output at ``max_soft_tokens``."""
ctx = build_model_context(
model_id,
mm_processor_kwargs={"do_pan_and_scan": True},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
num_soft_tokens = processor.info._compute_num_soft_tokens(
image_width=image_width,
image_height=image_height,
max_soft_tokens=max_soft_tokens,
)
assert num_soft_tokens <= max_soft_tokens, (
f"_compute_num_soft_tokens returned {num_soft_tokens} for "
f"image_width={image_width}, image_height={image_height}, "
f"max_soft_tokens={max_soft_tokens} — exceeds the cap."
)
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_image_tokens"),
[
({}, 280),
({"max_soft_tokens": 70}, 70),
({"max_soft_tokens": 280}, 280),
({"max_soft_tokens": 1120}, 1120),
({"images_kwargs": {"max_soft_tokens": 560}}, 560),
({"images_kwargs": None}, 280),
({"images_kwargs": "not-a-dict"}, 280),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_image_tokens: int,
):
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"image": 1, "video": 1},
)
assert tokens is not None
assert tokens["image"] == expected_image_tokens
assert tokens["video"] == 32 * (70 + 2 + 6)
@pytest.mark.parametrize(
("limit_mm_per_prompt", "expected_video_tokens"),
[
({"video": 1}, 32 * (70 + 2 + 6)),
({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
model_id: str,
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
expected_video_tokens: int,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt=limit_mm_per_prompt,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"video": 1},
)
assert tokens is not None
assert tokens["image"] == 280
assert tokens["video"] == expected_video_tokens
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
ctx = build_model_context(
model_id,
mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = PILImage.new("RGB", (1000, 1000), color="white")
image_size = image.size
mm_items = processor.info.parse_mm_data({"image": image})
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
replacement = prompt_update.resolve(0).content.full
expected = processor.info.get_image_repl(
image_width=image_size[0],
image_height=image_size[1],
processor=processor.info.get_hf_processor(),
max_soft_tokens=560,
).full
assert replacement == expected
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_limit_mm_per_prompt(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that limit_mm_per_prompt restricts multiple images correctly."""
ctx = build_model_context(
model_id,
mm_processor_kwargs={},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<image><image>"
images = [asset.pil_image for asset in image_assets][:2]
if len(images) < 2:
images = [images[0], images[0]]
mm_data = {"image": images}
with pytest.raises(VLLMValidationError, match="At most 1 image"):
processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
def test_dummy_audio_length_falls_back_for_the_unified_extractor():
"""The unified audio feature extractor does not define ``fft_length``.
``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute
unconditionally, so memory profiling raised ``AttributeError`` for every
unified Gemma 4 deployment in which audio is the modality with the largest
per-item token count -- which is the case as soon as ``audio_seq_length``
is raised above the video budget.
"""
class _UnifiedFeatureExtractor:
sampling_rate = 16000
class _UnifiedProcessor:
feature_extractor = _UnifiedFeatureExtractor()
audio_seq_length = 3000
audio_ms_per_token = 40
# 3000 tokens * 40 ms = 120 s at 16 kHz
assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000
def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor():
"""The mel/tower extractor keeps its existing behaviour."""
class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor):
def __init__(self): # noqa: D107 - bypass the real feature-extractor init
self.fft_length = 512
self.sampling_rate = 16000
class _TowerProcessor:
feature_extractor = _TowerFeatureExtractor()
audio_seq_length = 750
audio_ms_per_token = 40
assert _dummy_audio_num_samples(_TowerProcessor()) == 512
@@ -1,201 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import Mock
import pytest
from vllm.assets.video import VideoAsset
from vllm.model_executor.models.glm4_1v import (
Glm4vForConditionalGeneration,
Glm4vProcessingInfo,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.multimodal.video import DynamicVideoBackend, VideoBackend
from ...utils import build_model_context
@pytest.mark.parametrize(
(
"max_video_pixels",
"max_tokens",
"expected_num_frames",
),
[
(47_040_000, 124_988, 11),
(47_040_000, 30_000, 24),
(100_352_000, 124_988, 21),
(100_352_000, 30_000, 7),
(100_352_000, 0, 1),
],
)
def test_get_max_video_frames_matches_glm_resize(
max_video_pixels: int,
max_tokens: int,
expected_num_frames: int,
):
info = Mock(spec=Glm4vProcessingInfo)
info.get_image_size_with_most_features.return_value = (2184, 2184)
info._get_video_max_pixels.return_value = max_video_pixels
vision_config = info.get_hf_config.return_value.vision_config
vision_config.patch_size = 14
vision_config.spatial_merge_size = 2
vision_config.temporal_patch_size = 2
info._get_vision_info.side_effect = lambda **kwargs: (
Glm4vProcessingInfo._get_vision_info(info, **kwargs)
)
num_frames = Glm4vProcessingInfo._get_max_video_frames(
info,
max_tokens=max_tokens,
)
assert num_frames == expected_num_frames
assert info._get_video_max_pixels.call_count == 1
assert info._get_vision_info.call_count == 600
def test_encoder_cudagraph_uses_model_video_frame_limit():
model = Mock()
assert Glm4vForConditionalGeneration.get_max_frames_per_video(model) == 600
@pytest.mark.parametrize(
("temporal_patch_size", "expected_grid_t"),
[(2, 9), (4, 5), (8, 3)],
)
def test_vision_info_rounds_up_temporal_frames(
temporal_patch_size: int,
expected_grid_t: int,
):
info = Mock(spec=Glm4vProcessingInfo)
vision_config = info.get_hf_config.return_value.vision_config
vision_config.patch_size = 14
vision_config.spatial_merge_size = 2
vision_config.temporal_patch_size = temporal_patch_size
_, num_vision_tokens = Glm4vProcessingInfo._get_vision_info(
info,
image_width=28,
image_height=28,
num_frames=17,
do_resize=False,
)
assert num_vision_tokens == expected_grid_t
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
@pytest.mark.parametrize("expected_toks_per_frame", [299])
@pytest.mark.parametrize(
"num_frames, fps, expected_grid_t",
[
# pre-sampled fixed frames (unexpected behavior,
# but we still expect it to work without errors)
(32, 1, 16),
(32, 2, 16),
(128, 1, 64),
(128, 2, 64),
# post-sampled frames (expected behavior)
(-1, 1, 5),
(-1, 2, 10),
],
)
def test_processor_override(
model_id: str,
expected_toks_per_frame: int,
expected_grid_t: int,
fps: int,
num_frames: int,
):
"""Ensure GLM4vMultiModalProcessor can handle video frames properly."""
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {"fps": fps}
# Build the image str / prompt based on the number of images we pass
video_assets = VideoAsset(name="baby_reading", num_frames=num_frames)
prompt = "<|begin_of_video|><|video|><|end_of_video|>"
video, metadata = video_assets.np_ndarrays, video_assets.metadata
metadata["fps"] = fps
mm_data = {"video": [(video, metadata)]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token_id = hf_processor.image_token_id
video_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
grid_t, _, _ = processed_inputs["mm_kwargs"].get_data()["video_grid_thw"][0]
assert grid_t == expected_grid_t
assert video_tok_count == expected_toks_per_frame * grid_t
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
@pytest.mark.parametrize("fps", [2])
@pytest.mark.parametrize("backend", ["opencv"])
def test_video_loader_consistency(
model_id: str,
fps: int,
backend: str,
):
"""Ensure dynamic video loader (pre-sampled by loader) and normal video
loader (post-sampled by processor) produce same video processing outputs.
"""
ctx = build_model_context(
model_id,
mm_processor_kwargs=None,
limit_mm_per_prompt={"video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {"fps": fps}
# Build the image str / prompt based on the number of images we pass
prompt = "<|begin_of_video|><|video|><|end_of_video|>"
video_path = VideoAsset(name="baby_reading", num_frames=-1).video_path
with open(video_path, "rb") as f:
video_bytes = f.read()
static_video, static_metadata = VideoBackend.load_bytes(
video_bytes, backend=backend
)
dynamic_video, dynamic_metadata = DynamicVideoBackend.load_bytes(
video_bytes, fps=fps, backend=backend
)
# pre-sampled loader shouldn't read all frames
assert len(dynamic_video) < len(static_video)
static_mm_data = {"video": [(static_video, static_metadata)]}
dynamic_mm_data = {"video": [(dynamic_video, dynamic_metadata)]}
static_outputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(static_mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
dynamic_outputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(dynamic_mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
assert static_outputs["prompt_token_ids"] == dynamic_outputs["prompt_token_ids"]
assert batched_tensors_equal(
static_outputs["mm_kwargs"].get_data(),
dynamic_outputs["mm_kwargs"].get_data(),
)
@@ -1,95 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import pytest
from vllm.model_executor.models.llava_next_video import (
LlavaNextVideoForConditionalGeneration,
)
from vllm.model_executor.models.vision import get_vision_encoder_info
from ...utils import build_model_context
class _StubModel:
"""Carries only the attributes the token-count method reads from
`self`, so the real methods can be exercised without constructing
the full `nn.Module` (vision tower, language model, etc.)."""
patch_grid_length: int
pooled_grid_length: int
get_mm_lora_token_counts = (
LlavaNextVideoForConditionalGeneration.get_mm_lora_token_counts
)
@pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"])
def test_num_mm_tokens_match_real_config(model_id):
"""The stored grid lengths must match what `__init__` derives from
the real HF config, and the LoRA token counts must preserve the
frame-level scaling exactly."""
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
config = ctx.model_config.hf_config
vision_encoder_info = get_vision_encoder_info(config)
patch_grid_length = vision_encoder_info.get_patch_grid_length()
pooled_grid_length = math.ceil(patch_grid_length / config.spatial_pool_stride)
stub = _StubModel()
stub.patch_grid_length = patch_grid_length
stub.pooled_grid_length = pooled_grid_length
for num_frames in (1, 2, 8, 16, 32):
num_video_tokens = num_frames * pooled_grid_length**2
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
stub,
modality="video",
mm_kwargs=None,
num_mm_embeds=num_video_tokens,
)
assert encoder_tokens == num_frames * patch_grid_length**2
assert connector_tokens == num_video_tokens
@pytest.mark.parametrize(
("patch_grid_length", "pooled_grid_length", "num_frames"),
[
(24, 12, 1), # llava-hf/LLaVA-NeXT-Video-7B-hf: 336 / 14, stride 2
(24, 12, 16),
(27, 14, 5), # non-power-of-2 pooled grid (ceil rounding)
(16, 8, 6),
],
)
def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_frames):
stub = _StubModel()
stub.patch_grid_length = patch_grid_length
stub.pooled_grid_length = pooled_grid_length
num_video_tokens = num_frames * pooled_grid_length**2
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
stub,
modality="video",
mm_kwargs=None,
num_mm_embeds=num_video_tokens,
)
assert encoder_tokens == num_frames * patch_grid_length**2
assert connector_tokens == num_video_tokens
def test_num_mm_tokens_zero():
stub = _StubModel()
stub.patch_grid_length = 24
stub.pooled_grid_length = 12
assert get_mm_lora_token_counts(
stub,
modality="video",
mm_kwargs=None,
num_mm_embeds=0,
) == (0, 0)
@@ -1,302 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the LLaVA-OneVision-2 codec-video marker mechanism.
The codec video backend cannot be exercised end-to-end in CI because it
relies on the model's ``trust_remote_code`` package and real video bytes.
These tests cover the pure-Python marker plumbing that wraps a video *path*
for vLLM's ``MultiModalDataParser``:
* :func:`prepare_codec_video_input` must encode a per-path hash into its
dummy ndarray so distinct codec videos do not collide on ``mm_hash``
(otherwise ``EncoderCacheManager`` would skip the encoder for every video
after the first).
* :func:`_extract_codec_video_paths` must round-trip the marker back to the
original path(s) after the parser strips the metadata dict, and must return
``None`` for non-codec inputs.
"""
import types
import numpy as np
import pytest
from vllm.model_executor.models.llava_onevision2 import (
_CODEC_VIDEO_MARKER,
LlavaOnevision2VideoBackend,
_extract_codec_video_paths,
_frame_video_to_pil_and_timestamps,
_validate_video_source,
prepare_codec_video_input,
)
from vllm.multimodal.video import VideoSourceMetadata, VideoTargetMetadata
def _model_config(local: str = "", domains=None):
# Minimal stand-in for vLLM's ModelConfig: the validator only reads
# ``allowed_local_media_path`` and ``allowed_media_domains``.
return types.SimpleNamespace(
allowed_local_media_path=local,
allowed_media_domains=domains,
)
def test_prepare_codec_video_input_shape_and_marker():
dummy, meta = prepare_codec_video_input("/data/foo.mp4")
# 4-D ndarray satisfies MultiModalDataParser's video shape check.
assert isinstance(dummy, np.ndarray)
assert dummy.shape == (1, 1, 16, 3)
assert dummy.dtype == np.uint8
# Metadata carries the exact path under the codec marker key.
assert meta == {_CODEC_VIDEO_MARKER: "/data/foo.mp4"}
def test_prepare_codec_video_input_is_deterministic():
# Same path must yield identical dummy bytes (stable mm_hash).
a, _ = prepare_codec_video_input("/data/foo.mp4")
b, _ = prepare_codec_video_input("/data/foo.mp4")
assert a.tobytes() == b.tobytes()
def test_prepare_codec_video_input_distinct_paths_distinct_bytes():
# Distinct paths must yield distinct dummy bytes so the parser-visible
# ndarray (the only part reaching MultiModalHasher) varies per video.
paths = ["/data/a.mp4", "/data/b.mp4", "/data/c.mp4", "/data/a_.mp4"]
payloads = {prepare_codec_video_input(p)[0].tobytes() for p in paths}
assert len(payloads) == len(paths)
def test_extract_codec_video_paths_parser_list_shape():
# Parser yields list-of-(ndarray, metadata-dict) for tuple inputs.
items = [prepare_codec_video_input(p) for p in ("/x/1.mp4", "/x/2.mp4")]
assert _extract_codec_video_paths(items) == ["/x/1.mp4", "/x/2.mp4"]
def test_extract_codec_video_paths_single_raw_tuple():
# Single raw (ndarray, dict) tuple (pre-parser path) is also accepted.
item = prepare_codec_video_input("/x/solo.mp4")
assert _extract_codec_video_paths(item) == ["/x/solo.mp4"]
def test_extract_codec_video_paths_non_codec_returns_none():
# Plain decoded-frame inputs (ndarray / list of ndarray) are not codec
# markers and must fall through to the frame backend.
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
assert _extract_codec_video_paths(plain) is None
assert _extract_codec_video_paths([plain, plain]) is None
assert _extract_codec_video_paths([]) is None
# Tuple without the marker key is ignored.
assert _extract_codec_video_paths((plain, {"fps": 2.0})) is None
def test_extract_codec_video_paths_mixed_batch_returns_none():
# If any item in the batch lacks the marker, the whole batch is treated
# as non-codec (the backend does not mix codec and frame videos).
codec = prepare_codec_video_input("/x/1.mp4")
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
assert _extract_codec_video_paths([codec, plain]) is None
# ---------------------------------------------------------------------------
# Media access controls (_validate_video_source)
#
# The codec backend keeps the raw path string alive past vLLM's
# MultiModalDataParser and hands it to the trust-remote-code codec module,
# which opens it directly (cv2/ffmpeg) outside vLLM's MediaConnector. So the
# codec backend is restricted to *local files* confined to
# --allowed-local-media-path; remote http(s)/data URLs are rejected (they must
# use the frame backend, which rides the connector). The validator also returns
# the *resolved* path so the codec module opens exactly what was validated
# (no symlink-retarget / validate-vs-open gap).
# ---------------------------------------------------------------------------
def test_validate_http_url_rejected():
# Codec backend is local-only: remote URLs bypass the connector's domain
# and redirect controls, so they are rejected regardless of allowlist.
with pytest.raises(ValueError):
_validate_video_source("http://example.com/v.mp4", _model_config())
def test_validate_https_url_rejected_even_when_host_allowlisted():
with pytest.raises(ValueError):
_validate_video_source(
"https://good.com/v.mp4",
_model_config(domains=["good.com"]),
)
def test_validate_data_url_rejected():
# data: URLs are a remote/inline source for the frame backend, not codec.
with pytest.raises(ValueError):
_validate_video_source("data:video/mp4;base64,AAAA", _model_config())
def test_validate_local_file_blocked_without_allowed_path():
# Local file access is opt-in: without --allowed-local-media-path the
# bare path is rejected.
with pytest.raises(ValueError):
_validate_video_source("/data/v.mp4", _model_config())
def test_validate_local_file_allowed_inside_allowed_dir(tmp_path):
f = tmp_path / "v.mp4"
f.touch()
assert _validate_video_source(str(f), _model_config(local=str(tmp_path))) == str(f)
def test_validate_local_file_traversal_blocked():
# Path traversal escaping the allowed root is rejected after resolution.
with pytest.raises(ValueError):
_validate_video_source(
"/tmp/ov2/../../etc/passwd",
_model_config(local="/tmp/ov2"),
)
def test_validate_file_scheme_allowed_inside_allowed_dir(tmp_path):
f = tmp_path / "v.mp4"
f.touch()
assert _validate_video_source(
f.as_uri(), _model_config(local=str(tmp_path))
) == str(f)
def test_validate_file_scheme_percent_encoded_traversal_blocked():
# ``%2e%2e`` decodes to ``..``; the confinement check URL-decodes first
# (url2pathname) so the path cannot stay literally under the allowed root.
with pytest.raises(ValueError):
_validate_video_source(
"file:///tmp/ov2/%2e%2e/%2e%2e/etc/passwd",
_model_config(local="/tmp/ov2"),
)
def test_validate_unsupported_scheme_blocked():
with pytest.raises(ValueError):
_validate_video_source("ftp://example.com/v.mp4", _model_config())
def test_validate_returns_resolved_path_through_symlink(tmp_path):
# The validator resolves symlinks and returns the *real* path, so the codec
# module opens exactly what was validated (closes the validate-vs-open gap).
real = tmp_path / "real.mp4"
real.touch()
link = tmp_path / "link.mp4"
link.symlink_to(real)
assert _validate_video_source(str(link), _model_config(local=str(tmp_path))) == str(
real
)
def test_validate_relative_bare_path_blocked():
# Bare (scheme=="") paths come only from the codec backend and must be
# absolute: resolving a relative path against an ambiguous CWD before the
# confinement check is brittle/unsafe, so it is rejected outright.
with pytest.raises(ValueError):
_validate_video_source("ov2/v.mp4", _model_config(local="/tmp/ov2"))
# ---------------------------------------------------------------------------
# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
#
# Non-codec videos reach _apply_hf_processor_main as a ``(frames_ndarray, metadata)``
# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
# ``video_url`` inputs, or by the dummy-inputs builder during profiling
# (``video_needs_metadata=True``). The helper materialises PIL frames and
# per-frame timestamps (``frame_index / fps``), padding the frame count up to
# the even temporal-merge boundary.
# ---------------------------------------------------------------------------
def test_frame_video_to_pil_and_timestamps_basic():
frames = np.zeros((4, 8, 8, 3), dtype=np.uint8)
metadata = {"fps": 2.0, "frames_indices": [0, 4, 8, 12]}
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
assert len(pil_frames) == 4
assert all(f.size == (8, 8) for f in pil_frames)
# timestamps = frame_index / fps
assert timestamps == [0.0, 2.0, 4.0, 6.0]
def test_frame_video_to_pil_and_timestamps_even_pads_odd_frame_count():
# Odd frame count -> last frame repeated to satisfy temporal merge=2.
frames = np.zeros((3, 8, 8, 3), dtype=np.uint8)
metadata = {"fps": 1.0, "frames_indices": [0, 1, 2]}
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
assert len(pil_frames) == 4
assert len(timestamps) == 4
# The padded frame reuses the final index/timestamp.
assert timestamps == [0.0, 1.0, 2.0, 2.0]
def test_frame_video_to_pil_and_timestamps_defaults_when_metadata_sparse():
# Missing frames_indices -> sequential range; missing/zero fps -> default.
frames = np.zeros((2, 8, 8, 3), dtype=np.uint8)
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, {}))
assert len(pil_frames) == 2
# Default fps is 1.0, indices fall back to range(T).
assert timestamps == [0.0, 1.0]
def test_frame_video_to_pil_and_timestamps_rejects_non_tuple():
# Bare arrays (no metadata) must be rejected: the frame backend requires
# the (frames, metadata) tuple produced by the registered loader.
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
with pytest.raises(ValueError):
_frame_video_to_pil_and_timestamps(plain)
# ---------------------------------------------------------------------------
# LlavaOnevision2VideoBackend.compute_frames_index_to_sample honors the caller
# supplied VideoTargetMetadata (passed via --media-io-kwargs) so benchmarks can
# override the conservative defaults (fps=1.0, max_frames=32). Unset target
# fields (sentinel <= 0) fall back to those OV2 hf-chat reference constants.
# ---------------------------------------------------------------------------
def _src(total_frames: int, fps: float) -> VideoSourceMetadata:
return VideoSourceMetadata(
total_frames_num=total_frames,
original_fps=fps,
duration=total_frames / fps if fps > 0 else 0.0,
)
def test_backend_defaults_cap_at_32_frames():
# 300 frames @ 1fps source, target unset -> capped at default max_frames=32.
src = _src(300, 1.0)
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=300.0)
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
assert len(idx) == 32
assert idx[0] == 0
assert idx[-1] == 299
assert len(idx) % 2 == 0
def test_backend_target_num_frames_overrides_default_cap():
# VSI-Bench parity: target.num_frames=128 must lift the 32-frame cap.
src = _src(300, 1.0)
target = VideoTargetMetadata(num_frames=128, fps=-1, max_duration=300.0)
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
assert len(idx) == 128
assert idx[0] == 0
assert idx[-1] == 299
def test_backend_target_fps_controls_sampling_when_below_cap():
# 60 frames @ 30fps (2s) with target fps=1 -> ~2 frames (even-padded).
src = _src(60, 30.0)
target = VideoTargetMetadata(num_frames=128, fps=1.0, max_duration=300.0)
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
# fps-derived nframes (2) is below the 128 cap, so fps wins.
assert len(idx) <= 8
assert len(idx) % 2 == 0
@@ -1,330 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Mistral3's multimodal preprocessing kwargs."""
import pytest
import torch
import torch.nn.functional as F
from PIL import Image
from transformers import AutoProcessor, BatchFeature
from transformers.models.pixtral import PixtralProcessor
from vllm.model_executor.layers.fusion.mm_input_norm import (
FusedMMInputNorm,
IdentityInputNorm,
build_mm_input_norm,
)
from vllm.model_executor.models.lightonocr import (
LightOnOCRForConditionalGeneration,
LightOnOCRProcessingInfo,
)
from vllm.model_executor.models.mistral3 import Mistral3HFEncoderInfo
from vllm.model_executor.models.pixtral import (
PixtralHFEncoderInfo,
pixtral_patch_embed,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItems
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from ...utils import build_model_context
# This repo ships both params.json (Mistral) and config.json (HF). Auto config
# selects PixtralForConditionalGeneration; force HF to exercise Mistral3.
_MODEL_CONFIG_KWARGS = {"config_format": "hf"}
_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
_RGB_RESCALE = 1.0 / 255.0
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.skipif(
not HAS_TRITON or not current_platform.is_cuda(), reason="CUDA Triton kernel"
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("normalize", [False, True])
@pytest.mark.parametrize(("channels", "patch_shape"), [(3, (4, 4)), (2, (2, 4))])
def test_pixtral_patch_embed_preserves_image_order_and_strides(
dtype: torch.dtype,
normalize: bool,
channels: int,
patch_shape: tuple[int, int],
):
"""Packed projection matches CHW normalization and patch convolution."""
device = torch.device(current_platform.device_type)
norm = (
FusedMMInputNorm(
_RGB_MEAN[:channels],
_RGB_STD[:channels],
_RGB_RESCALE,
channel=channels,
).to(device)
if normalize
else IdentityInputNorm()
)
weight = torch.randn(32, channels, *patch_shape, device=device, dtype=dtype)
images = [
torch.randint(0, 256, (channels, 17, 18), device=device, dtype=torch.uint8),
torch.randint(0, 256, (channels, 25, 17), device=device, dtype=torch.uint8)[
:, 1:17, 1:17
],
torch.randint(0, 256, (channels, 19, 20), device=device, dtype=torch.uint8)[
:, 1:17, 2:18
],
]
reference = []
for image in images:
if normalize:
normalized = (
image.float() * norm.weight[:, None, None] + norm.bias[:, None, None]
)
else:
normalized = image
reference.append(
F.conv2d(normalized.to(dtype).unsqueeze(0), weight, stride=patch_shape)
)
packed, actual = pixtral_patch_embed(images, weight, norm)
for expected, result in zip(reference, actual):
torch.testing.assert_close(result, expected, rtol=0.01, atol=0.02)
expected_packed = torch.cat(
[result.flatten(2).transpose(1, 2) for result in reference], dim=1
)
torch.testing.assert_close(packed, expected_packed, rtol=0.01, atol=0.02)
def _process_images_with_hf(
hf_processor,
images: list[Image.Image],
mm_processor_kwargs: dict[str, object],
) -> tuple[list[torch.Tensor], list[int]]:
"""Process images and placeholders through the public HF processor."""
hf_out = hf_processor(
text=hf_processor.image_token * len(images),
images=images,
return_tensors="pt",
**mm_processor_kwargs,
)
pixel_values = hf_out["pixel_values"]
image_sizes = hf_out["image_sizes"]
unpadded = [p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)]
special_token_ids = {
hf_processor.image_token_id,
hf_processor.image_break_token_id,
hf_processor.image_end_token_id,
}
placeholder_tokens = [
token_id
for token_id in hf_out["input_ids"][0].tolist()
if token_id in special_token_ids
]
return unpadded, placeholder_tokens
def _placeholder_tokens_from_prompt_updates(
processor,
images: list[Image.Image],
pixel_values: list[torch.Tensor],
mm_processor_kwargs: dict[str, object],
) -> list[int]:
hf_inputs = BatchFeature({"pixel_values": pixel_values})
fields_config = processor._get_mm_fields_config(hf_inputs, mm_processor_kwargs)
out_mm_kwargs = MultiModalKwargsItems.from_hf_inputs(hf_inputs, fields_config)
# Prompt updates use raw PIL sizes and must predict the processed grid.
mm_items = processor.info.parse_mm_data({"image": images})
updates = processor._get_prompt_updates(
mm_items, mm_processor_kwargs, out_mm_kwargs
)
placeholder_tokens: list[int] = []
for item_idx in range(len(images)):
details = updates[0].resolve(item_idx).content
placeholder_tokens.extend(details.full)
return placeholder_tokens
def _expected_placeholder_tokens_per_image(
hf_processor,
pixel_values: torch.Tensor,
) -> int:
"""Count projected tokens from the actual HF-processed H×W."""
image_h, image_w = pixel_values.shape[-2:]
patch_size = hf_processor.image_processor.patch_size
if isinstance(patch_size, dict):
patch_h = patch_size["height"]
patch_w = patch_size["width"]
else:
patch_h = patch_w = int(patch_size)
spatial_merge_size = getattr(hf_processor, "spatial_merge_size", 1)
merged_patch_h = patch_h * spatial_merge_size
merged_patch_w = patch_w * spatial_merge_size
assert image_h % merged_patch_h == 0
assert image_w % merged_patch_w == 0
return (image_h // merged_patch_h) * (image_w // merged_patch_w)
@pytest.mark.parametrize("model_id", [_MODEL_ID])
@pytest.mark.parametrize(
("mm_processor_kwargs", "image_size", "expected_toks_per_img"),
[
({}, (448, 448), 256),
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
(
{"images_kwargs": {"size": {"longest_edge": 1008}}},
(1540, 1540),
1296,
),
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
({"size": {"longest_edge": 1008}}, (29, 29), 4),
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_size_override(
model_id: str,
mm_processor_kwargs: dict[str, object],
image_size: tuple[int, int],
expected_toks_per_img: int,
num_imgs: int,
kwargs_on_init: bool,
):
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
vllm_hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
assert isinstance(vllm_hf_processor, PixtralProcessor)
hf_processor = AutoProcessor.from_pretrained(
model_id,
fix_mistral_regex=True,
)
dummy_image = Image.new("RGB", image_size, color=(127, 127, 127))
images = [dummy_image] * num_imgs
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
hf_processor, images, merged_mm_kwargs
)
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
processor, images, pixel_values, hf_processor_mm_kwargs
)
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
hf_processor, pixel_values[0]
)
assert expected_from_pixel_values == expected_toks_per_img
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == (
expected_from_pixel_values * num_imgs
)
assert prompt_update_tokens == hf_placeholder_tokens
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
ctx = build_model_context(
_MODEL_ID,
limit_mm_per_prompt={"image": 2},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (31, 47), color=(17, 89, 231)),
Image.new("RGB", (48, 32), color=(201, 13, 127)),
]
prompt = [processor.info.get_hf_config().image_token_index] * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert all(value.dtype == torch.uint8 for value in raw_values)
input_norm = build_mm_input_norm(ctx.model_config)
patch_size = processor.info.get_hf_config().vision_config.patch_size
def pack_patches(image: torch.Tensor) -> torch.Tensor:
channels, height, width = image.shape
rows = height // patch_size
cols = width // patch_size
return (
image[:, : rows * patch_size, : cols * patch_size]
.reshape(channels, rows, patch_size, cols, patch_size)
.permute(1, 3, 0, 2, 4)
.reshape(-1, channels * patch_size**2)
)
for raw, normalized in zip(raw_values, normalized_values):
output = input_norm(pack_patches(raw), normalized.dtype)
torch.testing.assert_close(
output, pack_patches(normalized), rtol=1e-5, atol=1e-6
)
def test_scoped_request_size_overrides_configured_flat_size():
ctx = build_model_context(
_MODEL_ID,
mm_processor_kwargs={"size": {"longest_edge": 1008}},
limit_mm_per_prompt={"image": 1},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = AutoProcessor.from_pretrained(
_MODEL_ID,
fix_mistral_regex=True,
)
hf_processor_mm_kwargs: dict[str, object] = {
"images_kwargs": {"size": {"longest_edge": 1288}}
}
image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
hf_processor, [image], merged_mm_kwargs
)
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
processor, [image], pixel_values, hf_processor_mm_kwargs
)
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
hf_processor, pixel_values[0]
)
assert expected_from_pixel_values == 1656
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
assert prompt_update_tokens == hf_placeholder_tokens
def test_lightonocr_keeps_vision_config_image_size():
assert not LightOnOCRForConditionalGeneration.supports_mm_device_do_normalize
ctx = build_model_context(
_LIGHTON_MODEL_ID,
mm_processor_kwargs={"size": {"longest_edge": 1008}},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
assert isinstance(processor.info, LightOnOCRProcessingInfo)
encoder_info = processor.info.get_vision_encoder_info()
assert isinstance(encoder_info, PixtralHFEncoderInfo)
assert not isinstance(encoder_info, Mistral3HFEncoderInfo)
assert encoder_info.get_image_size() == (
processor.info.get_hf_config().vision_config.image_size
)
@@ -1,92 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
from vllm.model_executor.models.molmo2 import (
Molmo2DummyInputsBuilder,
build_flat_image_bool_length,
)
def test_build_flat_image_bool_length_matches_molmoweb_processor_tokens():
hf_config = SimpleNamespace(
image_patch_id=151938,
low_res_image_start_token_id=151940,
image_start_token_id=151936,
image_col_id=151939,
image_end_token_id=151937,
)
image_grids = torch.tensor([[14, 14, 14, 23]], dtype=torch.long)
image_tokens, num_image_tokens = build_flat_image_bool_length(
image_grids,
hf_config,
image_use_col_tokens=True,
use_single_crop_col_tokens=None,
use_single_crop_start_token=False,
)
assert num_image_tokens.tolist() == [550]
assert len(image_tokens) == 550
assert image_tokens[0].item() == hf_config.image_start_token_id
assert (image_tokens == hf_config.image_col_id).sum().item() == 28
def test_build_flat_image_bool_length_respects_disabled_col_tokens():
hf_config = SimpleNamespace(
image_patch_id=151938,
low_res_image_start_token_id=151940,
image_start_token_id=151936,
image_col_id=151939,
image_end_token_id=151937,
)
image_grids = torch.tensor([[2, 3, 5, 7]], dtype=torch.long)
image_tokens, num_image_tokens = build_flat_image_bool_length(
image_grids,
hf_config,
image_use_col_tokens=False,
use_single_crop_col_tokens=False,
use_single_crop_start_token=True,
)
assert num_image_tokens.tolist() == [45]
assert len(image_tokens) == 45
assert image_tokens[0].item() == hf_config.low_res_image_start_token_id
assert (image_tokens == hf_config.image_col_id).sum().item() == 0
@pytest.mark.parametrize(
("num_frames_override", "expected_frames"),
[(1, 2), (2, 2), (3, 3)],
)
def test_dummy_video_num_frames_override_honors_min_of_two(
num_frames_override, expected_frames
):
"""A ``num_frames`` override below 2 must be ignored (the model needs at
least 2 frames), matching the "cannot be less than 2, will be ignored"
warning."""
builder = object.__new__(Molmo2DummyInputsBuilder)
builder.info = SimpleNamespace(
get_hf_processor=lambda: SimpleNamespace(
video_processor=SimpleNamespace(size={"width": 64, "height": 64})
),
get_num_frames_with_most_features=lambda seq_len, mm_counts: 16,
get_image_size_with_most_features=lambda: (64, 64),
)
data = builder.get_dummy_mm_data(
seq_len=128,
mm_counts={"image": 0, "video": 1},
mm_options=MultiModalDummyOptions(
video=VideoDummyOptions(num_frames=num_frames_override)
),
)
video, _metadata = data["video"][0]
assert video.shape[0] == expected_frames
@@ -1,647 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from transformers import Qwen3Config
from vllm.model_executor.models.interfaces import SupportsLoRA, supports_lora
from vllm.model_executor.models.moss_audio import (
MOSS_AUDIO_BOS_TOKEN,
MOSS_AUDIO_BOS_TOKEN_ID,
MOSS_AUDIO_EOS_TOKEN,
MOSS_AUDIO_EOS_TOKEN_ID,
MOSS_AUDIO_PLACEHOLDER,
MOSS_AUDIO_TOKEN,
MOSS_AUDIO_TOKEN_ID,
MossAudioConfig,
MossAudioDummyInputsBuilder,
MossAudioEncoder,
MossAudioEncoderConfig,
MossAudioModel,
MossAudioMultiModalProcessor,
MossAudioProcessingInfo,
MossAudioProcessor,
MossQwen3ForCausalLM,
MossQwen3Model,
)
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.sequence import IntermediateTensors
class _Tokenizer:
def encode(self, text, **kwargs):
return [ord(char) for char in text]
def decode(self, token_ids, **kwargs):
return "".join(chr(token_id) for token_id in token_ids)
def batch_decode(self, batch_token_ids, **kwargs):
return [self.decode(token_ids, **kwargs) for token_ids in batch_token_ids]
class _MMConfig:
enable_mm_embeds = False
mm_processor_cache_gb = 1
mm_hasher_algorithm = "blake3"
def merge_mm_processor_kwargs(self, kwargs):
return dict(kwargs)
def get_limit_per_prompt(self, modality):
del modality
return 3
class _ModelConfig:
def __init__(self):
self.model = "OpenMOSS-Team/MOSS-Audio-4B-Instruct"
self.revision = None
self.max_model_len = 4096
self.encoder_config = {}
self.dtype = torch.float32
self.hf_config = MossAudioConfig(language_config=Qwen3Config())
self.multimodal_config = _MMConfig()
def get_multimodal_config(self):
return self.multimodal_config
def get_inputs_embeds_size(self):
return None
class _ProcessingContext:
def __init__(self):
self.model_config = _ModelConfig()
self.tokenizer = _Tokenizer()
def get_tokenizer(self):
return self.tokenizer
def get_hf_config(self):
return self.model_config.hf_config
def get_mm_config(self):
return self.model_config.get_multimodal_config()
def get_merged_mm_kwargs(self, kwargs):
return self.get_mm_config().merge_mm_processor_kwargs(kwargs)
def call_hf_processor(self, hf_processor, data, kwargs):
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
merged_kwargs.setdefault("return_tensors", "pt")
return hf_processor(**data, **merged_kwargs)
class _TestMossAudioProcessingInfo(MossAudioProcessingInfo):
def _get_processor_config_defaults(self):
return {}
def _vllm_config(tensor_parallel_size=1, pipeline_parallel_size=1, hf_config=None):
if hf_config is None:
hf_config = MossAudioConfig(language_config=Qwen3Config())
return SimpleNamespace(
model_config=SimpleNamespace(
hf_config=hf_config,
multimodal_config=None,
),
quant_config=None,
parallel_config=SimpleNamespace(
tensor_parallel_size=tensor_parallel_size,
pipeline_parallel_size=pipeline_parallel_size,
),
)
class _FakeAudioEncoder:
dtype = torch.float32
def __init__(self, deepstack_layers=0):
self.deepstack_layers = deepstack_layers
self.output_deepstack_hidden_states = None
self.input_shape = None
self.feature_lens = None
def __call__(self, audio_data, *, feature_lens, output_deepstack_hidden_states):
self.input_shape = tuple(audio_data.shape)
self.feature_lens = feature_lens.detach().cpu().clone()
self.output_deepstack_hidden_states = output_deepstack_hidden_states
lengths = MossAudioEncoder._compute_downsampled_length(feature_lens)
hidden_states = torch.ones(1, int(lengths.sum().item()), 8)
if not output_deepstack_hidden_states:
return hidden_states, None
return hidden_states, [
hidden_states * scale for scale in range(2, 2 + self.deepstack_layers)
]
def _patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=1, tp_rank=0):
import vllm.model_executor.layers.linear as linear_layers
import vllm.model_executor.models.moss_audio as moss_audio_module
import vllm.model_executor.parameter as parameter_module
for module in (moss_audio_module, linear_layers, parameter_module):
monkeypatch.setattr(
module, "get_tensor_model_parallel_world_size", lambda: tp_size
)
monkeypatch.setattr(
linear_layers, "get_tensor_model_parallel_rank", lambda: tp_rank
)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_rank", lambda: tp_rank
)
monkeypatch.setattr(
linear_layers, "tensor_model_parallel_all_reduce", lambda tensor: tensor
)
def _build_moss_audio_processor(cache=None):
ctx = _ProcessingContext()
info = _TestMossAudioProcessingInfo(ctx)
return (
MossAudioMultiModalProcessor(
info,
MossAudioDummyInputsBuilder(info),
),
ctx,
)
def _assert_mm_inputs_equal(left, right):
assert left["prompt_token_ids"] == right["prompt_token_ids"]
assert left["mm_hashes"] == right["mm_hashes"]
left_placeholder = left["mm_placeholders"]["audio"][0]
right_placeholder = right["mm_placeholders"]["audio"][0]
assert left_placeholder.offset == right_placeholder.offset
assert left_placeholder.length == right_placeholder.length
assert left_placeholder.is_embed.tolist() == right_placeholder.is_embed.tolist()
assert batched_tensors_equal(
left["mm_kwargs"].get_data(),
right["mm_kwargs"].get_data(),
)
@pytest.mark.parametrize(
("prompt", "prefix"),
[
(
f"before {MOSS_AUDIO_PLACEHOLDER} after",
[*[ord(char) for char in "before "], MOSS_AUDIO_BOS_TOKEN_ID],
),
(
f"before {MOSS_AUDIO_BOS_TOKEN}{MOSS_AUDIO_TOKEN}"
f"{MOSS_AUDIO_TOKEN}{MOSS_AUDIO_EOS_TOKEN} after",
[*[ord(char) for char in "before "], MOSS_AUDIO_BOS_TOKEN_ID],
),
("Describe this audio.", [MOSS_AUDIO_BOS_TOKEN_ID]),
],
)
def test_moss_audio_processor_expands_audio_placeholders(prompt, prefix):
raw_mel_len = 17
processed = MossAudioProcessor(_Tokenizer())(
text=prompt, audio=[torch.zeros(160 * raw_mel_len)]
)
input_ids = processed["input_ids"][0].tolist()
assert input_ids[: len(prefix)] == prefix
assert input_ids.count(MOSS_AUDIO_BOS_TOKEN_ID) == 1
assert input_ids.count(MOSS_AUDIO_EOS_TOKEN_ID) == 1
assert input_ids.count(MOSS_AUDIO_TOKEN_ID) == (
MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
)
assert processed["audio_data"].shape == (1, 128, raw_mel_len)
assert processed["audio_data_seqlens"].tolist() == [raw_mel_len]
def test_moss_audio_processor_preserves_placeholder_without_audio():
processed = MossAudioProcessor(_Tokenizer())(
text=f"before {MOSS_AUDIO_PLACEHOLDER} after"
)
assert processed["input_ids"][0].tolist() == [
*[ord(char) for char in "before "],
MOSS_AUDIO_BOS_TOKEN_ID,
MOSS_AUDIO_TOKEN_ID,
MOSS_AUDIO_EOS_TOKEN_ID,
*[ord(char) for char in " after"],
]
assert "audio_data" not in processed
assert "audio_data_seqlens" not in processed
def test_moss_audio_multimodal_processor_handles_token_and_cache_paths():
raw_mel_len = 17
audio = np.zeros(160 * raw_mel_len, dtype=np.float32)
prompt = f"{MOSS_AUDIO_PLACEHOLDER}\nTranscribe this audio."
baseline_processor, ctx = _build_moss_audio_processor()
mm_items = baseline_processor.info.parse_mm_data({"audio": [audio]})
token_prompt = ctx.get_tokenizer().encode(prompt, add_special_tokens=False)
baseline_text = baseline_processor(
prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
baseline_token = baseline_processor(
token_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
cached_processor, _ = _build_moss_audio_processor()
cached_text_miss = cached_processor(
prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
cache=cache,
)
cached_text_hit = cached_processor(
prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
cache=cache,
)
cached_token_hit = cached_processor(
token_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
cache=cache,
)
expected_audio_tokens = MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
prompt_token_ids = baseline_text["prompt_token_ids"]
assert prompt_token_ids.count(MOSS_AUDIO_TOKEN_ID) == expected_audio_tokens
assert baseline_text["mm_placeholders"]["audio"][0].length == (
expected_audio_tokens + 2
)
_assert_mm_inputs_equal(baseline_text, baseline_token)
_assert_mm_inputs_equal(baseline_text, cached_text_miss)
_assert_mm_inputs_equal(baseline_text, cached_text_hit)
_assert_mm_inputs_equal(baseline_text, cached_token_hit)
def test_moss_audio_supports_language_model_lora_only():
assert supports_lora(MossAudioModel)
model = object.__new__(MossAudioModel)
assert isinstance(model, SupportsLoRA)
mapping = model.get_mm_mapping()
assert mapping.language_model == ["language_model."]
assert mapping.tower_model == []
assert mapping.connector == []
def test_moss_audio_error_paths():
model = object.__new__(MossAudioModel)
with pytest.raises(ValueError, match="DeepStack audio token count mismatch"):
model._cache_deepstack_input_embeds(
inputs_embeds=torch.zeros(4, 8),
deepstack_embeddings=((torch.ones(1, 8),),),
is_multimodal=torch.tensor([False, True, True, False]),
)
with pytest.raises(ValueError, match="too short"):
MossAudioProcessor(_Tokenizer())(
text=MOSS_AUDIO_PLACEHOLDER, audio=[torch.empty(0)]
)
with pytest.raises(ValueError, match="too short"):
model._parse_and_validate_audio_input(
audio_data=torch.zeros(1, 128, 1),
audio_data_seqlens=torch.tensor([0], dtype=torch.long),
)
def test_moss_audio_validates_tp_config():
vllm_config = _vllm_config(tensor_parallel_size=2)
vllm_config.model_config.hf_config.adapter_hidden_size = 7
with pytest.raises(ValueError, match="adapter_hidden_size"):
MossAudioModel(vllm_config=vllm_config)
vllm_config = _vllm_config(tensor_parallel_size=2)
vllm_config.model_config.hf_config.audio_config.d_model = 6
vllm_config.model_config.hf_config.audio_config.encoder_attention_heads = 3
with pytest.raises(ValueError, match="encoder_attention_heads"):
MossAudioModel(vllm_config=vllm_config)
def test_moss_audio_rejects_audio_data_list_seqlen_count_mismatch():
model = object.__new__(MossAudioModel)
with pytest.raises(ValueError, match="audio_data batch size"):
model._parse_and_validate_audio_input(
audio_data=[torch.zeros(128, 8), torch.zeros(128, 11)],
audio_data_seqlens=torch.tensor([8], dtype=torch.long),
)
@pytest.mark.parametrize("deepstack_scales", [(), (7, 11)])
def test_moss_audio_embed_multimodal_packs_by_audio(deepstack_scales):
model = object.__new__(MossAudioModel)
model.audio_encoder = _FakeAudioEncoder(len(deepstack_scales))
model.audio_adapter = lambda hidden_states: hidden_states * 5
model.deepstack_audio_merger_list = [
lambda hidden_states, scale=scale: hidden_states * scale
for scale in deepstack_scales
]
model.deepstack_input_embeds = None
embeddings = model.embed_multimodal(
audio_data=torch.zeros(2, 128, 9),
audio_data_seqlens=torch.tensor([8, 9], dtype=torch.long),
)
assert model.audio_encoder.output_deepstack_hidden_states is bool(deepstack_scales)
assert [embeds.shape for embeds in embeddings] == [
torch.Size([1, 8 * (1 + len(deepstack_scales))]),
torch.Size([2, 8 * (1 + len(deepstack_scales))]),
]
if not deepstack_scales:
assert model.deepstack_input_embeds is None
return
main_embeddings, deepstack_embeddings = model._split_multimodal_embeddings(
embeddings, hidden_size=8
)
assert [embeds.shape for embeds in main_embeddings] == [
torch.Size([1, 8]),
torch.Size([2, 8]),
]
assert [[e.shape for e in layer] for layer in deepstack_embeddings] == [
[torch.Size([1, 8]), torch.Size([2, 8])] for _ in deepstack_scales
]
assert torch.equal(main_embeddings[0], torch.full((1, 8), 5.0))
for idx, scale in enumerate(deepstack_scales):
assert torch.equal(
deepstack_embeddings[idx][0],
torch.full((1, 8), float((idx + 2) * scale)),
)
def test_moss_audio_embed_input_ids_caches_packed_deepstack():
class _FakeLanguageModel:
def embed_input_ids(self, input_ids):
return torch.zeros(input_ids.shape[0], 8)
model = object.__new__(MossAudioModel)
model.language_model = _FakeLanguageModel()
model.deepstack_audio_merger_list = [object(), object()]
model.deepstack_input_embeds = None
multimodal_embeddings = (
torch.cat([torch.full((1, 8), x) for x in (5.0, 14.0, 33.0)], dim=-1),
torch.cat([torch.full((2, 8), x) for x in (7.0, 22.0, 44.0)], dim=-1),
)
is_multimodal = torch.tensor([False, True, True, True, False])
inputs_embeds = model.embed_input_ids(
input_ids=torch.arange(5),
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
assert torch.equal(inputs_embeds[1], torch.full((8,), 5.0))
assert torch.equal(inputs_embeds[2], torch.full((8,), 7.0))
assert torch.equal(inputs_embeds[3], torch.full((8,), 7.0))
assert model.deepstack_input_embeds is not None
tensors = model.deepstack_input_embeds.tensors
assert set(tensors) == {"deepstack_input_embeds_0", "deepstack_input_embeds_1"}
for tensor in tensors.values():
assert tensor[is_multimodal].abs().sum() > 0
assert torch.equal(tensor[~is_multimodal], torch.zeros(2, 8))
def _patch_pp_group(monkeypatch, *, first=True, last=True):
import vllm.model_executor.models.moss_audio as moss_audio_module
monkeypatch.setattr(
moss_audio_module,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=first, is_last_rank=last),
)
def test_moss_audio_pp_forward_routes_deepstack(monkeypatch):
for first in (True, False):
calls: list[dict[str, object]] = []
def fake_lm(*args, _calls=calls, **kwargs):
del args
_calls.append(kwargs)
return torch.ones(1, 1)
_patch_pp_group(monkeypatch, first=first)
model = object.__new__(MossAudioModel)
torch.nn.Module.__init__(model)
model.language_model = fake_lm
cached = IntermediateTensors({"deepstack_input_embeds_0": torch.ones(3, 8)})
inter = IntermediateTensors(
{
"hidden_states": torch.ones(3, 8),
"residual": torch.zeros(3, 8),
"deepstack_input_embeds_0": torch.full((3, 8), 5.0),
}
)
inputs_embeds = torch.full((3, 8), 9.0)
model.deepstack_input_embeds = cached
model.forward(
input_ids=None,
positions=torch.arange(3),
intermediate_tensors=None if first else inter,
inputs_embeds=inputs_embeds if first else None,
)
kwargs = calls[0]
assert kwargs["inputs_embeds"] is (inputs_embeds if first else None)
assert kwargs["deepstack_input_embeds"] is (cached if first else inter)
assert model.deepstack_input_embeds is None
calls = []
def fake_lm_non_first_rank(*args, **kwargs):
del args
calls.append(kwargs)
return torch.ones(1, 1)
_patch_pp_group(monkeypatch, first=False)
model = object.__new__(MossAudioModel)
torch.nn.Module.__init__(model)
model.language_model = fake_lm_non_first_rank
model.deepstack_input_embeds = IntermediateTensors({})
inter = IntermediateTensors(
{
"hidden_states": torch.ones(3, 8),
"residual": torch.zeros(3, 8),
}
)
model.forward(
input_ids=None,
positions=torch.arange(3),
intermediate_tensors=inter,
inputs_embeds=torch.ones(3, 8),
)
assert calls[0]["inputs_embeds"] is None
assert calls[0]["deepstack_input_embeds"] is inter
def test_moss_qwen3_deepstack_keys_for_pp(monkeypatch):
class AddOne(torch.nn.Module):
def forward(self, positions, hidden_states, residual):
del positions, residual
return hidden_states + 1, torch.zeros_like(hidden_states)
def make_model(num_layers, deepstack_layers=None):
model = object.__new__(MossQwen3Model)
torch.nn.Module.__init__(model)
model.start_layer, model.end_layer = 0, num_layers
model.layers = torch.nn.ModuleList([AddOne() for _ in range(num_layers)])
model.norm = lambda hidden_states, residual: (hidden_states, residual)
model._maybe_add_hidden_state = lambda aux, *args: aux
model.deepstack_inject_layer_indices = (
range(0) if deepstack_layers is None else deepstack_layers
)
return model
_patch_pp_group(monkeypatch, first=True, last=True)
output = make_model(3).forward(
input_ids=None,
positions=torch.arange(2),
inputs_embeds=torch.zeros(2, 4),
deepstack_input_embeds=IntermediateTensors(
{
"deepstack_input_embeds_2": torch.full((2, 4), 5.0),
}
),
)
assert torch.equal(output, torch.full((2, 4), 8.0))
_patch_pp_group(monkeypatch, first=True, last=False)
deepstack = IntermediateTensors(
{
"deepstack_input_embeds_0": torch.full((2, 4), 7.0),
"deepstack_input_embeds_3": torch.full((2, 4), 11.0),
}
)
output = make_model(2, range(4)).forward(
input_ids=None,
positions=torch.arange(2),
inputs_embeds=torch.zeros(2, 4),
deepstack_input_embeds=deepstack,
)
assert isinstance(output, IntermediateTensors)
assert set(output.tensors) == {
"hidden_states",
"residual",
"deepstack_input_embeds_2",
"deepstack_input_embeds_3",
}
assert torch.equal(output["hidden_states"], torch.full((2, 4), 9.0))
assert torch.equal(output["deepstack_input_embeds_2"], torch.zeros(2, 4))
assert output["deepstack_input_embeds_3"] is deepstack["deepstack_input_embeds_3"]
inner_model = make_model(0, range(2))
inner_model.make_empty_intermediate_tensors = lambda batch, dtype, device: (
IntermediateTensors(
{
"hidden_states": torch.zeros(batch, 4, dtype=dtype, device=device),
"residual": torch.zeros(batch, 4, dtype=dtype, device=device),
}
)
)
language_model = object.__new__(MossQwen3ForCausalLM)
torch.nn.Module.__init__(language_model)
language_model.model = inner_model
language_model.config = SimpleNamespace(hidden_size=4)
language_model.deepstack_inject_layer_indices = range(2)
tensors = MossQwen3ForCausalLM.make_empty_intermediate_tensors(
language_model,
batch_size=3,
dtype=torch.float16,
device=torch.device("cpu"),
)
assert set(tensors.tensors) == {
"hidden_states",
"residual",
"deepstack_input_embeds_0",
"deepstack_input_embeds_1",
}
assert tensors["deepstack_input_embeds_0"].shape == (3, 4)
assert tensors["deepstack_input_embeds_0"].dtype == torch.float16
_patch_pp_group(monkeypatch, first=True, last=False)
forward_tensors = inner_model.forward(
input_ids=None,
positions=torch.arange(3),
inputs_embeds=torch.ones(3, 4, dtype=torch.float16),
deepstack_input_embeds=None,
)
assert isinstance(forward_tensors, IntermediateTensors)
assert set(forward_tensors.tensors) == set(tensors.tensors)
def test_moss_audio_encoder_loads_realistic_attention_weight_names(monkeypatch):
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.device import DeviceConfig
_patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=2)
config = MossAudioEncoderConfig(
d_model=8,
output_dim=8,
num_mel_bins=8,
encoder_layers=1,
encoder_attention_heads=2,
encoder_ffn_dim=16,
downsample_hidden_size=2,
deepstack_encoder_layer_indexes=[],
)
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
encoder = MossAudioEncoder(config)
attention = encoder.layers[0].self_attn
assert all(hasattr(attention, name) for name in ("q_proj", "k_proj", "v_proj"))
assert hasattr(attention, "out_proj")
assert not hasattr(attention, "qkv")
assert attention.k_proj.bias is None
weight_names = [
"layers.0.self_attn.q_proj.weight",
"layers.0.self_attn.q_proj.bias",
"layers.0.self_attn.k_proj.weight",
"layers.0.self_attn.v_proj.weight",
"layers.0.self_attn.v_proj.bias",
"layers.0.self_attn.out_proj.weight",
"layers.0.self_attn.out_proj.bias",
"conv1.weight",
"conv1.bias",
]
params = dict(encoder.named_parameters(remove_duplicate=False))
assert "layers.0.self_attn.k_proj.bias" not in params
weights = {
name: torch.full_like(params[name], fill_value=float(i + 1))
for i, name in enumerate(weight_names)
}
loaded = AutoWeightsLoader(encoder).load_weights(weights.items())
assert "load_weights" not in MossAudioEncoder.__dict__
assert loaded == set(weight_names)
assert not any(".qkv." in name for name in loaded)
assert torch.equal(
params["layers.0.self_attn.q_proj.weight"],
weights["layers.0.self_attn.q_proj.weight"],
)
@@ -1,108 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.models.moss_transcribe_diarize import (
MossTranscribeDiarizeForConditionalGeneration,
)
def test_parse_diarized_transcript_preserves_moss_segments():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0.48][S01]Welcome[1.66][12.26][S02]Ready[13.81]"
)
assert [
(segment.start, segment.end, segment.speaker, segment.text)
for segment in segments
] == [
(0.48, 1.66, "S01", "Welcome"),
(12.26, 13.81, "S02", "Ready"),
]
def test_parse_diarized_transcript_preserves_overlapping_segments():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01]First speaker[2][1][S02]Second speaker[3]"
)
assert [
(segment.start, segment.end, segment.speaker, segment.text)
for segment in segments
] == [
(0.0, 2.0, "S01", "First speaker"),
(1.0, 3.0, "S02", "Second speaker"),
]
def test_parse_diarized_transcript_preserves_numeric_text_markers():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01]The [2024] report is ready.[4]"
)
assert [segment.text for segment in segments] == ["The [2024] report is ready."]
def test_parse_diarized_transcript_ignores_whitespace_between_segments():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01]Hello[1]\n [2][S02]Hi[3]"
)
assert [(segment.start, segment.end, segment.text) for segment in segments] == [
(0.0, 1.0, "Hello"),
(2.0, 3.0, "Hi"),
]
def test_parse_diarized_transcript_ignores_noise_before_a_segment():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"noise [bad][0.1][S01]Hello[0.9]"
)
assert [(segment.start, segment.end, segment.text) for segment in segments] == [
(0.1, 0.9, "Hello"),
]
def test_parse_diarized_transcript_preserves_timestamps_before_the_end():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[2][S01]The earlier timestamp is [1] not the end[3]"
)
assert [segment.text for segment in segments] == [
"The earlier timestamp is [1] not the end",
]
def test_parse_diarized_transcript_skips_empty_segments():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01][1][2][S02]Complete[3]"
)
assert [(segment.speaker, segment.text) for segment in segments] == [
("S02", "Complete"),
]
def test_parse_diarized_transcript_fails_closed_for_incomplete_output():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01]Complete[1][2][S02]Incomplete"
)
assert segments == []
def test_parse_diarized_transcript_fails_closed_for_trailing_text():
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
"[0][S01]Complete[1] trailing text"
)
assert segments == []
def test_parse_diarized_transcript_preserves_overlong_timestamp_markers():
marker = f"[{'1' * 33}]"
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
f"[0][S01]Value {marker}[1]"
)
assert [segment.text for segment in segments] == [f"Value {marker}"]
@@ -1,205 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for OpenVLA multimodal preprocessing."""
import numpy as np
import pytest
import torch
from PIL import Image
from transformers import LlamaConfig
from vllm.model_executor.models.openvla import (
OpenVLAForActionPrediction,
OpenVLAMultiModalProcessor,
OpenVLAProcessingInfo,
)
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
from vllm.transformers_utils.configs.openvla import OpenVLAConfig
from vllm.transformers_utils.processors.openvla import (
IMAGENET_MEAN,
IMAGENET_STD,
SIGLIP_MEAN,
SIGLIP_STD,
OpenVLAImageProcessor,
OpenVLAProcessor,
preprocess_openvla_image,
to_rgb_image,
)
pytestmark = pytest.mark.cpu_test
class _FakeTokenizer:
bos_token_id = 1
init_kwargs: dict[str, object] = {}
def encode(self, prompt: str, **kwargs: object) -> list[int]:
assert prompt == "In: test\nOut:"
if kwargs == {"add_special_tokens": True}:
return [self.bos_token_id, 10, 11]
assert kwargs == {"add_special_tokens": False}
return [10, 11]
def __call__(self, text: str, **kwargs: object) -> dict[str, list[list[int]]]:
return {"input_ids": [self.encode(text, **kwargs)]}
class _FakeProcessingInfo:
def __init__(self) -> None:
self.config = OpenVLAConfig()
def get_hf_config(self) -> OpenVLAConfig:
return self.config
def get_tokenizer(self) -> _FakeTokenizer:
return _FakeTokenizer()
def get_num_image_tokens(self, *, image_width: int, image_height: int) -> int:
assert image_width > 0
assert image_height > 0
return 256
class _FakeOpenVLAProcessingInfo(OpenVLAProcessingInfo):
def get_hf_config(self) -> OpenVLAConfig:
return OpenVLAConfig()
def _make_processor() -> OpenVLAMultiModalProcessor:
processor = OpenVLAMultiModalProcessor.__new__(OpenVLAMultiModalProcessor)
processor.info = _FakeProcessingInfo()
return processor
def test_openvla_config_converts_text_config_dict() -> None:
config = OpenVLAConfig(
text_config={
"vocab_size": 123,
"hidden_size": 64,
"intermediate_size": 128,
"num_hidden_layers": 2,
"num_attention_heads": 4,
},
)
assert isinstance(config.text_config, LlamaConfig)
assert config.text_config.vocab_size == 123
assert config.text_config.hidden_size == 64
assert config.text_config.architectures == ["LlamaForCausalLM"]
@pytest.mark.parametrize(
("image", "expected_size", "expected_pixel"),
[
(
torch.tensor(
[
[[1.0, 1.0], [1.0, 1.0]],
[[0.0, 0.0], [0.0, 0.0]],
[[0.0, 0.0], [0.0, 0.0]],
]
),
(2, 2),
(255, 0, 0),
),
(
np.full((4, 5, 1), 128, dtype=np.uint8),
(5, 4),
(128, 128, 128),
),
],
)
def test_openvla_to_rgb_image(
image: torch.Tensor | np.ndarray,
expected_size: tuple[int, int],
expected_pixel: tuple[int, int, int],
) -> None:
rgb_image = to_rgb_image(image)
assert rgb_image.mode == "RGB"
assert rgb_image.size == expected_size
assert rgb_image.getpixel((0, 0)) == expected_pixel
def test_openvla_preprocess_image_matches_expected_normalization() -> None:
image = Image.fromarray(
np.arange(12 * 10 * 3, dtype=np.uint8).reshape(10, 12, 3),
mode="RGB",
)
pixel_values = preprocess_openvla_image(image, image_size=224)
resized = image.resize((224, 224), Image.Resampling.BICUBIC)
raw = np.asarray(resized, dtype=np.float32) / 255.0
expected_dinov2 = ((raw - IMAGENET_MEAN) / IMAGENET_STD).transpose(2, 0, 1)
expected_siglip = ((raw - SIGLIP_MEAN) / SIGLIP_STD).transpose(2, 0, 1)
expected = np.concatenate([expected_dinov2, expected_siglip], axis=0)
assert pixel_values.shape == (6, 224, 224)
assert pixel_values.dtype == torch.float32
torch.testing.assert_close(pixel_values, torch.from_numpy(expected))
def test_openvla_processor_outputs_pixel_values() -> None:
processor = OpenVLAProcessor(
image_processor=OpenVLAImageProcessor(image_size=224),
tokenizer=_FakeTokenizer(),
)
image = Image.new("RGB", (8, 8), color=(255, 0, 0))
batch = processor(
text="In: test\nOut:",
images=image,
text_kwargs={"add_special_tokens": True},
)
assert batch["input_ids"] == [[1, 10, 11]]
assert batch["pixel_values"].shape == (1, 6, 224, 224)
assert batch["pixel_values"].dtype == torch.float32
def test_openvla_image_processor_outputs_pixel_values() -> None:
processor = OpenVLAImageProcessor(image_size=224)
image = Image.new("RGB", (8, 8), color=(255, 0, 0))
output = processor(images=image)
assert output["pixel_values"].shape == (1, 6, 224, 224)
assert output["pixel_values"].dtype == torch.float32
def test_openvla_processing_info_token_counts() -> None:
info = _FakeOpenVLAProcessingInfo.__new__(_FakeOpenVLAProcessingInfo)
assert info.get_supported_mm_limits() == {"image": 1}
assert info.get_num_image_tokens(image_width=640, image_height=480) == 256
assert info.get_image_size_with_most_features().width == 224
assert info.get_image_size_with_most_features().height == 224
assert info.get_mm_max_tokens_per_item(seq_len=2048, mm_counts={"image": 1}) == {
"image": 256
}
def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
processor = _make_processor()
image = Image.new("RGB", (640, 480), color=(255, 255, 255))
mm_items = MultiModalDataItems({"image": ImageProcessorItems([image])})
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
resolved = prompt_update.resolve(0)
content = resolved.content
assert resolved.modality == "image"
assert [
(match.start_idx, match.end_idx)
for match in resolved.iter_token_matches([1, 10, 11])
] == [(1, 1)]
assert content.full == [32000] * 256
is_embed = content.is_embed(content.full)
assert is_embed.dtype == torch.bool
assert is_embed.tolist() == [True] * 256
def test_openvla_placeholder_string() -> None:
assert OpenVLAForActionPrediction.get_placeholder_str("image", 0) is None
@@ -1,58 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for phi3v's multimodal preprocessing kwargs."""
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["microsoft/Phi-3.5-vision-instruct"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img"),
[
({"num_crops": 4}, 757),
({"num_crops": 16}, 1921),
# the default num_crops of phi-3.5-vision is 4
({}, 757),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, int],
expected_toks_per_img: int,
num_imgs: int,
kwargs_on_init: bool,
):
"""Ensure Phi3VMultiModalProcessor handles num_crops properly."""
# Avoid initializing CUDA early
from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
assert img_tok_count == expected_toks_per_img * num_imgs
@@ -1,64 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for phi4mm's multimodal preprocessing kwargs."""
import pytest
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize("model_id", ["microsoft/Phi-4-multimodal-instruct"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img"),
[
({"dynamic_hd": 4}, 1329),
({"dynamic_hd": 16}, 4433),
# the default num_crops of phi-4-multimodal is 36
({}, 9585),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, int],
expected_toks_per_img: int,
num_imgs: int,
kwargs_on_init: bool,
):
"""Ensure Phi4MMMultiModalProcessor handles dynamic_hd properly."""
# Avoid initializing CUDA early
from vllm.model_executor.models.phi4mm import _IMAGE_PLACEHOLDER_TOKEN_ID
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
image_size = ctx.get_hf_config().embd_layer["image_embd_layer"]["crop_size"]
dummy_image_size = (image_size * 7, image_size * 7)
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
mm_data = {"image": [dummy_image] * num_imgs}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
img_tok_count = processed_inputs["prompt_token_ids"].count(
_IMAGE_PLACEHOLDER_TOKEN_ID
)
assert img_tok_count == expected_toks_per_img * num_imgs
@@ -1,457 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for Qwen2.5-Omni embed_input_ids to verify embeddings are
correctly assigned to audio/image/video token positions.
Regression test for: https://github.com/vllm-project/vllm/issues/34506
- Non-interleaved mixed modalities (audio + image + video) should correctly
assign audio embeddings to audio positions, image to image, video to video.
- Interleaved (use_audio_in_video) should also work correctly.
"""
from unittest.mock import Mock
import pytest
import torch
from vllm.model_executor.models.qwen2_5_omni_thinker import (
check_interleaved_audio_video,
merge_interleaved_embeddings,
)
from vllm.multimodal.utils import set_mm_embedding_modality
# Fake token IDs
AUDIO_TOKEN_ID = 1001
IMAGE_TOKEN_ID = 1002
VIDEO_TOKEN_ID = 1003
TEXT_TOKEN_ID = 0
def _mm_embed(shape: tuple[int, ...], value: float, modality: str) -> torch.Tensor:
return set_mm_embedding_modality(torch.full(shape, value), modality)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def make_token_seq(
audio_n: int, image_n: int, video_n: int, text_prefix: int = 3, text_sep: int = 2
):
"""Build a flat token sequence:
[text_prefix] [AUDIO * audio_n] [text_sep] [IMAGE * image_n]
[text_sep] [VIDEO * video_n] [text_sep]
Returns (input_ids tensor, is_multimodal mask, positions dict).
"""
tokens = (
[TEXT_TOKEN_ID] * text_prefix
+ [AUDIO_TOKEN_ID] * audio_n
+ [TEXT_TOKEN_ID] * text_sep
+ [IMAGE_TOKEN_ID] * image_n
+ [TEXT_TOKEN_ID] * text_sep
+ [VIDEO_TOKEN_ID] * video_n
+ [TEXT_TOKEN_ID] * text_sep
)
input_ids = torch.tensor(tokens)
is_multimodal = (
(input_ids == AUDIO_TOKEN_ID)
| (input_ids == IMAGE_TOKEN_ID)
| (input_ids == VIDEO_TOKEN_ID)
)
return input_ids, is_multimodal
def make_interleaved_seq(
video_chunks: list[int], audio_chunks: list[int], text_prefix: int = 2
):
"""Build an interleaved sequence like use_audio_in_video:
[text] [V*v0] [A*a0] [V*v1] [A*a1] ...
"""
tokens = [TEXT_TOKEN_ID] * text_prefix
for v, a in zip(video_chunks, audio_chunks):
tokens += [VIDEO_TOKEN_ID] * v + [AUDIO_TOKEN_ID] * a
input_ids = torch.tensor(tokens)
is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID)
return input_ids, is_multimodal
# ---------------------------------------------------------------------------
# Tests for check_interleaved_audio_video
# ---------------------------------------------------------------------------
class TestCheckInterleavedAudioVideo:
def test_non_interleaved_audio_then_video(self):
"""Audio entirely before video → not interleaved."""
input_ids, is_multimodal = make_token_seq(5, 0, 4)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_non_interleaved_with_image(self):
"""Audio + image + video (the mixed_modalities case) → not interleaved."""
input_ids, is_multimodal = make_token_seq(5, 4, 6)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_no_audio(self):
"""Video only → not interleaved."""
input_ids, is_multimodal = make_token_seq(0, 0, 6)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_interleaved(self):
"""V A V A interleaved → True."""
input_ids, is_multimodal = make_interleaved_seq([4, 4], [3, 3])
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_multi_video_with_boundary_tokens(self):
"""Two interleaved videos separated by boundary tokens → still True.
use_audio_in_video expands each video into a local V/A span bounded by
non-pad tokens. A global density check would fail across those gaps.
"""
# [text][V A V A][text boundary][V A V A]
first_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=1)
second_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=0)
boundary = torch.tensor([TEXT_TOKEN_ID, TEXT_TOKEN_ID])
input_ids = torch.cat([first_ids, boundary, second_ids])
is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_batched_non_interleaved_no_false_positive(self):
"""Regression test for https://github.com/vllm-project/vllm/issues/35394.
5 identical non-interleaved mixed-modality requests batched together:
each has [audio][image][video] in separate blocks with text between them.
Across the batch, audio from request N falls between video blocks of
request N and request N+1, causing the global ranges to overlap.
check_interleaved_audio_video must return False (not a false positive).
"""
# Build one request: [text][audio*5][text][image*4][text][video*6][text]
single_ids, _ = make_token_seq(5, 4, 6)
# Batch 5 identical requests (separated by text tokens to simulate padding)
sep = torch.tensor([TEXT_TOKEN_ID] * 3)
batched_ids = torch.cat([single_ids, sep] * 5)
is_multimodal = (
(batched_ids == AUDIO_TOKEN_ID)
| (batched_ids == IMAGE_TOKEN_ID)
| (batched_ids == VIDEO_TOKEN_ID)
)
is_video = is_multimodal & (batched_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (batched_ids == AUDIO_TOKEN_ID)
assert not check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
), "Batched non-interleaved requests should not be detected as interleaved"
# ---------------------------------------------------------------------------
# Tests for embed_input_ids via a minimal mock
# ---------------------------------------------------------------------------
def make_mock_model(hidden: int = 8):
"""Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration
that has enough structure to run embed_input_ids.
"""
from vllm.model_executor.models.qwen2_5_omni_thinker import (
Qwen2_5OmniThinkerForConditionalGeneration,
)
model = Mock(spec=Qwen2_5OmniThinkerForConditionalGeneration)
# Config with token IDs
cfg = Mock()
cfg.video_token_index = VIDEO_TOKEN_ID
cfg.audio_token_index = AUDIO_TOKEN_ID
model.config = cfg
# embed_input_ids: simply embed each token as a one-hot-like vector
# token_id * ones so we can verify which embedding ends up where.
def fake_lm_embed(ids: torch.Tensor) -> torch.Tensor:
# Use .clone() so the tensor is contiguous (expand() creates a strided
# view with shared memory, which masked_scatter_ cannot handle).
return ids.float().unsqueeze(-1).expand(-1, hidden).clone()
lang_model = Mock()
lang_model.embed_input_ids = fake_lm_embed
model.get_language_model = Mock(return_value=lang_model)
# _embed_text_input_ids: delegate to SupportsMultiModal's implementation
from vllm.model_executor.models.interfaces import SupportsMultiModal
model._embed_text_input_ids = (
lambda *a, **kw: SupportsMultiModal._embed_text_input_ids(model, *a, **kw)
)
# super().embed_input_ids → use SupportsMultiModal.embed_input_ids
def fake_super_embed(
ids,
mm_embs=None,
*,
is_multimodal=None,
):
return SupportsMultiModal.embed_input_ids(
model,
ids,
mm_embs,
is_multimodal=is_multimodal,
)
# Bind embed_input_ids as the real method
model.embed_input_ids = (
lambda *a, **kw: Qwen2_5OmniThinkerForConditionalGeneration.embed_input_ids(
model, *a, **kw
)
)
# Store super-embed for use inside the method
model._super_embed_input_ids = fake_super_embed
return model, hidden
def build_mm_embeds(
audio_n, image_n, video_n, hidden, audio_val=10.0, image_val=20.0, video_val=30.0
):
"""Build multimodal_embeddings list in position order (audio, image, video).
Each embedding is filled with a distinct constant so we can verify placement.
"""
embs = []
if audio_n:
embs.append(torch.full((audio_n, hidden), audio_val))
if image_n:
embs.append(torch.full((image_n, hidden), image_val))
if video_n:
embs.append(torch.full((video_n, hidden), video_val))
return embs
class TestEmbedInputIds:
def _run(self, audio_n, image_n, video_n, hidden=8):
"""Run embed_input_ids for a non-interleaved mixed-modality sequence.
Returns (result_embeds, input_ids, is_multimodal).
"""
input_ids, is_multimodal = make_token_seq(audio_n, image_n, video_n)
mm_embeds = build_mm_embeds(audio_n, image_n, video_n, hidden)
model, _ = make_mock_model(hidden)
result = model.embed_input_ids(
input_ids, mm_embeds, is_multimodal=is_multimodal
)
return result, input_ids, is_multimodal
def test_audio_only(self):
"""Audio-only: audio positions get audio embeddings."""
audio_n, hidden = 5, 8
audio_val = 10.0
result, input_ids, is_multimodal = self._run(audio_n, 0, 0, hidden)
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
"Audio positions should get audio embeddings"
)
def test_video_only(self):
"""Video-only: video positions get video embeddings."""
video_n, hidden = 6, 8
video_val = 30.0
result, input_ids, is_multimodal = self._run(0, 0, video_n, hidden)
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
"Video positions should get video embeddings"
)
def test_mixed_modalities_audio_goes_to_audio_pos(self):
"""Regression test for GitHub issue #34506:
With audio + image + video (non-interleaved), audio positions must
receive audio embeddings (not image or video embeddings).
"""
audio_n, image_n, video_n, hidden = 5, 4, 6, 8
audio_val, image_val, video_val = 10.0, 20.0, 30.0
result, input_ids, is_multimodal = self._run(audio_n, image_n, video_n, hidden)
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
image_pos = (input_ids == IMAGE_TOKEN_ID).nonzero(as_tuple=True)[0]
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
mean_a = result[audio_pos].mean().item()
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
f"Audio emb wrong: expected {audio_val}, got mean={mean_a:.1f}"
)
mean_i = result[image_pos].mean().item()
assert result[image_pos].allclose(torch.full((image_n, hidden), image_val)), (
f"Image emb wrong: expected {image_val}, got mean={mean_i:.1f}"
)
mean_v = result[video_pos].mean().item()
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
f"Video emb wrong: expected {video_val}, got mean={mean_v:.1f}"
)
def test_text_positions_unchanged(self):
"""Text positions should keep their text embeddings."""
audio_n, image_n, video_n, hidden = 3, 2, 4, 8
result, input_ids, is_multimodal = self._run(audio_n, image_n, video_n, hidden)
text_pos = (~is_multimodal).nonzero(as_tuple=True)[0]
# Text tokens have value TEXT_TOKEN_ID=0, so embed → 0.0
assert result[text_pos].allclose(torch.zeros(len(text_pos), hidden)), (
"Text positions should keep text embeddings"
)
def test_interleaved_use_audio_in_video(self):
"""Interleaved (use_audio_in_video): video chunks interleaved with audio.
Video embeddings must go to video positions, audio to audio positions.
"""
hidden = 8
audio_val, video_val = 10.0, 30.0
# Two video chunks of 4, two audio chunks of 3
video_chunks = [4, 4]
audio_chunks = [3, 3]
input_ids, is_multimodal = make_interleaved_seq(video_chunks, audio_chunks)
video_n = sum(video_chunks) # 8
audio_n = sum(audio_chunks) # 6
# mm_embeds come in [video, audio] order (video feature first in
# mm_features when positions are the same for use_audio_in_video)
mm_embeds = [
_mm_embed((video_n, hidden), video_val, "video"),
_mm_embed((audio_n, hidden), audio_val, "audio"),
]
model, _ = make_mock_model(hidden)
# Modalities are attached on the embedding tensors (as in encoder gather).
result = model.embed_input_ids(
input_ids,
mm_embeds,
is_multimodal=is_multimodal,
)
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
"Interleaved: video positions should get video embeddings"
)
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
"Interleaved: audio positions should get audio embeddings"
)
# ---------------------------------------------------------------------------
# Tests for merge_interleaved_embeddings helper
# ---------------------------------------------------------------------------
class TestMergeInterleavedEmbeddings:
def test_basic_interleaved(self):
"""Video chunks + audio chunks scattered to correct positions."""
hidden = 4
input_ids, is_multimodal = make_interleaved_seq([3, 3], [2, 2])
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
num_video = is_video.sum().item() # 6
num_audio = is_audio.sum().item() # 4
inputs_embeds = torch.zeros(len(input_ids), hidden)
mm_embeds = [
_mm_embed((num_video, hidden), 30.0, "video"),
_mm_embed((num_audio, hidden), 10.0, "audio"),
]
result = merge_interleaved_embeddings(
inputs_embeds,
mm_embeds,
is_video,
is_audio,
is_multimodal,
)
video_pos = is_video.nonzero(as_tuple=True)[0]
audio_pos = is_audio.nonzero(as_tuple=True)[0]
assert result[video_pos].allclose(torch.full((num_video, hidden), 30.0))
assert result[audio_pos].allclose(torch.full((num_audio, hidden), 10.0))
def test_image_and_video_mixed(self):
"""Image embeddings must not be misclassified as video."""
hidden = 4
# [text][I I][V A V A]
tokens = (
[TEXT_TOKEN_ID] * 2
+ [IMAGE_TOKEN_ID] * 2
+ [VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID]
)
input_ids = torch.tensor(tokens)
is_multimodal = (
(input_ids == IMAGE_TOKEN_ID)
| (input_ids == VIDEO_TOKEN_ID)
| (input_ids == AUDIO_TOKEN_ID)
)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
is_image = is_multimodal & (input_ids == IMAGE_TOKEN_ID)
inputs_embeds = torch.zeros(len(input_ids), hidden)
mm_embeds = [
_mm_embed((2, hidden), 20.0, "image"),
_mm_embed((2, hidden), 30.0, "video"),
_mm_embed((2, hidden), 10.0, "audio"),
]
result = merge_interleaved_embeddings(
inputs_embeds,
mm_embeds,
is_video,
is_audio,
is_multimodal,
)
assert result[is_image.nonzero(as_tuple=True)[0]].allclose(
torch.full((2, hidden), 20.0)
)
assert result[is_video.nonzero(as_tuple=True)[0]].allclose(
torch.full((2, hidden), 30.0)
)
assert result[is_audio.nonzero(as_tuple=True)[0]].allclose(
torch.full((2, hidden), 10.0)
)
def test_missing_modality_raises(self):
hidden = 2
input_ids, is_multimodal = make_interleaved_seq([2], [2])
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
with pytest.raises(ValueError, match="Missing modality"):
merge_interleaved_embeddings(
torch.zeros(len(input_ids), hidden),
[torch.zeros(2, hidden), torch.zeros(2, hidden)],
is_video,
is_audio,
is_multimodal,
)
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -1,484 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
import torch
from PIL import Image
from vllm.config.multimodal import MultiModalConfig
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.multimodal.processing.context import InputProcessingContext
from vllm.platforms import current_platform
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize(
("mm_kwargs", "expected"),
[
(
{"size": {"shortest_edge": 64, "longest_edge": 1024}},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 64,
"max_pixels": 1024,
},
),
(
{"min_pixels": 64, "max_pixels": 1024},
{
"min_pixels": 64,
"max_pixels": 1024,
"size": {"shortest_edge": 64, "longest_edge": 1024},
},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
),
(
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {"max_pixels": 4096},
"videos_kwargs": {"size": {"longest_edge": 8192}},
},
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {
"max_pixels": 4096,
"size": {"longest_edge": 4096},
},
"videos_kwargs": {
"size": {"longest_edge": 8192},
"max_pixels": 8192,
},
},
),
(
{"min_pixels": 0},
{"min_pixels": 0, "size": {"shortest_edge": 0}},
),
(
{"size": None, "min_pixels": 64},
{"size": {"shortest_edge": 64}, "min_pixels": 64},
),
],
)
def test_complete_mm_processor_size_aliases(
mm_kwargs: dict[str, object],
expected: dict[str, object],
) -> None:
original = deepcopy(mm_kwargs)
actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
assert actual == expected
assert mm_kwargs == original
def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
mm_config = MultiModalConfig(
mm_processor_kwargs={
"images_kwargs": {"min_pixels": 64},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None)
info = Qwen2VLProcessingInfo(ctx)
monkeypatch.setattr(
info,
"get_supported_mm_processor_kwargs",
lambda: {"images_kwargs": {"size", "min_pixels"}},
)
assert info._merge_and_resolve_mm_processor_kwargs(
{"size": {"shortest_edge": 128}}
) == {
"images_kwargs": {
"size": {"shortest_edge": 128},
"min_pixels": 128,
},
}
@pytest.mark.parametrize(
("mm_kwargs", "default_size", "expected"),
[
(
{},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 32, "longest_edge": 1024},
),
(
{"size": {"shortest_edge": 64}},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 1024},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": 128,
"max_pixels": 4096,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 128, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64},
"max_pixels": 4096,
},
None,
{"shortest_edge": 64, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": None,
"max_pixels": 0,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 0},
),
],
)
def test_get_vision_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None,
expected: dict[str, object],
) -> None:
original_mm_kwargs = {
key: dict(value) if isinstance(value, dict) else value
for key, value in mm_kwargs.items()
}
original_default_size = dict(default_size) if default_size is not None else None
actual = Qwen2VLProcessingInfo._get_vision_size(
mm_kwargs,
default_size=default_size,
)
assert actual == expected
assert mm_kwargs == original_mm_kwargs
assert default_size == original_default_size
@pytest.mark.parametrize(
("modality", "scope"),
[
("image", "images_kwargs"),
("video", "videos_kwargs"),
],
)
def test_get_vision_info_reads_merged_modality_scope(
monkeypatch: pytest.MonkeyPatch,
modality: str,
scope: str,
) -> None:
vision_config = SimpleNamespace(
patch_size=14,
spatial_merge_size=2,
temporal_patch_size=2,
)
ctx = SimpleNamespace(
get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
)
info = Qwen2VLProcessingInfo(ctx)
merged = {
"images_kwargs": {"size": {"shortest_edge": 111}},
"videos_kwargs": {"size": {"shortest_edge": 222}},
}
monkeypatch.setattr(
info,
"_merge_and_resolve_mm_processor_kwargs",
lambda _: merged,
)
seen: list[dict[str, object]] = []
def get_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None = None,
) -> dict[str, object]:
seen.append(mm_kwargs)
return dict(default_size or {})
monkeypatch.setattr(info, "_get_vision_size", get_size)
image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
info._get_vision_info(
image_width=28,
image_height=28,
num_frames=2,
do_resize=False,
image_processor=image_processor,
mm_kwargs={},
modality=modality,
)
assert seen == [merged[scope]]
def test_jina_vl_processing_order() -> None:
"""Jina's document-first prompt keeps cached features and hashes aligned."""
ctx = build_model_context(
"jinaai/jina-reranker-m0",
runner="pooling",
limit_mm_per_prompt={"image": 2},
mm_processor_cache_gb=1,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
processor = MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
query_image = Image.new("RGB", (128, 160), color=(255, 0, 0))
document_image = Image.new("RGB", (192, 128), color=(0, 255, 0))
def process(images: list[Image.Image]):
return processor(
placeholder * len(images),
mm_items=processor.info.parse_mm_data({"image": images}),
cache=cache,
)
query = process([query_image])
document = process([document_image])
pair = process([query_image, document_image])
pair_items = pair["mm_kwargs"]["image"]
assert pair["mm_hashes"]["image"] == [
document["mm_hashes"]["image"][0],
query["mm_hashes"]["image"][0],
]
assert batched_tensors_equal(
pair_items[0].get_data(),
document["mm_kwargs"]["image"][0].get_data(),
)
assert batched_tensors_equal(
pair_items[1].get_data(),
query["mm_kwargs"]["image"][0].get_data(),
)
assert [item.length for item in pair["mm_placeholders"]["image"]] == [
document["mm_placeholders"]["image"][0].length,
query["mm_placeholders"]["image"][0].length,
]
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"),
[
({}, 1426, (5704, 1176)),
({"min_pixels": 64**2, "max_pixels": 512**2}, 330, (1320, 1176)),
(
{
"size": {
"shortest_edge": 64**2,
"longest_edge": 512**2,
},
},
330,
(1320, 1176),
),
],
)
@pytest.mark.parametrize("num_imgs", [1, 2])
@pytest.mark.parametrize("kwargs_on_init", [True, False])
def test_processor_override(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_toks_per_img: int,
expected_pixels_shape: tuple[int, int],
num_imgs: int,
kwargs_on_init: bool,
):
"""Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
limit_mm_per_prompt={"image": num_imgs},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokenizer = processor.info.get_tokenizer()
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
# Build the image str / prompt based on the number of images we pass
prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
)
# Ensure we have the right number of placeholders per num_crops size
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token)
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values"].shape
assert img_tok_count == expected_toks_per_img * num_imgs
assert pixel_shape[0] == expected_pixels_shape[0] * num_imgs
assert pixel_shape[1] == expected_pixels_shape[1]
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
@pytest.mark.parametrize(
"mm_processor_kwargs",
[
{"min_pixels": 28 * 28, "max_pixels": 1280 * 28 * 28},
{"min_pixels": 28 * 28, "max_pixels": 1283 * 28 * 28},
{"size": {"shortest_edge": 28 * 28, "longest_edge": 1280 * 28 * 28}},
{"size": {"shortest_edge": 28 * 28, "longest_edge": 1283 * 28 * 28}},
],
)
def test_get_image_size_with_most_features(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
):
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
max_image_size = processor.info.get_image_size_with_most_features()
max_tokens = processor.info.get_num_image_tokens(
image_width=max_image_size.width,
image_height=max_image_size.height,
image_processor=hf_processor.image_processor,
mm_kwargs=mm_processor_kwargs,
)
prompt = "<|vision_start|><|image_pad|><|vision_end|>"
for asset in image_assets:
mm_data = {"image": [asset.pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
grid_thw = processed_inputs["mm_kwargs"].get_data()["image_grid_thw"].tolist()
t, h, w = grid_thw[0]
tokens = (t * h * w) // (merge_size**2)
assert tokens < max_tokens
def _build_qwen2_5_vl_video_mm_data(num_frames: int, fps: float) -> dict[str, Any]:
video = np.zeros((num_frames, 56, 56, 3), dtype=np.uint8)
metadata = {
"fps": fps,
"duration": num_frames / fps,
"total_num_frames": num_frames,
"frames_indices": list(range(num_frames)),
"video_backend": "opencv",
"do_sample_frames": False,
}
return {"video": [(video, metadata)]}
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2.5-VL-3B-Instruct"])
def test_qwen2_5_vl_video_fps_to_second_per_grid_ts(model_id: str) -> None:
ctx = build_model_context(model_id, limit_mm_per_prompt={"image": 0, "video": 1})
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
temporal_patch_size = (
processor.info.get_hf_processor().video_processor.temporal_patch_size
)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
for fps in (1.0, 30.0):
mm_data = _build_qwen2_5_vl_video_mm_data(num_frames=32, fps=fps)
processed = processor(prompt, mm_items=processor.info.parse_mm_data(mm_data))
spg = processed["mm_kwargs"]["video"][0].get_data()["second_per_grid_ts"]
assert float(spg) == pytest.approx(temporal_patch_size / fps, rel=2e-2)
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize(
"model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]
)
@pytest.mark.parametrize("num_imgs", [1, 2])
def test_mm_device_do_normalize(
image_assets: ImageTestAssets, model_id: str, num_imgs: int
) -> None:
"""Device-side normalisation must reproduce the on-CPU processor result.
Runs on any platform: the CPU platform exercises the ``forward_native``
fallback, accelerators exercise the fused kernel.
"""
device = current_platform.device_type
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": num_imgs},
)
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Build the image str / prompt based on the number of images we pass
prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
processed_inputs_with_normalize = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
)
pixel_values_with_normalize = processed_inputs_with_normalize[
"mm_kwargs"
].get_data()["pixel_values"]
dtype = pixel_values_with_normalize.dtype
processed_inputs_without_normalize = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={"do_normalize": False, "do_rescale": False},
)
pixel_values_without_normalize = processed_inputs_without_normalize[
"mm_kwargs"
].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
input_norm = build_mm_input_norm(ctx.model_config).to(device)
# With normalisation disabled, the processor emits raw uint8 pixels,
# matching the production mm_device_do_normalize path.
assert pixel_values_without_normalize.dtype == torch.uint8
pixel_values_do_input_norm = input_norm(
pixel_values_without_normalize.to(device), dtype
)
torch.testing.assert_close(
pixel_values_with_normalize.to(device), pixel_values_do_input_norm
)
@@ -1,604 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for Qwen3-VL multimodal processing."""
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
import torch
from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
from vllm.config import ModelConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.processing.context import BaseProcessingInfo
from ...registry import HF_EXAMPLE_MODELS
from ...utils import build_model_context
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
assert "size" in native_video_kwargs
assert "min_pixels" not in native_video_kwargs
assert "max_pixels" not in native_video_kwargs
info = Qwen3VLProcessingInfo(SimpleNamespace())
parent_supported = {
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels", "max_pixels"},
"videos_kwargs": set(native_video_kwargs),
}
monkeypatch.setattr(
BaseProcessingInfo,
"get_supported_mm_processor_kwargs",
lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
)
supported = info.supported_mm_processor_kwargs
assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
assert parent_supported["videos_kwargs"] == native_video_kwargs
def _build_video_mm_data(
num_frames: int,
width: int = 128,
height: int = 128,
original_fps: float = 30.0,
) -> dict[str, Any]:
"""Create synthetic video data with metadata indicating that
HF processor should re-sample frames (do_sample_frames=True).
``total_num_frames`` is set equal to the ndarray frame count so
that HF's ``sample_frames`` indices stay within bounds of the
actual tensor that is passed."""
video = np.zeros((num_frames, height, width, 3), dtype=np.uint8)
metadata = {
"fps": original_fps,
"duration": num_frames / original_fps,
"total_num_frames": num_frames,
"frames_indices": list(range(num_frames)),
"video_backend": "opencv",
"do_sample_frames": True,
}
return {"video": [(video, metadata)]}
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_image_size(model_id: str) -> None:
"""Partial image ``size`` must be completed and applied before the HF call.
Keep video compatibility and per-video kwargs in an image-only call as well:
Qwen3 must sanitize the video scope before HF validates it. Regression for
#56363.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = np.zeros((128, 128, 3), dtype=np.uint8)
min_pixels = 256 * 32 * 32
default_size = dict(processor.info.get_image_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [image]})
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"image_grid_thw"
]
stock_grid = process({})
complete_grid = process({"images_kwargs": {"size": complete_size}})
partial_grid = process(
{
"images_kwargs": {
"size": {"shortest_edge": min_pixels},
},
"videos_kwargs": {
"min_pixels": 4096,
"max_pixels": 469_762_048,
"fps": [2.0],
"num_frames": [8],
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_video_size(model_id: str) -> None:
"""Partial scoped video ``size`` must be completed and applied before HF."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
min_pixels = 512 * 32 * 32
default_size = dict(processor.info.get_video_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
complete_grid = process({"videos_kwargs": {"size": complete_size}})
partial_grid = process(
{
"videos_kwargs": {
"size": {"shortest_edge": min_pixels},
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
"""Flat min/max pixel aliases keep their historical Qwen3 video effect."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
pixel_budget = 512 * 32 * 32
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
scoped_grid = process(
{
"videos_kwargs": {
"size": {
"shortest_edge": pixel_budget,
"longest_edge": pixel_budget,
},
},
}
)
flat_grid = process(
{
"min_pixels": pixel_budget,
"max_pixels": pixel_budget,
}
)
assert torch.equal(flat_grid, scoped_grid)
assert not torch.equal(flat_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"num_frames",
[8, 16],
)
def test_processor_num_frames_timestamp(
model_id: str,
num_frames: int,
) -> None:
"""Regression test: using ``num_frames`` (without ``fps``) must not
cause a timestamp / token-count mismatch.
Before the fix, ``_get_video_second_idx`` ignored the explicit
``num_frames`` and fell back to an fps-based calculation, which
produced a different number of timestamp entries and ultimately led
to shape mismatches in downstream token construction.
We deliberately choose ``num_frames`` values (8, 16) that differ
from what the default fps-based path would compute (which clamps
to ``min_frames=4`` for a short video at 30 fps), so this test
would fail without the fix.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
mm_data = _build_video_mm_data(num_frames=num_frames)
# Process with explicit num_frames (no fps) -- this is the path
# that was broken before the fix.
hf_mm_kwargs: dict[str, Any] = {"num_frames": num_frames}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
# Basic sanity: the processor must produce video tokens.
token_ids = processed["prompt_token_ids"]
assert len(token_ids) > 0, "Processor produced empty token list"
# Verify that video placeholders were actually inserted.
assert "mm_placeholders" in processed
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 1, (
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
)
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize("num_videos", [2, 4])
def test_processor_multi_video(
model_id: str,
num_videos: int,
) -> None:
"""Verify that multi-video processing produces correct placeholders.
This exercises the token-level replacement path in
``_apply_hf_processor_main`` which avoids the quadratic text-level
prompt expansion.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": num_videos},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * num_videos
mm_data = {"video": [_build_video_mm_data(num_frames=8)["video"][0]] * num_videos}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={"num_frames": 8},
)
token_ids = processed["prompt_token_ids"]
assert len(token_ids) > 0
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == num_videos, (
f"Expected {num_videos} video placeholders, got {len(video_phs)}"
)
# All placeholders should have the same length (same video params)
# and must not overlap.
lengths = {ph.length for ph in video_phs}
assert len(lengths) == 1, f"Placeholder lengths differ: {lengths}"
for i in range(1, len(video_phs)):
prev_end = video_phs[i - 1].offset + video_phs[i - 1].length
assert video_phs[i].offset >= prev_end, (
f"Placeholder {i} overlaps with placeholder {i - 1}"
)
# Qwen3-VL / Qwen3.8 "Long Video Understanding" pixel budget from the
# model card. Used to check --mm-processor-kwargs scoping (#52834).
_LONG_VIDEO_SIZE = {"longest_edge": 469762048, "shortest_edge": 4096}
def _probe_mm_token_budgets(
model_id: str, mm_processor_kwargs: dict[str, Any] | None
) -> tuple[int, int]:
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
)
info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info
video = info.get_max_video_tokens(
seq_len=500000, mm_counts={"video": 1, "image": 1}
)
return video, info.get_max_image_tokens()
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
model_id: str,
) -> None:
"""``videos_kwargs.size`` must raise only the video token budget.
A flat ``size`` override still applies to both modalities (the previous
shared-namespace behavior). Regression for #52834.
"""
stock_video, stock_image = _probe_mm_token_budgets(model_id, None)
scoped_video, scoped_image = _probe_mm_token_budgets(
model_id, {"videos_kwargs": {"size": _LONG_VIDEO_SIZE}}
)
flat_video, flat_image = _probe_mm_token_budgets(
model_id, {"size": _LONG_VIDEO_SIZE}
)
assert scoped_video == flat_video
assert scoped_video > stock_video
assert scoped_image == stock_image
assert flat_image > stock_image
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
("hf_mm_kwargs", "scalar_kwargs"),
[
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [2.0, 4.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
({"num_frames": [8, 16, 32]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [2.0, 4.0, 8.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
],
)
@pytest.mark.parametrize("model_kwargs", [False, True])
@pytest.mark.parametrize("use_uuids", [False, True])
@pytest.mark.parametrize("scoped", [False, True])
@pytest.mark.skip_global_cleanup
def test_processor_multi_video_list_kwargs(
model_id: str,
hf_mm_kwargs: dict[str, Any],
scalar_kwargs: list[dict[str, Any]],
model_kwargs: bool,
use_uuids: bool,
scoped: bool,
) -> None:
"""Per-video options must survive duplicate items and partial cache hits."""
if scoped:
hf_mm_kwargs = {"videos_kwargs": hf_mm_kwargs}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 2},
mm_processor_kwargs=hf_mm_kwargs if model_kwargs else None,
mm_processor_cache_gb=1,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
videos = [
_build_video_mm_data(num_frames=32, original_fps=8.0)["video"][0]
for _ in range(3)
]
for i, (frames, _) in enumerate(videos):
frames.fill(i)
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
single = processor(
single_prompt,
mm_items=processor.info.parse_mm_data({"video": [videos[0]]}),
hf_processor_mm_kwargs=hf_kwargs,
)
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
assert not torch.equal(expected_grids[0], expected_grids[1])
def run(indices, *, cache=None, cached_only=False):
return processor(
prompt,
mm_items=processor.info.parse_mm_data(
{"video": [None if cached_only else videos[i] for i in indices]}
),
mm_uuid_items=(
{"video": [f"video-{i}" for i in indices]} if use_uuids else None
),
hf_processor_mm_kwargs={} if model_kwargs else hf_mm_kwargs,
cache=cache,
)
for indices in ([0, 0], [0, 1], [0, 2], [2, 0]):
expected = run(indices)
actual = run(indices, cache=cache)
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
assert actual["mm_placeholders"] == expected["mm_placeholders"]
assert actual["mm_kwargs"] == expected["mm_kwargs"]
assert actual["mm_hashes"] == expected["mm_hashes"]
assert len(set(actual["mm_hashes"]["video"])) == 2
multi_grids = actual["mm_kwargs"].get_data()["video_grid_thw"]
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
if use_uuids:
cached_only = run([2, 0], cache=cache, cached_only=True)
assert cached_only["prompt_token_ids"] == actual["prompt_token_ids"]
assert cached_only["mm_kwargs"] == actual["mm_kwargs"]
assert hf_mm_kwargs == hf_mm_kwargs_before
@pytest.mark.parametrize(("key", "value"), [("fps", 2.0), ("num_frames", 8)])
@pytest.mark.parametrize("length", [0, 1])
@pytest.mark.parametrize("use_cache", [False, True])
@pytest.mark.parametrize("scoped", [False, True])
def test_processor_rejects_short_per_video_options(
key: str, value: float | int, length: int, use_cache: bool, scoped: bool
) -> None:
"""Reject malformed option lists with and without a warm processor cache."""
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 0, "video": 2},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
cache = MultiModalProcessorOnlyCache(ctx.model_config) if use_cache else None
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
videos = _build_video_mm_data(num_frames=32, original_fps=8.0)["video"] * 2
def process(values: list[float | int]) -> None:
kwargs = {key: values}
processor(
prompt,
mm_items=processor.info.parse_mm_data({"video": videos}),
hf_processor_mm_kwargs={"videos_kwargs": kwargs} if scoped else kwargs,
cache=cache,
)
if use_cache:
process([value, value])
with pytest.raises(ValueError, match=f"{key} must have at least 2 entries"):
process([value] * length)
@pytest.mark.parametrize("kwargs", [{"fps": [2.0, 4.0]}, {"num_frames": [8, 16]}])
def test_processor_profiles_fewer_videos_than_configured_options(
kwargs: dict[str, list[float] | list[int]],
) -> None:
"""Profiling may use fewer videos than the configured per-video list."""
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 0, "video": 2},
mm_processor_kwargs=kwargs,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
profiled = processor.get_dummy_mm_inputs({"video": 1})
assert len(profiled["mm_placeholders"]["video"]) == 1
assert profiled["mm_kwargs"].get_data()["video_grid_thw"].shape[0] == 1
def _build_video_embeds_mm_data(
hidden_size: int,
grid_thw: tuple[int, int, int] = (2, 4, 4),
) -> dict[str, Any]:
"""Create an embeds-only video item as an EPD consumer receives it:
pre-computed embeddings plus the metadata published by the encoder."""
import torch
t, h, w = grid_thw
num_tokens = t * h * w // 4 # spatial_merge_size ** 2
return {
"video": {
"video_embeds": torch.zeros(num_tokens, hidden_size),
"video_grid_thw": torch.tensor([grid_thw]),
"timestamps": torch.tensor([[float(i) for i in range(t)]]),
}
}
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
"""Embeds-only video input must size the placeholder range from the
grid and the real timestamps published by the encoder (EC consumer
path); synthesized or missing timestamps would change the token count
and break embedding merging downstream."""
# `build_model_context` forces enable_mm_embeds off, so build the
# config directly; keep the same online-availability skip behavior.
HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip")
model_config = ModelConfig(
model_id,
runner="generate",
limit_mm_per_prompt={"image": 0, "video": 1},
enable_mm_embeds=True,
)
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
tokenizer = processor.info.get_tokenizer()
grid_thw = (2, 4, 4)
mm_data = _build_video_embeds_mm_data(
model_config.get_inputs_embeds_size(), grid_thw
)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 1, (
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
)
t, h, w = grid_thw
tokens_per_frame = h * w // 4
timestamp_tokens = sum(
len(tokenizer.encode(f"<{float(i):.1f} seconds>", add_special_tokens=False))
for i in range(t)
)
# per frame: timestamp tokens + vision_start + video tokens + vision_end
expected_len = timestamp_tokens + t * (tokens_per_frame + 2)
assert video_phs[0].length == expected_len
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_video_embeds_missing_timestamps(model_id: str) -> None:
"""Timestamps are required metadata for video embeds: they size the
placeholder range, so omitting them must fail loudly at parse time
instead of silently producing a wrong prompt."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
mm_data = _build_video_embeds_mm_data(ctx.model_config.get_inputs_embeds_size())
del mm_data["video"]["timestamps"]
with pytest.raises(ValueError, match="timestamps"):
processor.info.parse_mm_data(mm_data)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None:
"""Regression test for memory profiling with ``cap_pixels_per_frame``.
With the HF per-frame pixel cap enabled (transformers#48071) and a
raised video budget, a 2-frame profiling dummy would be processed at
only 2 * cap pixels, underestimating the largest possible video (a
fully sampled one still fills the whole ``longest_edge`` budget). The
dummy builder must spread the budget over enough frames that the cap
is not binding.
"""
# 16 capped frames' worth of budget on top of the default per-frame
# ceiling (max_video_tokens=768 at patch 16, merge 2 -> 786,432
# pixels per frame).
per_frame_cap = 768 * (16 * 2) ** 2
budget = per_frame_cap * 16
size = {"longest_edge": budget, "shortest_edge": 4096}
capped_ctx = build_model_context(
model_id,
mm_processor_kwargs={
"videos_kwargs": {
"size": size,
"cap_pixels_per_frame": True,
},
},
limit_mm_per_prompt={"image": 0, "video": 1},
)
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
capped_dummy = capped.dummy_inputs.get_dummy_mm_data(
1024, {"video": 1}, MultiModalDummyOptions()
)
capped_frames = capped_dummy["video"][0][0].shape[0]
assert capped_frames == 16, (
f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}"
)
uncapped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"videos_kwargs": {"size": size}},
limit_mm_per_prompt={"image": 0, "video": 1},
)
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(
1024, {"video": 1}, MultiModalDummyOptions()
)
uncapped_frames = uncapped_dummy["video"][0][0].shape[0]
assert uncapped_frames == 2, (
f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}"
)
@@ -1,58 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Step3-VL precomputed image embedding inputs."""
import pytest
import torch
from vllm.model_executor.models.step3_vl import (
Step3VLForConditionalGeneration,
Step3VLImageEmbeddingInputs,
)
class _FakeStep3VL:
@staticmethod
def _process_image_features(image_features: torch.Tensor) -> torch.Tensor:
return image_features
def test_image_embedding_inputs_construction():
"""Step3VLImageEmbeddingInputs should store embeddings in the data field."""
image_embeds = torch.randn(2, 16, 64)
inputs = Step3VLImageEmbeddingInputs(
type="image_embeds",
data=image_embeds,
)
assert inputs["type"] == "image_embeds"
assert torch.equal(inputs["data"], image_embeds)
assert torch.equal(inputs.data, image_embeds)
def test_image_embedding_inputs_validation_rejects_wrong_rank():
"""Validation should reject tensors with wrong rank."""
with pytest.raises(ValueError, match="rank"):
Step3VLImageEmbeddingInputs(
type="image_embeds",
data=torch.randn(16, 64),
)
def test_process_image_embeds_does_not_require_pixel_input_fields():
"""The image_embeds branch should not reference patch pixel metadata."""
image_embeds = torch.randn(2, 4, 8)
image_input = Step3VLImageEmbeddingInputs(
type="image_embeds",
data=image_embeds,
)
outputs = Step3VLForConditionalGeneration._process_image_input(
_FakeStep3VL(),
image_input,
)
assert len(outputs) == 2
assert torch.equal(outputs[0], image_embeds[0])
assert torch.equal(outputs[1], image_embeds[1])