[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,154 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2025 The vLLM team.
|
||||
# Copyright 2025 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
|
||||
# reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
|
||||
|
||||
class MockAudioFlamingo3Config(PreTrainedConfig):
|
||||
model_type = "audioflamingo3"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.audio_config = PreTrainedConfig()
|
||||
self.text_config = PreTrainedConfig()
|
||||
|
||||
|
||||
class MockAudioFlamingo3Processor:
|
||||
def __init__(self):
|
||||
self.audio_token = "<sound>"
|
||||
self.audio_token_id = 12345
|
||||
self.max_audio_len = 60
|
||||
self.feature_extractor = MockFeatureExtractor()
|
||||
self.tokenizer = self._tokenize
|
||||
|
||||
def __call__(self, text=None, audio=None, **kwargs):
|
||||
return {
|
||||
"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
|
||||
"input_features": torch.zeros((3, 80, 3000)),
|
||||
"input_features_mask": torch.ones((3, 3000), dtype=torch.long),
|
||||
}
|
||||
|
||||
def _tokenize(self, text, **kwargs):
|
||||
return {"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long)}
|
||||
|
||||
|
||||
class MockFeatureExtractor:
|
||||
def __init__(self):
|
||||
self.sampling_rate = 16000
|
||||
self.chunk_length = 30
|
||||
self.hop_length = 160
|
||||
|
||||
def __call__(self, audios, **kwargs):
|
||||
return {
|
||||
"input_features": torch.zeros((len(audios), 80, 3000)),
|
||||
"attention_mask": torch.ones((len(audios), 3000), dtype=torch.long),
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
config = MockAudioFlamingo3Config()
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.get_hf_config.return_value = config
|
||||
ctx.get_hf_processor.return_value = MockAudioFlamingo3Processor()
|
||||
ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
|
||||
**data, **kwargs
|
||||
)
|
||||
ctx.model_config.hf_config = config
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def check_transformers_version():
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info("AudioFlamingo3ForConditionalGeneration")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
|
||||
def test_audio_chunk_counting(mock_ctx):
|
||||
from vllm.model_executor.models.audioflamingo3 import (
|
||||
AudioFlamingo3DummyInputsBuilder,
|
||||
AudioFlamingo3MultiModalProcessor,
|
||||
AudioFlamingo3ProcessingInfo,
|
||||
)
|
||||
|
||||
info = AudioFlamingo3ProcessingInfo(mock_ctx)
|
||||
processor = AudioFlamingo3MultiModalProcessor(
|
||||
info, AudioFlamingo3DummyInputsBuilder(info)
|
||||
)
|
||||
|
||||
sr = 16000
|
||||
audio_1 = np.zeros(30 * sr)
|
||||
audio_2 = np.zeros(75 * sr)
|
||||
|
||||
mm_data = {"audio": [audio_1, audio_2]}
|
||||
|
||||
mm_items = processor.info.parse_mm_data(mm_data, validate=False)
|
||||
processed = processor._apply_hf_processor_main(mm_items, {})
|
||||
|
||||
chunk_counts = processed["chunk_counts"]
|
||||
|
||||
assert chunk_counts[0].item() == 1
|
||||
assert chunk_counts[1].item() == 2
|
||||
assert len(chunk_counts) == 2
|
||||
assert processed["feature_attention_mask"].shape == (3, 3000)
|
||||
|
||||
|
||||
def test_dummy_data_generation(mock_ctx):
|
||||
from vllm.model_executor.models.audioflamingo3 import (
|
||||
AudioFlamingo3DummyInputsBuilder,
|
||||
AudioFlamingo3ProcessingInfo,
|
||||
)
|
||||
|
||||
info = AudioFlamingo3ProcessingInfo(mock_ctx)
|
||||
builder = AudioFlamingo3DummyInputsBuilder(info)
|
||||
|
||||
mm_counts = {"audio": 2}
|
||||
dummy_data = builder.get_dummy_mm_data(100, mm_counts, MultiModalDummyOptions())
|
||||
|
||||
assert "audio" in dummy_data
|
||||
assert len(dummy_data["audio"]) == 2
|
||||
|
||||
expected_len = 60 * 16000
|
||||
assert len(dummy_data["audio"][0]) == expected_len
|
||||
|
||||
|
||||
def test_audio_token_count_matches_hf_processor_math():
|
||||
from vllm.model_executor.models.audioflamingo3 import (
|
||||
_count_audio_tokens_from_mask,
|
||||
)
|
||||
|
||||
feature_attention_mask = torch.zeros((3, 3000), dtype=torch.long)
|
||||
feature_attention_mask[0, :2999] = 1
|
||||
feature_attention_mask[1, :2999] = 1
|
||||
feature_attention_mask[2, :1500] = 1
|
||||
chunk_counts = torch.tensor([2, 1], dtype=torch.long)
|
||||
|
||||
assert (
|
||||
_count_audio_tokens_from_mask(feature_attention_mask, chunk_counts, 0) == 1499
|
||||
)
|
||||
assert _count_audio_tokens_from_mask(feature_attention_mask, chunk_counts, 1) == 375
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Bailing prompt expansion tests without downloading a checkpoint."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from tokenizers import Tokenizer, decoders, models, pre_tokenizers
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm.model_executor.models.bailing_moe_v3_vl import (
|
||||
IMAGE_PLACEHOLDER,
|
||||
IMAGE_TOKEN,
|
||||
BailingMoeV3VLMultiModalProcessor,
|
||||
)
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def processor():
|
||||
vocab = {
|
||||
token: index
|
||||
for index, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
|
||||
}
|
||||
# A newline added by the processor must merge with an existing newline.
|
||||
vocab["ĊĊ"] = len(vocab)
|
||||
backend = Tokenizer(models.BPE(vocab=vocab, merges=[("Ċ", "Ċ")]))
|
||||
backend.pre_tokenizer = pre_tokenizers.ByteLevel(
|
||||
add_prefix_space=False, use_regex=False
|
||||
)
|
||||
backend.decoder = decoders.ByteLevel()
|
||||
tokenizer = TokenizersBackend(
|
||||
tokenizer_object=backend,
|
||||
additional_special_tokens=[
|
||||
"<|vision_start|>",
|
||||
IMAGE_TOKEN,
|
||||
"<|vision_end|>",
|
||||
],
|
||||
clean_up_tokenization_spaces=False,
|
||||
)
|
||||
processor = object.__new__(BailingMoeV3VLMultiModalProcessor)
|
||||
processor.info = SimpleNamespace(
|
||||
get_tokenizer=lambda: tokenizer,
|
||||
get_hf_config=lambda: SimpleNamespace(
|
||||
vision_config=SimpleNamespace(spatial_merge_size=2)
|
||||
),
|
||||
)
|
||||
return processor
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "expected_text", "token_counts"),
|
||||
[
|
||||
(
|
||||
f"before{IMAGE_PLACEHOLDER}after",
|
||||
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
|
||||
[2],
|
||||
),
|
||||
(
|
||||
f"before{IMAGE_TOKEN}after",
|
||||
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
|
||||
[2],
|
||||
),
|
||||
(
|
||||
f"before{IMAGE_PLACEHOLDER}\nafter",
|
||||
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n\nafter",
|
||||
[2],
|
||||
),
|
||||
(
|
||||
f"before{IMAGE_PLACEHOLDER}{IMAGE_PLACEHOLDER}after",
|
||||
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n"
|
||||
f"<|vision_start|>{IMAGE_TOKEN * 3}<|vision_end|>\nafter",
|
||||
[2, 3],
|
||||
),
|
||||
],
|
||||
ids=["wrapped", "bare", "existing_newline", "adjacent_images"],
|
||||
)
|
||||
def test_image_expansion_matches_wrappers_and_newline_tokenization(
|
||||
processor, prompt, expected_text, token_counts
|
||||
):
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
image_id = tokenizer.get_vocab()[IMAGE_TOKEN]
|
||||
grids = torch.tensor([[1, 2, 2 * count] for count in token_counts])
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
{"image_grid_thw": grids},
|
||||
{"image_grid_thw": MultiModalFieldConfig.batched("image")},
|
||||
)
|
||||
mm_items = MultiModalDataItems(
|
||||
{"image": ImageProcessorItems([Image.new("RGB", (1, 1))] * len(token_counts))}
|
||||
)
|
||||
updates = processor._get_mm_prompt_updates(mm_items, {}, mm_kwargs)
|
||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
expected_ids = tokenizer.encode(expected_text, add_special_tokens=False)
|
||||
|
||||
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, updates)
|
||||
|
||||
assert actual_ids == expected_ids
|
||||
assert len(placeholders["image"]) == len(token_counts)
|
||||
found = processor._find_mm_placeholders(actual_ids, updates)
|
||||
search_offset = 0
|
||||
for index, count in enumerate(token_counts):
|
||||
placeholder = placeholders["image"][index]
|
||||
expected_offset = expected_ids.index(image_id, search_offset)
|
||||
assert placeholder.start_idx == expected_offset
|
||||
assert placeholder.tokens == [image_id] * count
|
||||
assert placeholder.to_range().get_num_embeds() == count
|
||||
assert placeholder.to_range() == found["image"][index].to_range()
|
||||
search_offset = expected_offset + count
|
||||
|
||||
|
||||
def test_text_only_prompt_preserves_original_token_ids(processor):
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
prompt_ids = tokenizer.encode("before\n", add_special_tokens=False)
|
||||
prompt_ids += tokenizer.encode("\nafter", add_special_tokens=False)
|
||||
# Deliberately use two newline tokens instead of their merged BPE token.
|
||||
assert prompt_ids != tokenizer.encode("before\n\nafter", add_special_tokens=False)
|
||||
|
||||
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, {})
|
||||
|
||||
assert actual_ids == prompt_ids
|
||||
assert placeholders == {}
|
||||
@@ -1,244 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for DeepSeek-OCR TensorSchema validation with empty images_crop.
|
||||
|
||||
When using the Gundam preset (BASE_SIZE=1024, IMAGE_SIZE=640, CROP_MODE=True),
|
||||
images that are small enough to not require cropping produce an empty
|
||||
images_crop tensor with shape (0, 3, 640, 640). The _parse_and_validate_image_input
|
||||
method must correctly read image_size from this tensor's shape rather than
|
||||
falling back to base_size, which would cause a TensorSchema mismatch.
|
||||
|
||||
Run with:
|
||||
pytest tests/models/multimodal/processing/test_deepseek_ocr.py -v
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.model_executor.models.deepseek_ocr import (
|
||||
DeepseekOCRForCausalLM,
|
||||
DeepseekOCRImagePixelInputs,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_ocr2 import DeepseekOCR2ForCausalLM
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import (
|
||||
DeepseekOCRProcessor,
|
||||
ImageTransform,
|
||||
)
|
||||
|
||||
MODEL_ID = "deepseek-ai/DeepSeek-OCR"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def processor():
|
||||
"""Load the DeepseekOCRProcessor with tokenizer from HuggingFace."""
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
||||
return DeepseekOCRProcessor(tokenizer=tokenizer)
|
||||
|
||||
|
||||
class TestDeepseekOCREmptyImagesCrop:
|
||||
"""Verify TensorSchema validation handles empty images_crop correctly."""
|
||||
|
||||
def test_empty_images_crop_small_image(self, processor):
|
||||
"""A small image (<=640px) produces empty images_crop and should
|
||||
not crash the TensorSchema validation.
|
||||
|
||||
Previously, the code used ``numel() > 0`` to decide whether to read
|
||||
image_size from the tensor shape. When numel()==0, it fell back to
|
||||
base_size=1024, mismatching the actual tensor dim of 640.
|
||||
"""
|
||||
# Small image: both dims <= IMAGE_SIZE (640) → no crops
|
||||
small_image = Image.new("RGB", (100, 100), color="red")
|
||||
|
||||
result = processor(
|
||||
prompt="<image>\nDescribe this image.",
|
||||
images=[small_image],
|
||||
)
|
||||
|
||||
pixel_values = result["pixel_values"]
|
||||
images_crop = result["images_crop"]
|
||||
images_spatial_crop = result["images_spatial_crop"]
|
||||
|
||||
# Processor must produce an empty crop tensor for a small image
|
||||
assert images_crop.shape[0] == 0
|
||||
|
||||
base_size = pixel_values.shape[-1]
|
||||
image_size = images_crop.shape[-1] if images_crop is not None else base_size
|
||||
|
||||
# This should NOT raise ValueError
|
||||
schema = DeepseekOCRImagePixelInputs(
|
||||
type="pixel_values",
|
||||
data=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
resolve_bindings={
|
||||
"base_size": base_size,
|
||||
"image_size": image_size,
|
||||
},
|
||||
)
|
||||
|
||||
assert schema.data.shape == (1, 3, 1024, 1024)
|
||||
assert schema.images_crop.shape == (0, 3, 640, 640)
|
||||
|
||||
def test_populated_images_crop_large_image(self, processor):
|
||||
"""A large image (>640px) produces populated images_crop."""
|
||||
# Large image: exceeds IMAGE_SIZE (640) → dynamic crop tiles
|
||||
large_image = Image.new("RGB", (1200, 800), color="blue")
|
||||
|
||||
result = processor(
|
||||
prompt="<image>\nDescribe this image.",
|
||||
images=[large_image],
|
||||
)
|
||||
|
||||
pixel_values = result["pixel_values"]
|
||||
images_crop = result["images_crop"]
|
||||
images_spatial_crop = result["images_spatial_crop"]
|
||||
|
||||
assert images_crop.shape[0] > 0
|
||||
|
||||
base_size = pixel_values.shape[-1]
|
||||
image_size = images_crop.shape[-1]
|
||||
|
||||
schema = DeepseekOCRImagePixelInputs(
|
||||
type="pixel_values",
|
||||
data=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
resolve_bindings={
|
||||
"base_size": base_size,
|
||||
"image_size": image_size,
|
||||
},
|
||||
)
|
||||
|
||||
assert schema.data.shape == (1, 3, 1024, 1024)
|
||||
assert schema.images_crop.shape[-1] == 640
|
||||
|
||||
def test_mismatched_image_size_raises(self, processor):
|
||||
"""Deliberately wrong image_size binding should still be caught
|
||||
by TensorSchema validation."""
|
||||
small_image = Image.new("RGB", (100, 100), color="green")
|
||||
|
||||
result = processor(
|
||||
prompt="<image>\nDescribe this image.",
|
||||
images=[small_image],
|
||||
)
|
||||
|
||||
pixel_values = result["pixel_values"]
|
||||
images_crop = result["images_crop"]
|
||||
images_spatial_crop = result["images_spatial_crop"]
|
||||
|
||||
with pytest.raises(ValueError, match="images_crop"):
|
||||
DeepseekOCRImagePixelInputs(
|
||||
type="pixel_values",
|
||||
data=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
resolve_bindings={
|
||||
"base_size": 1024,
|
||||
"image_size": 1024, # Wrong! Tensor has 640
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class TestDeepseekOCRInputValidation:
|
||||
"""Bare ``assert`` used to validate user input here returned HTTP 500
|
||||
(AssertionError) instead of HTTP 400 (ValueError) and disappeared under
|
||||
``python -O``, silently corrupting the tokenized sequence. See #53850.
|
||||
"""
|
||||
|
||||
def test_mismatched_image_count_raises_value_error(self, processor):
|
||||
"""Two ``<image>`` tokens with one image must raise ValueError, not
|
||||
AssertionError (which would yield HTTP 500 and, under ``-O``, silently
|
||||
drop the middle text segment)."""
|
||||
image = Image.new("RGB", (100, 100), color="red")
|
||||
with pytest.raises(ValueError, match="does not match number of images"):
|
||||
processor(prompt="<image> and <image>", images=[image])
|
||||
|
||||
def test_missing_prompt_raises_value_error(self, processor):
|
||||
image = Image.new("RGB", (100, 100), color="red")
|
||||
with pytest.raises(ValueError, match="prompt and images"):
|
||||
processor(prompt=None, images=[image])
|
||||
|
||||
def test_missing_images_raises_value_error(self, processor):
|
||||
with pytest.raises(ValueError, match="prompt and images"):
|
||||
processor(prompt="<image>\nDescribe this image.", images=None)
|
||||
|
||||
|
||||
def _balanced_normalized_pixels(
|
||||
batch: int = 1, channels: int = 3, height: int = 64, width: int = 64
|
||||
) -> torch.Tensor:
|
||||
"""Build a pixel tensor whose values cancel to an exact sum of 0.
|
||||
|
||||
Matches the normalized half-black / half-white case from the DeepSeek-OCR
|
||||
processor (Normalize(0.5, 0.5) maps 0→-1 and 255→+1).
|
||||
"""
|
||||
assert width % 2 == 0
|
||||
pixel_values = torch.ones(batch, channels, height, width)
|
||||
pixel_values[..., : width // 2] = -1.0
|
||||
assert pixel_values.sum().item() == 0.0
|
||||
return pixel_values
|
||||
|
||||
|
||||
class TestDeepseekOCRZeroSumPixelsAccepted:
|
||||
"""Zero-sum normalized pixels must still produce validated image inputs.
|
||||
|
||||
Returning None from ``_parse_and_validate_image_input`` makes
|
||||
``embed_multimodal`` return None and trips the worker encoder-output
|
||||
sanity check, killing EngineCore. The zero-sum shortcut is therefore
|
||||
removed; balanced images must parse like any other valid tensor.
|
||||
"""
|
||||
|
||||
def test_ocr_accepts_balanced_normalized_pixels(self):
|
||||
pixel_values = _balanced_normalized_pixels()
|
||||
images_crop = torch.zeros(0, 3, 40, 40)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
|
||||
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
|
||||
None,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
|
||||
assert image_input is not None
|
||||
assert image_input.data is pixel_values
|
||||
|
||||
def test_ocr2_accepts_balanced_normalized_pixels(self):
|
||||
pixel_values = _balanced_normalized_pixels()
|
||||
images_crop = torch.zeros(0, 3, 40, 40)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
model = SimpleNamespace(vision_config=SimpleNamespace(image_size=64))
|
||||
|
||||
image_input = DeepseekOCR2ForCausalLM._parse_and_validate_image_input(
|
||||
model,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
|
||||
assert image_input is not None
|
||||
assert image_input.data is pixel_values
|
||||
|
||||
def test_image_transform_half_black_half_white_sums_to_zero(self):
|
||||
"""A 1024×1024 half-black/half-white PNG through ImageTransform
|
||||
(ToTensor + Normalize(0.5, 0.5)) yields an exact zero sum, and parse
|
||||
must still accept it.
|
||||
"""
|
||||
image = Image.new("RGB", (1024, 1024), (0, 0, 0))
|
||||
image.paste(Image.new("RGB", (512, 1024), (255, 255, 255)), (512, 0))
|
||||
pixel_values = ImageTransform()(image).unsqueeze(0)
|
||||
assert pixel_values.sum().item() == 0.0
|
||||
|
||||
images_crop = torch.zeros(0, 3, 640, 640)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
|
||||
None,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
assert image_input is not None
|
||||
assert torch.equal(image_input.data, pixel_values)
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4 VL processing helpers."""
|
||||
|
||||
from PIL import Image
|
||||
from transformers import DeepseekV4Config
|
||||
|
||||
from vllm.models.deepseek_v4.common.mm_preprocess import (
|
||||
DeepseekV4VLProcessingInfo,
|
||||
DeepseekV4VLProcessor,
|
||||
build_image_block_pad_free,
|
||||
)
|
||||
|
||||
# Vision fields of deepseek-ai/DeepSeek-V4-Flash-Vision-Exp
|
||||
VISION_CONFIG = dict(
|
||||
vision_n_layers=24,
|
||||
vision_dim=1024,
|
||||
vision_n_heads=16,
|
||||
vision_inter_dim=2816,
|
||||
vision_patch_size=14,
|
||||
vision_rope_theta=10000.0,
|
||||
vision_downsample_ratio=3,
|
||||
vision_max_n_token=384,
|
||||
vision_min_pixels=147456,
|
||||
vision_max_wh_ratio=8,
|
||||
)
|
||||
|
||||
|
||||
class _ConfigCtx:
|
||||
def __init__(self, config: DeepseekV4Config) -> None:
|
||||
self._config = config
|
||||
|
||||
def get_hf_config(self, *args, **kwargs):
|
||||
return self._config
|
||||
|
||||
|
||||
def _feature_counts(
|
||||
processor: DeepseekV4VLProcessor, width: int, height: int
|
||||
) -> tuple[int, int]:
|
||||
out = processor(images=[Image.new("RGB", (width, height))])
|
||||
n_llm_h, n_llm_w = out["llm_grid"][0].tolist()
|
||||
types, _ = build_image_block_pad_free(n_llm_h, n_llm_w)
|
||||
return len(types), out["patches"].shape[0]
|
||||
|
||||
|
||||
def test_get_image_size_with_most_features_beats_wide_real_image():
|
||||
"""Dummy size must maximize tokens/patches within the budget (#59271).
|
||||
|
||||
A square under-counts because each aligner row pays an IMAGE_NEW_LINE.
|
||||
"""
|
||||
config = DeepseekV4Config(**VISION_CONFIG)
|
||||
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
|
||||
processor = DeepseekV4VLProcessor(config)
|
||||
|
||||
dummy = info.get_image_size_with_most_features()
|
||||
dummy_features = _feature_counts(processor, dummy.width, dummy.height)
|
||||
wide_features = _feature_counts(processor, 1932, 336)
|
||||
|
||||
assert dummy == (1932, 336)
|
||||
assert wide_features <= dummy_features
|
||||
|
||||
|
||||
def test_get_image_size_with_most_features_respects_max_wh_ratio():
|
||||
config = DeepseekV4Config(**VISION_CONFIG)
|
||||
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
|
||||
dummy = info.get_image_size_with_most_features()
|
||||
assert dummy.width / dummy.height <= config.vision_max_wh_ratio
|
||||
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.gemma3n_audio_utils import (
|
||||
adjust_audio_features_to_expected_length,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
# Gemma3 (image) model
|
||||
GEMMA3_MODEL_ID = "google/gemma-3-4b-it"
|
||||
|
||||
# Gemma3n (multimodal with audio) model
|
||||
GEMMA3N_MODEL_ID = "google/gemma-3n-E2B-it"
|
||||
|
||||
# Expected audio tokens for Gemma3n (audio_soft_tokens_per_image)
|
||||
GEMMA3N_EXPECTED_AUDIO_TOKENS = 188
|
||||
|
||||
|
||||
class TestGemma3nAudioTensorLogic:
|
||||
"""CPU-based tests for Gemma3n audio feature tensor manipulation.
|
||||
|
||||
These tests validate the padding/truncation logic in
|
||||
adjust_audio_features_to_expected_length() which fixes the
|
||||
integer overflow in _process_audio_input when audio_seq_len > 188.
|
||||
"""
|
||||
|
||||
def test_padding_when_audio_short(self):
|
||||
"""Test that short audio is padded to expected length."""
|
||||
batch_size, seq_len, embed_dim = 1, 100, 256
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
audio_features = torch.randn(batch_size, seq_len, embed_dim)
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
result, tokens_truncated = adjust_audio_features_to_expected_length(
|
||||
audio_features, expected_tokens, padding_embs
|
||||
)
|
||||
|
||||
assert result.shape == (batch_size, expected_tokens, embed_dim)
|
||||
assert tokens_truncated == 0
|
||||
# First 100 tokens should be original, rest should be padding (zeros)
|
||||
assert torch.allclose(result[:, :seq_len, :], audio_features)
|
||||
assert torch.allclose(
|
||||
result[:, seq_len:, :],
|
||||
torch.zeros(batch_size, expected_tokens - seq_len, embed_dim),
|
||||
)
|
||||
|
||||
def test_truncation_when_audio_long(self):
|
||||
"""Test that long audio is truncated to expected length.
|
||||
|
||||
This is the key test for the overflow fix. Previously, when
|
||||
audio_seq_len > expected_tokens, the code would compute a negative
|
||||
padding value causing: RuntimeError: numel: integer multiplication overflow
|
||||
"""
|
||||
batch_size, seq_len, embed_dim = 1, 192, 256 # 192 > 188
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
audio_features = torch.randn(batch_size, seq_len, embed_dim)
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
result, tokens_truncated = adjust_audio_features_to_expected_length(
|
||||
audio_features, expected_tokens, padding_embs
|
||||
)
|
||||
|
||||
assert result.shape == (batch_size, expected_tokens, embed_dim)
|
||||
assert tokens_truncated == seq_len - expected_tokens # 192 - 188 = 4
|
||||
# Result should be first 188 tokens of original
|
||||
assert torch.allclose(result, audio_features[:, :expected_tokens, :])
|
||||
|
||||
def test_no_change_when_exact_length(self):
|
||||
"""Test that exact-length audio passes through unchanged."""
|
||||
batch_size, embed_dim = 1, 256
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
audio_features = torch.randn(batch_size, expected_tokens, embed_dim)
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
result, tokens_truncated = adjust_audio_features_to_expected_length(
|
||||
audio_features, expected_tokens, padding_embs
|
||||
)
|
||||
|
||||
assert result.shape == audio_features.shape
|
||||
assert tokens_truncated == 0
|
||||
assert torch.allclose(result, audio_features)
|
||||
|
||||
def test_original_bug_would_fail(self):
|
||||
"""Verify the original buggy implementation would cause overflow.
|
||||
|
||||
The original code always tried to pad, which fails when
|
||||
audio_seq_len > expected_tokens because expand() gets negative size.
|
||||
"""
|
||||
batch_size, seq_len, embed_dim = 1, 192, 256
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
# Original buggy logic (always pads, never truncates)
|
||||
extra_padding_tokens = expected_tokens - seq_len # = -4 (negative!)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
# This should fail with negative size error
|
||||
padding_embs.expand(batch_size, extra_padding_tokens, embed_dim)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_len",
|
||||
[50, 100, 150, 187, 188, 189, 192, 200, 300],
|
||||
)
|
||||
def test_various_audio_lengths(self, seq_len: int):
|
||||
"""Test padding/truncation with various audio lengths."""
|
||||
batch_size, embed_dim = 1, 256
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
audio_features = torch.randn(batch_size, seq_len, embed_dim)
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
# Should not raise any errors
|
||||
result, tokens_truncated = adjust_audio_features_to_expected_length(
|
||||
audio_features, expected_tokens, padding_embs
|
||||
)
|
||||
|
||||
# Output should always be expected_tokens length
|
||||
assert result.shape == (batch_size, expected_tokens, embed_dim)
|
||||
|
||||
# Verify truncation count is correct
|
||||
if seq_len > expected_tokens:
|
||||
assert tokens_truncated == seq_len - expected_tokens
|
||||
else:
|
||||
assert tokens_truncated == 0
|
||||
|
||||
def test_batch_processing(self):
|
||||
"""Test that batch processing works correctly."""
|
||||
batch_size, seq_len, embed_dim = 4, 192, 256
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
audio_features = torch.randn(batch_size, seq_len, embed_dim)
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
result, tokens_truncated = adjust_audio_features_to_expected_length(
|
||||
audio_features, expected_tokens, padding_embs
|
||||
)
|
||||
|
||||
assert result.shape == (batch_size, expected_tokens, embed_dim)
|
||||
assert tokens_truncated == seq_len - expected_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA3_MODEL_ID])
|
||||
@pytest.mark.parametrize("mm_processor_kwargs", [{}])
|
||||
def test_get_image_size_with_most_features(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"do_pan_and_scan": True},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
|
||||
|
||||
max_image_size = processor.info.get_image_size_with_most_features()
|
||||
max_tokens = processor.info.get_num_image_tokens(
|
||||
image_width=max_image_size.width,
|
||||
image_height=max_image_size.height,
|
||||
processor=hf_processor,
|
||||
mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
prompt = "<start_of_image>"
|
||||
image_seq_length = hf_processor.image_seq_length
|
||||
|
||||
for asset in image_assets:
|
||||
mm_data = {"image": [asset.pil_image]}
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
mm_kwargs_data = processed_inputs["mm_kwargs"].get_data()
|
||||
num_patches_tensor = mm_kwargs_data["num_patches"]
|
||||
tokens = int(num_patches_tensor.item()) * image_seq_length
|
||||
assert tokens <= max_tokens
|
||||
@@ -1,405 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image as PILImage
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.model_executor.models.gemma4_mm import (
|
||||
Gemma4ForConditionalGeneration,
|
||||
Gemma4ImagePixelInputs,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
# TODO: to be updated to "google/gemma-4-e2b-it" once the models are available
|
||||
GEMMA4_MODEL_ID = "google/gemma-4-E2B-it"
|
||||
|
||||
|
||||
def test_gemma4_image_schema_accepts_variable_patch_counts():
|
||||
Gemma4ImagePixelInputs(
|
||||
pixel_values=[
|
||||
torch.randn(10080, 768),
|
||||
torch.randn(2520, 768),
|
||||
],
|
||||
pixel_position_ids=[
|
||||
torch.zeros(10080, 2, dtype=torch.long),
|
||||
torch.zeros(2520, 2, dtype=torch.long),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_gemma4_image_batching_keeps_variable_patch_counts_unstacked():
|
||||
field = MultiModalFieldConfig.batched("image").field
|
||||
elems = field.build_elems(
|
||||
"image",
|
||||
"pixel_values",
|
||||
[torch.randn(10080, 768), torch.randn(2520, 768)],
|
||||
)
|
||||
|
||||
reduced = field.reduce_data(list(elems))
|
||||
|
||||
assert isinstance(reduced, list)
|
||||
assert [tensor.shape for tensor in reduced] == [
|
||||
torch.Size([10080, 768]),
|
||||
torch.Size([2520, 768]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_width,image_height,max_soft_tokens",
|
||||
[
|
||||
# Production repro: a 3x900 image (extreme aspect ratio) made the
|
||||
# prompt-side estimator return 289 while the HF Gemma 4 image
|
||||
# processor's vision tower output capped at 280, producing the
|
||||
# "Attempted to assign 280 multimodal tokens to 289 placeholders"
|
||||
# mismatch that crashed EngineCore.
|
||||
(900, 3, 280),
|
||||
(3, 900, 280),
|
||||
# Same pathology should hold for the video-frame budget (70 tokens).
|
||||
(900, 3, 70),
|
||||
# And for any other supported budget.
|
||||
(4000, 2, 1120),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
|
||||
model_id: str,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
max_soft_tokens: int,
|
||||
):
|
||||
"""Regression for the Gemma 3/4 multimodal crash.
|
||||
|
||||
`_compute_num_soft_tokens` must never return a value larger than
|
||||
`max_soft_tokens`. The HF Gemma 4 image processor clamps its vision
|
||||
tower output to that value; if the prompt-side estimator returns more,
|
||||
the prompt has more `image` placeholder tokens than the encoder will
|
||||
fill, and `_merge_multimodal_embeddings` raises `ValueError` deep in
|
||||
the model forward.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"do_pan_and_scan": True},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
num_soft_tokens = processor.info._compute_num_soft_tokens(
|
||||
image_width=image_width,
|
||||
image_height=image_height,
|
||||
max_soft_tokens=max_soft_tokens,
|
||||
)
|
||||
|
||||
assert num_soft_tokens <= max_soft_tokens, (
|
||||
f"_compute_num_soft_tokens returned {num_soft_tokens} for "
|
||||
f"image_width={image_width}, image_height={image_height}, "
|
||||
f"max_soft_tokens={max_soft_tokens} — exceeds the cap that the HF "
|
||||
f"image processor enforces on its vision tower output. This is "
|
||||
f"the placeholder/encoder count mismatch that crashes EngineCore."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_image_tokens"),
|
||||
[
|
||||
({}, 280),
|
||||
({"max_soft_tokens": 70}, 70),
|
||||
({"max_soft_tokens": 280}, 280),
|
||||
({"max_soft_tokens": 1120}, 1120),
|
||||
({"images_kwargs": {"max_soft_tokens": 560}}, 560),
|
||||
({"images_kwargs": None}, 280),
|
||||
({"images_kwargs": "not-a-dict"}, 280),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
expected_image_tokens: int,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
tokens = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=ctx.model_config.max_model_len,
|
||||
mm_counts={"image": 1, "video": 1},
|
||||
)
|
||||
|
||||
assert tokens is not None
|
||||
assert tokens["image"] == expected_image_tokens
|
||||
assert tokens["video"] == 32 * (70 + 2 + 6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("limit_mm_per_prompt", "expected_video_tokens"),
|
||||
[
|
||||
({"video": 1}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
|
||||
model_id: str,
|
||||
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
|
||||
expected_video_tokens: int,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
tokens = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=ctx.model_config.max_model_len,
|
||||
mm_counts={"video": 1},
|
||||
)
|
||||
|
||||
assert tokens is not None
|
||||
assert tokens["image"] == 280
|
||||
assert tokens["video"] == expected_video_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
image = PILImage.new("RGB", (1000, 1000), color="white")
|
||||
image_size = image.size
|
||||
mm_items = processor.info.parse_mm_data({"image": image})
|
||||
|
||||
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
|
||||
replacement = prompt_update.resolve(0).content.full
|
||||
expected = processor.info.get_image_repl(
|
||||
image_width=image_size[0],
|
||||
image_height=image_size[1],
|
||||
processor=processor.info.get_hf_processor(),
|
||||
max_soft_tokens=560,
|
||||
).full
|
||||
|
||||
assert replacement == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
|
||||
)
|
||||
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
|
||||
def test_video_cache_is_independent_of_image_kwargs(
|
||||
model_id: str,
|
||||
image_kwargs: dict[str, object],
|
||||
video_uuid: str | None,
|
||||
):
|
||||
"""Request image overrides must not change video frames or make results
|
||||
depend on cache warmth.
|
||||
"""
|
||||
request_kwargs = {"images_kwargs": image_kwargs}
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
image = PILImage.new("RGB", (48, 48), color=(128, 128, 128))
|
||||
frames = np.stack([np.asarray(image)] * 2)
|
||||
metadata = {"fps": 2.0, "frames_indices": [0, 1]}
|
||||
mm_items = processor.info.parse_mm_data(
|
||||
{"image": image, "video": [(frames, metadata)]}
|
||||
)
|
||||
|
||||
def process(mm_kwargs, cache):
|
||||
return processor(
|
||||
hf_processor.image_token + hf_processor.video_token,
|
||||
mm_items,
|
||||
mm_uuid_items={"video": [video_uuid]},
|
||||
hf_processor_mm_kwargs=mm_kwargs,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
# Populate the video cache without image overrides, then change only
|
||||
# images_kwargs on the next request. Since images_kwargs are excluded from
|
||||
# the video hash, and must not affect the processed video, the cached result
|
||||
# must match a fresh recomputation.
|
||||
baseline = process({}, cache=cache)
|
||||
cached = process(request_kwargs, cache=cache)
|
||||
cache.clear_cache()
|
||||
fresh = process(request_kwargs, cache=cache)
|
||||
|
||||
def pixels(result, modality, field):
|
||||
return result["mm_kwargs"][modality][0][field].data
|
||||
|
||||
video_pixels = pixels(fresh, "video", "pixel_values_videos")
|
||||
assert torch.equal(pixels(cached, "video", "pixel_values_videos"), video_pixels)
|
||||
assert torch.equal(pixels(baseline, "video", "pixel_values_videos"), video_pixels)
|
||||
assert baseline["mm_hashes"]["video"] == fresh["mm_hashes"]["video"]
|
||||
assert not torch.equal(
|
||||
pixels(baseline, "image", "pixel_values"),
|
||||
pixels(fresh, "image", "pixel_values"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_limit_mm_per_prompt(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that limit_mm_per_prompt accurately restricts multiple images."""
|
||||
# We only allow 1 image
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Provide 2 images in the prompt
|
||||
prompt = "<image><image>"
|
||||
# image_assets usually has multiple images
|
||||
images = [asset.pil_image for asset in image_assets][:2]
|
||||
if len(images) < 2:
|
||||
images = [images[0], images[0]]
|
||||
|
||||
mm_data = {"image": images}
|
||||
|
||||
# Expect ValueError when exceeding limit
|
||||
with pytest.raises(VLLMValidationError, match="At most 1 image"):
|
||||
processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
# Regression guard for PR #43169 follow-up: the batched Gemma4 vision encoder
|
||||
# admitted ``chunk ~= 53`` on a 22 GiB L4 with a 26B AWQ model loaded,
|
||||
# allocating 2.43 GiB int64 inside
|
||||
# ``F.one_hot(num_classes=position_embedding_size)`` and OOMing because only
|
||||
# 2.41 GiB was actually free. The fix sizes ``chunk`` from currently-free GPU
|
||||
# memory and counts the ``F.one_hot`` transient as the dominant cost.
|
||||
|
||||
_encoder_chunk = Gemma4ForConditionalGeneration._encoder_chunk
|
||||
|
||||
# Gemma4 vision config default (HF: configuration_gemma4.py).
|
||||
_POSITION_EMBEDDING_SIZE = 10240
|
||||
# Video frame: max_soft_tokens=70, pooling_kernel_size=2 -> 70 * 4 patches.
|
||||
_VIDEO_PATCHES_PER_FRAME = 280
|
||||
|
||||
|
||||
def test_encoder_chunk_tight_budget_fits_in_free():
|
||||
free = 3 * GiB_bytes # L4 22 GiB after 26B AWQ load.
|
||||
total = 22 * GiB_bytes
|
||||
chunk = _encoder_chunk(
|
||||
_VIDEO_PATCHES_PER_FRAME, free, total, _POSITION_EMBEDDING_SIZE
|
||||
)
|
||||
one_hot_bytes = chunk * _VIDEO_PATCHES_PER_FRAME * 2 * _POSITION_EMBEDDING_SIZE * 8
|
||||
assert one_hot_bytes <= free // 2
|
||||
|
||||
|
||||
def test_encoder_chunk_roomy_gpu_keeps_batching():
|
||||
chunk = _encoder_chunk(
|
||||
_VIDEO_PATCHES_PER_FRAME,
|
||||
60 * GiB_bytes,
|
||||
80 * GiB_bytes,
|
||||
_POSITION_EMBEDDING_SIZE,
|
||||
)
|
||||
assert chunk > 8
|
||||
|
||||
|
||||
def test_encoder_chunk_zero_patches_is_safe():
|
||||
assert (
|
||||
_encoder_chunk(0, 60 * GiB_bytes, 80 * GiB_bytes, _POSITION_EMBEDDING_SIZE) == 1
|
||||
)
|
||||
|
||||
|
||||
def test_encoder_chunk_zero_position_embedding_size_is_safe():
|
||||
# Degenerate config: must not raise ZeroDivisionError.
|
||||
assert (
|
||||
_encoder_chunk(_VIDEO_PATCHES_PER_FRAME, 60 * GiB_bytes, 80 * GiB_bytes, 0) == 1
|
||||
)
|
||||
|
||||
|
||||
def test_encoder_chunk_no_free_memory_falls_back_to_one():
|
||||
assert (
|
||||
_encoder_chunk(
|
||||
_VIDEO_PATCHES_PER_FRAME, 0, 22 * GiB_bytes, _POSITION_EMBEDDING_SIZE
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def _gemma4_for_encoder_cudagraph() -> Gemma4ForConditionalGeneration:
|
||||
model = object.__new__(Gemma4ForConditionalGeneration)
|
||||
vision_config = SimpleNamespace(
|
||||
patch_size=16, pooling_kernel_size=3, default_output_length=280
|
||||
)
|
||||
model.vision_tower = SimpleNamespace(config=vision_config)
|
||||
model.config = SimpleNamespace(vision_config=vision_config)
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("token_budget", "row_tokens"), [(4096, 70), (4096, 280), (70, 280)]
|
||||
)
|
||||
def test_encoder_cudagraph_capture_rows_fit_token_budget(
|
||||
token_budget: int, row_tokens: int
|
||||
):
|
||||
# Regression for #61045: rows were max_frames_per_batch (58 * 32) at
|
||||
# 280 tokens each, so the 4096-token buffer alone was 6.69 GiB.
|
||||
model = _gemma4_for_encoder_cudagraph()
|
||||
inputs = model.prepare_encoder_cudagraph_capture_inputs(
|
||||
token_budget=token_budget,
|
||||
max_batch_size=58,
|
||||
max_frames_per_batch=58 * 32,
|
||||
device="meta",
|
||||
dtype=torch.bfloat16,
|
||||
axis_keys=(row_tokens,),
|
||||
)
|
||||
|
||||
num_rows, row_patches, _ = inputs.values["pixel_values"].shape
|
||||
assert row_patches == row_tokens * 9
|
||||
assert num_rows == max(1, token_budget // row_tokens)
|
||||
|
||||
|
||||
def test_encoder_cudagraph_mixed_width_images_pack_by_padded_width():
|
||||
model = _gemma4_for_encoder_cudagraph()
|
||||
mm_kwargs = {
|
||||
"pixel_values": [torch.zeros(2520, 768), torch.zeros(630, 768)],
|
||||
"pixel_position_ids": [
|
||||
torch.zeros(2520, 2, dtype=torch.long),
|
||||
torch.zeros(630, 2, dtype=torch.long),
|
||||
],
|
||||
}
|
||||
|
||||
specs = model.get_encoder_cudagraph_item_specs(mm_kwargs)
|
||||
assert [spec.output_tokens for spec in specs] == [280, 70]
|
||||
assert [spec.get_path_output_tokens("default") for spec in specs] == [280, 280]
|
||||
|
||||
selected = model.select_encoder_cudagraph_items(mm_kwargs, [1])
|
||||
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (70,)
|
||||
selected = model.select_encoder_cudagraph_items(mm_kwargs, [0, 1])
|
||||
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (280,)
|
||||
@@ -1,248 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image as PILImage
|
||||
from transformers.models.gemma4 import Gemma4AudioFeatureExtractor
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.model_executor.models.gemma4_mm import (
|
||||
Gemma4ImagePixelInputs,
|
||||
_dummy_audio_num_samples,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
# The Unified model ID for testing purposes
|
||||
GEMMA4_UNIFIED_MODEL_ID = "google/gemma-4-12B-it"
|
||||
|
||||
|
||||
def test_gemma4_unified_image_schema_accepts_variable_patch_counts():
|
||||
Gemma4ImagePixelInputs(
|
||||
pixel_values=[
|
||||
torch.randn(10080, 768),
|
||||
torch.randn(2520, 768),
|
||||
],
|
||||
pixel_position_ids=[
|
||||
torch.zeros(10080, 2, dtype=torch.long),
|
||||
torch.zeros(2520, 2, dtype=torch.long),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_gemma4_unified_image_batching_keeps_variable_patch_counts_unstacked():
|
||||
field = MultiModalFieldConfig.batched("image").field
|
||||
elems = field.build_elems(
|
||||
"image",
|
||||
"pixel_values",
|
||||
[torch.randn(10080, 768), torch.randn(2520, 768)],
|
||||
)
|
||||
|
||||
reduced = field.reduce_data(list(elems))
|
||||
|
||||
assert isinstance(reduced, list)
|
||||
assert [tensor.shape for tensor in reduced] == [
|
||||
torch.Size([10080, 768]),
|
||||
torch.Size([2520, 768]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_width,image_height,max_soft_tokens",
|
||||
[
|
||||
(900, 3, 280),
|
||||
(3, 900, 280),
|
||||
(900, 3, 70),
|
||||
(4000, 2, 1120),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
|
||||
def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
|
||||
model_id: str,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
max_soft_tokens: int,
|
||||
):
|
||||
"""Verify ``_compute_num_soft_tokens`` caps output at ``max_soft_tokens``."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"do_pan_and_scan": True},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
num_soft_tokens = processor.info._compute_num_soft_tokens(
|
||||
image_width=image_width,
|
||||
image_height=image_height,
|
||||
max_soft_tokens=max_soft_tokens,
|
||||
)
|
||||
|
||||
assert num_soft_tokens <= max_soft_tokens, (
|
||||
f"_compute_num_soft_tokens returned {num_soft_tokens} for "
|
||||
f"image_width={image_width}, image_height={image_height}, "
|
||||
f"max_soft_tokens={max_soft_tokens} — exceeds the cap."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_image_tokens"),
|
||||
[
|
||||
({}, 280),
|
||||
({"max_soft_tokens": 70}, 70),
|
||||
({"max_soft_tokens": 280}, 280),
|
||||
({"max_soft_tokens": 1120}, 1120),
|
||||
({"images_kwargs": {"max_soft_tokens": 560}}, 560),
|
||||
({"images_kwargs": None}, 280),
|
||||
({"images_kwargs": "not-a-dict"}, 280),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
|
||||
def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
expected_image_tokens: int,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
tokens = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=ctx.model_config.max_model_len,
|
||||
mm_counts={"image": 1, "video": 1},
|
||||
)
|
||||
|
||||
assert tokens is not None
|
||||
assert tokens["image"] == expected_image_tokens
|
||||
assert tokens["video"] == 32 * (70 + 2 + 6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("limit_mm_per_prompt", "expected_video_tokens"),
|
||||
[
|
||||
({"video": 1}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
|
||||
def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
|
||||
model_id: str,
|
||||
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
|
||||
expected_video_tokens: int,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
tokens = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=ctx.model_config.max_model_len,
|
||||
mm_counts={"video": 1},
|
||||
)
|
||||
|
||||
assert tokens is not None
|
||||
assert tokens["image"] == 280
|
||||
assert tokens["video"] == expected_video_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
|
||||
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
image = PILImage.new("RGB", (1000, 1000), color="white")
|
||||
image_size = image.size
|
||||
mm_items = processor.info.parse_mm_data({"image": image})
|
||||
|
||||
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
|
||||
replacement = prompt_update.resolve(0).content.full
|
||||
expected = processor.info.get_image_repl(
|
||||
image_width=image_size[0],
|
||||
image_height=image_size[1],
|
||||
processor=processor.info.get_hf_processor(),
|
||||
max_soft_tokens=560,
|
||||
).full
|
||||
|
||||
assert replacement == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
|
||||
def test_limit_mm_per_prompt(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that limit_mm_per_prompt restricts multiple images correctly."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<image><image>"
|
||||
images = [asset.pil_image for asset in image_assets][:2]
|
||||
if len(images) < 2:
|
||||
images = [images[0], images[0]]
|
||||
|
||||
mm_data = {"image": images}
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="At most 1 image"):
|
||||
processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
def test_dummy_audio_length_falls_back_for_the_unified_extractor():
|
||||
"""The unified audio feature extractor does not define ``fft_length``.
|
||||
|
||||
``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute
|
||||
unconditionally, so memory profiling raised ``AttributeError`` for every
|
||||
unified Gemma 4 deployment in which audio is the modality with the largest
|
||||
per-item token count -- which is the case as soon as ``audio_seq_length``
|
||||
is raised above the video budget.
|
||||
"""
|
||||
|
||||
class _UnifiedFeatureExtractor:
|
||||
sampling_rate = 16000
|
||||
|
||||
class _UnifiedProcessor:
|
||||
feature_extractor = _UnifiedFeatureExtractor()
|
||||
audio_seq_length = 3000
|
||||
audio_ms_per_token = 40
|
||||
|
||||
# 3000 tokens * 40 ms = 120 s at 16 kHz
|
||||
assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000
|
||||
|
||||
|
||||
def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor():
|
||||
"""The mel/tower extractor keeps its existing behaviour."""
|
||||
|
||||
class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor):
|
||||
def __init__(self): # noqa: D107 - bypass the real feature-extractor init
|
||||
self.fft_length = 512
|
||||
self.sampling_rate = 16000
|
||||
|
||||
class _TowerProcessor:
|
||||
feature_extractor = _TowerFeatureExtractor()
|
||||
audio_seq_length = 750
|
||||
audio_ms_per_token = 40
|
||||
|
||||
assert _dummy_audio_num_samples(_TowerProcessor()) == 512
|
||||
@@ -1,201 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.model_executor.models.glm4_1v import (
|
||||
Glm4vForConditionalGeneration,
|
||||
Glm4vProcessingInfo,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.multimodal.video import DynamicVideoBackend, VideoBackend
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"max_video_pixels",
|
||||
"max_tokens",
|
||||
"expected_num_frames",
|
||||
),
|
||||
[
|
||||
(47_040_000, 124_988, 11),
|
||||
(47_040_000, 30_000, 24),
|
||||
(100_352_000, 124_988, 21),
|
||||
(100_352_000, 30_000, 7),
|
||||
(100_352_000, 0, 1),
|
||||
],
|
||||
)
|
||||
def test_get_max_video_frames_matches_glm_resize(
|
||||
max_video_pixels: int,
|
||||
max_tokens: int,
|
||||
expected_num_frames: int,
|
||||
):
|
||||
info = Mock(spec=Glm4vProcessingInfo)
|
||||
info.get_image_size_with_most_features.return_value = (2184, 2184)
|
||||
info._get_video_max_pixels.return_value = max_video_pixels
|
||||
vision_config = info.get_hf_config.return_value.vision_config
|
||||
vision_config.patch_size = 14
|
||||
vision_config.spatial_merge_size = 2
|
||||
vision_config.temporal_patch_size = 2
|
||||
info._get_vision_info.side_effect = lambda **kwargs: (
|
||||
Glm4vProcessingInfo._get_vision_info(info, **kwargs)
|
||||
)
|
||||
|
||||
num_frames = Glm4vProcessingInfo._get_max_video_frames(
|
||||
info,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
|
||||
assert num_frames == expected_num_frames
|
||||
assert info._get_video_max_pixels.call_count == 1
|
||||
assert info._get_vision_info.call_count == 600
|
||||
|
||||
|
||||
def test_encoder_cudagraph_uses_model_video_frame_limit():
|
||||
model = Mock()
|
||||
|
||||
assert Glm4vForConditionalGeneration.get_max_frames_per_video(model) == 600
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("temporal_patch_size", "expected_grid_t"),
|
||||
[(2, 9), (4, 5), (8, 3)],
|
||||
)
|
||||
def test_vision_info_rounds_up_temporal_frames(
|
||||
temporal_patch_size: int,
|
||||
expected_grid_t: int,
|
||||
):
|
||||
info = Mock(spec=Glm4vProcessingInfo)
|
||||
vision_config = info.get_hf_config.return_value.vision_config
|
||||
vision_config.patch_size = 14
|
||||
vision_config.spatial_merge_size = 2
|
||||
vision_config.temporal_patch_size = temporal_patch_size
|
||||
|
||||
_, num_vision_tokens = Glm4vProcessingInfo._get_vision_info(
|
||||
info,
|
||||
image_width=28,
|
||||
image_height=28,
|
||||
num_frames=17,
|
||||
do_resize=False,
|
||||
)
|
||||
|
||||
assert num_vision_tokens == expected_grid_t
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
|
||||
@pytest.mark.parametrize("expected_toks_per_frame", [299])
|
||||
@pytest.mark.parametrize(
|
||||
"num_frames, fps, expected_grid_t",
|
||||
[
|
||||
# pre-sampled fixed frames (unexpected behavior,
|
||||
# but we still expect it to work without errors)
|
||||
(32, 1, 16),
|
||||
(32, 2, 16),
|
||||
(128, 1, 64),
|
||||
(128, 2, 64),
|
||||
# post-sampled frames (expected behavior)
|
||||
(-1, 1, 5),
|
||||
(-1, 2, 10),
|
||||
],
|
||||
)
|
||||
def test_processor_override(
|
||||
model_id: str,
|
||||
expected_toks_per_frame: int,
|
||||
expected_grid_t: int,
|
||||
fps: int,
|
||||
num_frames: int,
|
||||
):
|
||||
"""Ensure GLM4vMultiModalProcessor can handle video frames properly."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {"fps": fps}
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
video_assets = VideoAsset(name="baby_reading", num_frames=num_frames)
|
||||
prompt = "<|begin_of_video|><|video|><|end_of_video|>"
|
||||
|
||||
video, metadata = video_assets.np_ndarrays, video_assets.metadata
|
||||
metadata["fps"] = fps
|
||||
mm_data = {"video": [(video, metadata)]}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token_id = hf_processor.image_token_id
|
||||
video_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
grid_t, _, _ = processed_inputs["mm_kwargs"].get_data()["video_grid_thw"][0]
|
||||
|
||||
assert grid_t == expected_grid_t
|
||||
assert video_tok_count == expected_toks_per_frame * grid_t
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
|
||||
@pytest.mark.parametrize("fps", [2])
|
||||
@pytest.mark.parametrize("backend", ["opencv"])
|
||||
def test_video_loader_consistency(
|
||||
model_id: str,
|
||||
fps: int,
|
||||
backend: str,
|
||||
):
|
||||
"""Ensure dynamic video loader (pre-sampled by loader) and normal video
|
||||
loader (post-sampled by processor) produce same video processing outputs.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=None,
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {"fps": fps}
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
prompt = "<|begin_of_video|><|video|><|end_of_video|>"
|
||||
|
||||
video_path = VideoAsset(name="baby_reading", num_frames=-1).video_path
|
||||
with open(video_path, "rb") as f:
|
||||
video_bytes = f.read()
|
||||
|
||||
static_video, static_metadata = VideoBackend.load_bytes(
|
||||
video_bytes, backend=backend
|
||||
)
|
||||
dynamic_video, dynamic_metadata = DynamicVideoBackend.load_bytes(
|
||||
video_bytes, fps=fps, backend=backend
|
||||
)
|
||||
|
||||
# pre-sampled loader shouldn't read all frames
|
||||
assert len(dynamic_video) < len(static_video)
|
||||
|
||||
static_mm_data = {"video": [(static_video, static_metadata)]}
|
||||
dynamic_mm_data = {"video": [(dynamic_video, dynamic_metadata)]}
|
||||
|
||||
static_outputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(static_mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
dynamic_outputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(dynamic_mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
assert static_outputs["prompt_token_ids"] == dynamic_outputs["prompt_token_ids"]
|
||||
assert batched_tensors_equal(
|
||||
static_outputs["mm_kwargs"].get_data(),
|
||||
dynamic_outputs["mm_kwargs"].get_data(),
|
||||
)
|
||||
@@ -1,95 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.llava_next_video import (
|
||||
LlavaNextVideoForConditionalGeneration,
|
||||
)
|
||||
from vllm.model_executor.models.vision import get_vision_encoder_info
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
class _StubModel:
|
||||
"""Carries only the attributes the token-count method reads from
|
||||
`self`, so the real methods can be exercised without constructing
|
||||
the full `nn.Module` (vision tower, language model, etc.)."""
|
||||
|
||||
patch_grid_length: int
|
||||
pooled_grid_length: int
|
||||
|
||||
|
||||
get_mm_lora_token_counts = (
|
||||
LlavaNextVideoForConditionalGeneration.get_mm_lora_token_counts
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/LLaVA-NeXT-Video-7B-hf"])
|
||||
def test_num_mm_tokens_match_real_config(model_id):
|
||||
"""The stored grid lengths must match what `__init__` derives from
|
||||
the real HF config, and the LoRA token counts must preserve the
|
||||
frame-level scaling exactly."""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"video": 1})
|
||||
config = ctx.model_config.hf_config
|
||||
|
||||
vision_encoder_info = get_vision_encoder_info(config)
|
||||
patch_grid_length = vision_encoder_info.get_patch_grid_length()
|
||||
pooled_grid_length = math.ceil(patch_grid_length / config.spatial_pool_stride)
|
||||
|
||||
stub = _StubModel()
|
||||
stub.patch_grid_length = patch_grid_length
|
||||
stub.pooled_grid_length = pooled_grid_length
|
||||
|
||||
for num_frames in (1, 2, 8, 16, 32):
|
||||
num_video_tokens = num_frames * pooled_grid_length**2
|
||||
|
||||
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=num_video_tokens,
|
||||
)
|
||||
assert encoder_tokens == num_frames * patch_grid_length**2
|
||||
assert connector_tokens == num_video_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("patch_grid_length", "pooled_grid_length", "num_frames"),
|
||||
[
|
||||
(24, 12, 1), # llava-hf/LLaVA-NeXT-Video-7B-hf: 336 / 14, stride 2
|
||||
(24, 12, 16),
|
||||
(27, 14, 5), # non-power-of-2 pooled grid (ceil rounding)
|
||||
(16, 8, 6),
|
||||
],
|
||||
)
|
||||
def test_num_mm_tokens_roundtrip(patch_grid_length, pooled_grid_length, num_frames):
|
||||
stub = _StubModel()
|
||||
stub.patch_grid_length = patch_grid_length
|
||||
stub.pooled_grid_length = pooled_grid_length
|
||||
|
||||
num_video_tokens = num_frames * pooled_grid_length**2
|
||||
|
||||
encoder_tokens, connector_tokens = get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=num_video_tokens,
|
||||
)
|
||||
assert encoder_tokens == num_frames * patch_grid_length**2
|
||||
assert connector_tokens == num_video_tokens
|
||||
|
||||
|
||||
def test_num_mm_tokens_zero():
|
||||
stub = _StubModel()
|
||||
stub.patch_grid_length = 24
|
||||
stub.pooled_grid_length = 12
|
||||
|
||||
assert get_mm_lora_token_counts(
|
||||
stub,
|
||||
modality="video",
|
||||
mm_kwargs=None,
|
||||
num_mm_embeds=0,
|
||||
) == (0, 0)
|
||||
@@ -1,302 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the LLaVA-OneVision-2 codec-video marker mechanism.
|
||||
|
||||
The codec video backend cannot be exercised end-to-end in CI because it
|
||||
relies on the model's ``trust_remote_code`` package and real video bytes.
|
||||
These tests cover the pure-Python marker plumbing that wraps a video *path*
|
||||
for vLLM's ``MultiModalDataParser``:
|
||||
|
||||
* :func:`prepare_codec_video_input` must encode a per-path hash into its
|
||||
dummy ndarray so distinct codec videos do not collide on ``mm_hash``
|
||||
(otherwise ``EncoderCacheManager`` would skip the encoder for every video
|
||||
after the first).
|
||||
* :func:`_extract_codec_video_paths` must round-trip the marker back to the
|
||||
original path(s) after the parser strips the metadata dict, and must return
|
||||
``None`` for non-codec inputs.
|
||||
"""
|
||||
|
||||
import types
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.models.llava_onevision2 import (
|
||||
_CODEC_VIDEO_MARKER,
|
||||
LlavaOnevision2VideoBackend,
|
||||
_extract_codec_video_paths,
|
||||
_frame_video_to_pil_and_timestamps,
|
||||
_validate_video_source,
|
||||
prepare_codec_video_input,
|
||||
)
|
||||
from vllm.multimodal.video import VideoSourceMetadata, VideoTargetMetadata
|
||||
|
||||
|
||||
def _model_config(local: str = "", domains=None):
|
||||
# Minimal stand-in for vLLM's ModelConfig: the validator only reads
|
||||
# ``allowed_local_media_path`` and ``allowed_media_domains``.
|
||||
return types.SimpleNamespace(
|
||||
allowed_local_media_path=local,
|
||||
allowed_media_domains=domains,
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_codec_video_input_shape_and_marker():
|
||||
dummy, meta = prepare_codec_video_input("/data/foo.mp4")
|
||||
|
||||
# 4-D ndarray satisfies MultiModalDataParser's video shape check.
|
||||
assert isinstance(dummy, np.ndarray)
|
||||
assert dummy.shape == (1, 1, 16, 3)
|
||||
assert dummy.dtype == np.uint8
|
||||
|
||||
# Metadata carries the exact path under the codec marker key.
|
||||
assert meta == {_CODEC_VIDEO_MARKER: "/data/foo.mp4"}
|
||||
|
||||
|
||||
def test_prepare_codec_video_input_is_deterministic():
|
||||
# Same path must yield identical dummy bytes (stable mm_hash).
|
||||
a, _ = prepare_codec_video_input("/data/foo.mp4")
|
||||
b, _ = prepare_codec_video_input("/data/foo.mp4")
|
||||
assert a.tobytes() == b.tobytes()
|
||||
|
||||
|
||||
def test_prepare_codec_video_input_distinct_paths_distinct_bytes():
|
||||
# Distinct paths must yield distinct dummy bytes so the parser-visible
|
||||
# ndarray (the only part reaching MultiModalHasher) varies per video.
|
||||
paths = ["/data/a.mp4", "/data/b.mp4", "/data/c.mp4", "/data/a_.mp4"]
|
||||
payloads = {prepare_codec_video_input(p)[0].tobytes() for p in paths}
|
||||
assert len(payloads) == len(paths)
|
||||
|
||||
|
||||
def test_extract_codec_video_paths_parser_list_shape():
|
||||
# Parser yields list-of-(ndarray, metadata-dict) for tuple inputs.
|
||||
items = [prepare_codec_video_input(p) for p in ("/x/1.mp4", "/x/2.mp4")]
|
||||
assert _extract_codec_video_paths(items) == ["/x/1.mp4", "/x/2.mp4"]
|
||||
|
||||
|
||||
def test_extract_codec_video_paths_single_raw_tuple():
|
||||
# Single raw (ndarray, dict) tuple (pre-parser path) is also accepted.
|
||||
item = prepare_codec_video_input("/x/solo.mp4")
|
||||
assert _extract_codec_video_paths(item) == ["/x/solo.mp4"]
|
||||
|
||||
|
||||
def test_extract_codec_video_paths_non_codec_returns_none():
|
||||
# Plain decoded-frame inputs (ndarray / list of ndarray) are not codec
|
||||
# markers and must fall through to the frame backend.
|
||||
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
|
||||
assert _extract_codec_video_paths(plain) is None
|
||||
assert _extract_codec_video_paths([plain, plain]) is None
|
||||
assert _extract_codec_video_paths([]) is None
|
||||
# Tuple without the marker key is ignored.
|
||||
assert _extract_codec_video_paths((plain, {"fps": 2.0})) is None
|
||||
|
||||
|
||||
def test_extract_codec_video_paths_mixed_batch_returns_none():
|
||||
# If any item in the batch lacks the marker, the whole batch is treated
|
||||
# as non-codec (the backend does not mix codec and frame videos).
|
||||
codec = prepare_codec_video_input("/x/1.mp4")
|
||||
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
|
||||
assert _extract_codec_video_paths([codec, plain]) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Media access controls (_validate_video_source)
|
||||
#
|
||||
# The codec backend keeps the raw path string alive past vLLM's
|
||||
# MultiModalDataParser and hands it to the trust-remote-code codec module,
|
||||
# which opens it directly (cv2/ffmpeg) outside vLLM's MediaConnector. So the
|
||||
# codec backend is restricted to *local files* confined to
|
||||
# --allowed-local-media-path; remote http(s)/data URLs are rejected (they must
|
||||
# use the frame backend, which rides the connector). The validator also returns
|
||||
# the *resolved* path so the codec module opens exactly what was validated
|
||||
# (no symlink-retarget / validate-vs-open gap).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_validate_http_url_rejected():
|
||||
# Codec backend is local-only: remote URLs bypass the connector's domain
|
||||
# and redirect controls, so they are rejected regardless of allowlist.
|
||||
with pytest.raises(ValueError):
|
||||
_validate_video_source("http://example.com/v.mp4", _model_config())
|
||||
|
||||
|
||||
def test_validate_https_url_rejected_even_when_host_allowlisted():
|
||||
with pytest.raises(ValueError):
|
||||
_validate_video_source(
|
||||
"https://good.com/v.mp4",
|
||||
_model_config(domains=["good.com"]),
|
||||
)
|
||||
|
||||
|
||||
def test_validate_data_url_rejected():
|
||||
# data: URLs are a remote/inline source for the frame backend, not codec.
|
||||
with pytest.raises(ValueError):
|
||||
_validate_video_source("data:video/mp4;base64,AAAA", _model_config())
|
||||
|
||||
|
||||
def test_validate_local_file_blocked_without_allowed_path():
|
||||
# Local file access is opt-in: without --allowed-local-media-path the
|
||||
# bare path is rejected.
|
||||
with pytest.raises(ValueError):
|
||||
_validate_video_source("/data/v.mp4", _model_config())
|
||||
|
||||
|
||||
def test_validate_local_file_allowed_inside_allowed_dir(tmp_path):
|
||||
f = tmp_path / "v.mp4"
|
||||
f.touch()
|
||||
assert _validate_video_source(str(f), _model_config(local=str(tmp_path))) == str(f)
|
||||
|
||||
|
||||
def test_validate_local_file_traversal_blocked():
|
||||
# Path traversal escaping the allowed root is rejected after resolution.
|
||||
with pytest.raises(ValueError):
|
||||
_validate_video_source(
|
||||
"/tmp/ov2/../../etc/passwd",
|
||||
_model_config(local="/tmp/ov2"),
|
||||
)
|
||||
|
||||
|
||||
def test_validate_file_scheme_allowed_inside_allowed_dir(tmp_path):
|
||||
f = tmp_path / "v.mp4"
|
||||
f.touch()
|
||||
assert _validate_video_source(
|
||||
f.as_uri(), _model_config(local=str(tmp_path))
|
||||
) == str(f)
|
||||
|
||||
|
||||
def test_validate_file_scheme_percent_encoded_traversal_blocked():
|
||||
# ``%2e%2e`` decodes to ``..``; the confinement check URL-decodes first
|
||||
# (url2pathname) so the path cannot stay literally under the allowed root.
|
||||
with pytest.raises(ValueError):
|
||||
_validate_video_source(
|
||||
"file:///tmp/ov2/%2e%2e/%2e%2e/etc/passwd",
|
||||
_model_config(local="/tmp/ov2"),
|
||||
)
|
||||
|
||||
|
||||
def test_validate_unsupported_scheme_blocked():
|
||||
with pytest.raises(ValueError):
|
||||
_validate_video_source("ftp://example.com/v.mp4", _model_config())
|
||||
|
||||
|
||||
def test_validate_returns_resolved_path_through_symlink(tmp_path):
|
||||
# The validator resolves symlinks and returns the *real* path, so the codec
|
||||
# module opens exactly what was validated (closes the validate-vs-open gap).
|
||||
real = tmp_path / "real.mp4"
|
||||
real.touch()
|
||||
link = tmp_path / "link.mp4"
|
||||
link.symlink_to(real)
|
||||
assert _validate_video_source(str(link), _model_config(local=str(tmp_path))) == str(
|
||||
real
|
||||
)
|
||||
|
||||
|
||||
def test_validate_relative_bare_path_blocked():
|
||||
# Bare (scheme=="") paths come only from the codec backend and must be
|
||||
# absolute: resolving a relative path against an ambiguous CWD before the
|
||||
# confinement check is brittle/unsafe, so it is rejected outright.
|
||||
with pytest.raises(ValueError):
|
||||
_validate_video_source("ov2/v.mp4", _model_config(local="/tmp/ov2"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
|
||||
#
|
||||
# Non-codec videos reach _apply_hf_processor_main as a ``(frames_ndarray, metadata)``
|
||||
# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
|
||||
# ``video_url`` inputs, or by the dummy-inputs builder during profiling
|
||||
# (``video_needs_metadata=True``). The helper materialises PIL frames and
|
||||
# per-frame timestamps (``frame_index / fps``), padding the frame count up to
|
||||
# the even temporal-merge boundary.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_frame_video_to_pil_and_timestamps_basic():
|
||||
frames = np.zeros((4, 8, 8, 3), dtype=np.uint8)
|
||||
metadata = {"fps": 2.0, "frames_indices": [0, 4, 8, 12]}
|
||||
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
|
||||
|
||||
assert len(pil_frames) == 4
|
||||
assert all(f.size == (8, 8) for f in pil_frames)
|
||||
# timestamps = frame_index / fps
|
||||
assert timestamps == [0.0, 2.0, 4.0, 6.0]
|
||||
|
||||
|
||||
def test_frame_video_to_pil_and_timestamps_even_pads_odd_frame_count():
|
||||
# Odd frame count -> last frame repeated to satisfy temporal merge=2.
|
||||
frames = np.zeros((3, 8, 8, 3), dtype=np.uint8)
|
||||
metadata = {"fps": 1.0, "frames_indices": [0, 1, 2]}
|
||||
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, metadata))
|
||||
|
||||
assert len(pil_frames) == 4
|
||||
assert len(timestamps) == 4
|
||||
# The padded frame reuses the final index/timestamp.
|
||||
assert timestamps == [0.0, 1.0, 2.0, 2.0]
|
||||
|
||||
|
||||
def test_frame_video_to_pil_and_timestamps_defaults_when_metadata_sparse():
|
||||
# Missing frames_indices -> sequential range; missing/zero fps -> default.
|
||||
frames = np.zeros((2, 8, 8, 3), dtype=np.uint8)
|
||||
pil_frames, timestamps = _frame_video_to_pil_and_timestamps((frames, {}))
|
||||
|
||||
assert len(pil_frames) == 2
|
||||
# Default fps is 1.0, indices fall back to range(T).
|
||||
assert timestamps == [0.0, 1.0]
|
||||
|
||||
|
||||
def test_frame_video_to_pil_and_timestamps_rejects_non_tuple():
|
||||
# Bare arrays (no metadata) must be rejected: the frame backend requires
|
||||
# the (frames, metadata) tuple produced by the registered loader.
|
||||
plain = np.zeros((4, 8, 8, 3), dtype=np.uint8)
|
||||
with pytest.raises(ValueError):
|
||||
_frame_video_to_pil_and_timestamps(plain)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LlavaOnevision2VideoBackend.compute_frames_index_to_sample honors the caller
|
||||
# supplied VideoTargetMetadata (passed via --media-io-kwargs) so benchmarks can
|
||||
# override the conservative defaults (fps=1.0, max_frames=32). Unset target
|
||||
# fields (sentinel <= 0) fall back to those OV2 hf-chat reference constants.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _src(total_frames: int, fps: float) -> VideoSourceMetadata:
|
||||
return VideoSourceMetadata(
|
||||
total_frames_num=total_frames,
|
||||
original_fps=fps,
|
||||
duration=total_frames / fps if fps > 0 else 0.0,
|
||||
)
|
||||
|
||||
|
||||
def test_backend_defaults_cap_at_32_frames():
|
||||
# 300 frames @ 1fps source, target unset -> capped at default max_frames=32.
|
||||
src = _src(300, 1.0)
|
||||
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=300.0)
|
||||
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
|
||||
|
||||
assert len(idx) == 32
|
||||
assert idx[0] == 0
|
||||
assert idx[-1] == 299
|
||||
assert len(idx) % 2 == 0
|
||||
|
||||
|
||||
def test_backend_target_num_frames_overrides_default_cap():
|
||||
# VSI-Bench parity: target.num_frames=128 must lift the 32-frame cap.
|
||||
src = _src(300, 1.0)
|
||||
target = VideoTargetMetadata(num_frames=128, fps=-1, max_duration=300.0)
|
||||
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
|
||||
|
||||
assert len(idx) == 128
|
||||
assert idx[0] == 0
|
||||
assert idx[-1] == 299
|
||||
|
||||
|
||||
def test_backend_target_fps_controls_sampling_when_below_cap():
|
||||
# 60 frames @ 30fps (2s) with target fps=1 -> ~2 frames (even-padded).
|
||||
src = _src(60, 30.0)
|
||||
target = VideoTargetMetadata(num_frames=128, fps=1.0, max_duration=300.0)
|
||||
idx = LlavaOnevision2VideoBackend.compute_frames_index_to_sample(src, target)
|
||||
|
||||
# fps-derived nframes (2) is below the 128 cap, so fps wins.
|
||||
assert len(idx) <= 8
|
||||
assert len(idx) % 2 == 0
|
||||
@@ -1,330 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Mistral3's multimodal preprocessing kwargs."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from transformers import AutoProcessor, BatchFeature
|
||||
from transformers.models.pixtral import PixtralProcessor
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
FusedMMInputNorm,
|
||||
IdentityInputNorm,
|
||||
build_mm_input_norm,
|
||||
)
|
||||
from vllm.model_executor.models.lightonocr import (
|
||||
LightOnOCRForConditionalGeneration,
|
||||
LightOnOCRProcessingInfo,
|
||||
)
|
||||
from vllm.model_executor.models.mistral3 import Mistral3HFEncoderInfo
|
||||
from vllm.model_executor.models.pixtral import (
|
||||
PixtralHFEncoderInfo,
|
||||
pixtral_patch_embed,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItems
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
# This repo ships both params.json (Mistral) and config.json (HF). Auto config
|
||||
# selects PixtralForConditionalGeneration; force HF to exercise Mistral3.
|
||||
_MODEL_CONFIG_KWARGS = {"config_format": "hf"}
|
||||
_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
|
||||
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
|
||||
|
||||
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
|
||||
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
|
||||
_RGB_RESCALE = 1.0 / 255.0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.skipif(
|
||||
not HAS_TRITON or not current_platform.is_cuda(), reason="CUDA Triton kernel"
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("normalize", [False, True])
|
||||
@pytest.mark.parametrize(("channels", "patch_shape"), [(3, (4, 4)), (2, (2, 4))])
|
||||
def test_pixtral_patch_embed_preserves_image_order_and_strides(
|
||||
dtype: torch.dtype,
|
||||
normalize: bool,
|
||||
channels: int,
|
||||
patch_shape: tuple[int, int],
|
||||
):
|
||||
"""Packed projection matches CHW normalization and patch convolution."""
|
||||
device = torch.device(current_platform.device_type)
|
||||
norm = (
|
||||
FusedMMInputNorm(
|
||||
_RGB_MEAN[:channels],
|
||||
_RGB_STD[:channels],
|
||||
_RGB_RESCALE,
|
||||
channel=channels,
|
||||
).to(device)
|
||||
if normalize
|
||||
else IdentityInputNorm()
|
||||
)
|
||||
weight = torch.randn(32, channels, *patch_shape, device=device, dtype=dtype)
|
||||
images = [
|
||||
torch.randint(0, 256, (channels, 17, 18), device=device, dtype=torch.uint8),
|
||||
torch.randint(0, 256, (channels, 25, 17), device=device, dtype=torch.uint8)[
|
||||
:, 1:17, 1:17
|
||||
],
|
||||
torch.randint(0, 256, (channels, 19, 20), device=device, dtype=torch.uint8)[
|
||||
:, 1:17, 2:18
|
||||
],
|
||||
]
|
||||
reference = []
|
||||
for image in images:
|
||||
if normalize:
|
||||
normalized = (
|
||||
image.float() * norm.weight[:, None, None] + norm.bias[:, None, None]
|
||||
)
|
||||
else:
|
||||
normalized = image
|
||||
reference.append(
|
||||
F.conv2d(normalized.to(dtype).unsqueeze(0), weight, stride=patch_shape)
|
||||
)
|
||||
packed, actual = pixtral_patch_embed(images, weight, norm)
|
||||
for expected, result in zip(reference, actual):
|
||||
torch.testing.assert_close(result, expected, rtol=0.01, atol=0.02)
|
||||
expected_packed = torch.cat(
|
||||
[result.flatten(2).transpose(1, 2) for result in reference], dim=1
|
||||
)
|
||||
torch.testing.assert_close(packed, expected_packed, rtol=0.01, atol=0.02)
|
||||
|
||||
|
||||
def _process_images_with_hf(
|
||||
hf_processor,
|
||||
images: list[Image.Image],
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
) -> tuple[list[torch.Tensor], list[int]]:
|
||||
"""Process images and placeholders through the public HF processor."""
|
||||
hf_out = hf_processor(
|
||||
text=hf_processor.image_token * len(images),
|
||||
images=images,
|
||||
return_tensors="pt",
|
||||
**mm_processor_kwargs,
|
||||
)
|
||||
pixel_values = hf_out["pixel_values"]
|
||||
image_sizes = hf_out["image_sizes"]
|
||||
unpadded = [p[:, :h, :w] for p, (h, w) in zip(pixel_values, image_sizes)]
|
||||
|
||||
special_token_ids = {
|
||||
hf_processor.image_token_id,
|
||||
hf_processor.image_break_token_id,
|
||||
hf_processor.image_end_token_id,
|
||||
}
|
||||
placeholder_tokens = [
|
||||
token_id
|
||||
for token_id in hf_out["input_ids"][0].tolist()
|
||||
if token_id in special_token_ids
|
||||
]
|
||||
return unpadded, placeholder_tokens
|
||||
|
||||
|
||||
def _placeholder_tokens_from_prompt_updates(
|
||||
processor,
|
||||
images: list[Image.Image],
|
||||
pixel_values: list[torch.Tensor],
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
) -> list[int]:
|
||||
hf_inputs = BatchFeature({"pixel_values": pixel_values})
|
||||
fields_config = processor._get_mm_fields_config(hf_inputs, mm_processor_kwargs)
|
||||
out_mm_kwargs = MultiModalKwargsItems.from_hf_inputs(hf_inputs, fields_config)
|
||||
# Prompt updates use raw PIL sizes and must predict the processed grid.
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
updates = processor._get_prompt_updates(
|
||||
mm_items, mm_processor_kwargs, out_mm_kwargs
|
||||
)
|
||||
placeholder_tokens: list[int] = []
|
||||
for item_idx in range(len(images)):
|
||||
details = updates[0].resolve(item_idx).content
|
||||
placeholder_tokens.extend(details.full)
|
||||
return placeholder_tokens
|
||||
|
||||
|
||||
def _expected_placeholder_tokens_per_image(
|
||||
hf_processor,
|
||||
pixel_values: torch.Tensor,
|
||||
) -> int:
|
||||
"""Count projected tokens from the actual HF-processed H×W."""
|
||||
image_h, image_w = pixel_values.shape[-2:]
|
||||
patch_size = hf_processor.image_processor.patch_size
|
||||
if isinstance(patch_size, dict):
|
||||
patch_h = patch_size["height"]
|
||||
patch_w = patch_size["width"]
|
||||
else:
|
||||
patch_h = patch_w = int(patch_size)
|
||||
|
||||
spatial_merge_size = getattr(hf_processor, "spatial_merge_size", 1)
|
||||
merged_patch_h = patch_h * spatial_merge_size
|
||||
merged_patch_w = patch_w * spatial_merge_size
|
||||
assert image_h % merged_patch_h == 0
|
||||
assert image_w % merged_patch_w == 0
|
||||
|
||||
return (image_h // merged_patch_h) * (image_w // merged_patch_w)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [_MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "image_size", "expected_toks_per_img"),
|
||||
[
|
||||
({}, (448, 448), 256),
|
||||
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
|
||||
(
|
||||
{"images_kwargs": {"size": {"longest_edge": 1008}}},
|
||||
(1540, 1540),
|
||||
1296,
|
||||
),
|
||||
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
|
||||
({"size": {"longest_edge": 1008}}, (29, 29), 4),
|
||||
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_size_override(
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
image_size: tuple[int, int],
|
||||
expected_toks_per_img: int,
|
||||
num_imgs: int,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
vllm_hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
assert isinstance(vllm_hf_processor, PixtralProcessor)
|
||||
|
||||
hf_processor = AutoProcessor.from_pretrained(
|
||||
model_id,
|
||||
fix_mistral_regex=True,
|
||||
)
|
||||
|
||||
dummy_image = Image.new("RGB", image_size, color=(127, 127, 127))
|
||||
images = [dummy_image] * num_imgs
|
||||
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
|
||||
hf_processor, images, merged_mm_kwargs
|
||||
)
|
||||
|
||||
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
|
||||
processor, images, pixel_values, hf_processor_mm_kwargs
|
||||
)
|
||||
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
|
||||
hf_processor, pixel_values[0]
|
||||
)
|
||||
assert expected_from_pixel_values == expected_toks_per_img
|
||||
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == (
|
||||
expected_from_pixel_values * num_imgs
|
||||
)
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_mm_device_do_normalize():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
images = [
|
||||
Image.new("RGB", (31, 47), color=(17, 89, 231)),
|
||||
Image.new("RGB", (48, 32), color=(201, 13, 127)),
|
||||
]
|
||||
prompt = [processor.info.get_hf_config().image_token_index] * len(images)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
|
||||
normalized_inputs = processor(prompt, mm_items=mm_items)
|
||||
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
raw_inputs = processor(prompt, mm_items=mm_items)
|
||||
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
assert all(value.dtype == torch.uint8 for value in raw_values)
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config)
|
||||
patch_size = processor.info.get_hf_config().vision_config.patch_size
|
||||
|
||||
def pack_patches(image: torch.Tensor) -> torch.Tensor:
|
||||
channels, height, width = image.shape
|
||||
rows = height // patch_size
|
||||
cols = width // patch_size
|
||||
return (
|
||||
image[:, : rows * patch_size, : cols * patch_size]
|
||||
.reshape(channels, rows, patch_size, cols, patch_size)
|
||||
.permute(1, 3, 0, 2, 4)
|
||||
.reshape(-1, channels * patch_size**2)
|
||||
)
|
||||
|
||||
for raw, normalized in zip(raw_values, normalized_values):
|
||||
output = input_norm(pack_patches(raw), normalized.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, pack_patches(normalized), rtol=1e-5, atol=1e-6
|
||||
)
|
||||
|
||||
|
||||
def test_scoped_request_size_overrides_configured_flat_size():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
mm_processor_kwargs={"size": {"longest_edge": 1008}},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = AutoProcessor.from_pretrained(
|
||||
_MODEL_ID,
|
||||
fix_mistral_regex=True,
|
||||
)
|
||||
|
||||
hf_processor_mm_kwargs: dict[str, object] = {
|
||||
"images_kwargs": {"size": {"longest_edge": 1288}}
|
||||
}
|
||||
image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
|
||||
|
||||
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
|
||||
hf_processor, [image], merged_mm_kwargs
|
||||
)
|
||||
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
|
||||
processor, [image], pixel_values, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
|
||||
hf_processor, pixel_values[0]
|
||||
)
|
||||
assert expected_from_pixel_values == 1656
|
||||
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
def test_lightonocr_keeps_vision_config_image_size():
|
||||
assert not LightOnOCRForConditionalGeneration.supports_mm_device_do_normalize
|
||||
|
||||
ctx = build_model_context(
|
||||
_LIGHTON_MODEL_ID,
|
||||
mm_processor_kwargs={"size": {"longest_edge": 1008}},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
assert isinstance(processor.info, LightOnOCRProcessingInfo)
|
||||
encoder_info = processor.info.get_vision_encoder_info()
|
||||
assert isinstance(encoder_info, PixtralHFEncoderInfo)
|
||||
assert not isinstance(encoder_info, Mistral3HFEncoderInfo)
|
||||
assert encoder_info.get_image_size() == (
|
||||
processor.info.get_hf_config().vision_config.image_size
|
||||
)
|
||||
@@ -1,92 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions
|
||||
from vllm.model_executor.models.molmo2 import (
|
||||
Molmo2DummyInputsBuilder,
|
||||
build_flat_image_bool_length,
|
||||
)
|
||||
|
||||
|
||||
def test_build_flat_image_bool_length_matches_molmoweb_processor_tokens():
|
||||
hf_config = SimpleNamespace(
|
||||
image_patch_id=151938,
|
||||
low_res_image_start_token_id=151940,
|
||||
image_start_token_id=151936,
|
||||
image_col_id=151939,
|
||||
image_end_token_id=151937,
|
||||
)
|
||||
image_grids = torch.tensor([[14, 14, 14, 23]], dtype=torch.long)
|
||||
|
||||
image_tokens, num_image_tokens = build_flat_image_bool_length(
|
||||
image_grids,
|
||||
hf_config,
|
||||
image_use_col_tokens=True,
|
||||
use_single_crop_col_tokens=None,
|
||||
use_single_crop_start_token=False,
|
||||
)
|
||||
|
||||
assert num_image_tokens.tolist() == [550]
|
||||
assert len(image_tokens) == 550
|
||||
assert image_tokens[0].item() == hf_config.image_start_token_id
|
||||
assert (image_tokens == hf_config.image_col_id).sum().item() == 28
|
||||
|
||||
|
||||
def test_build_flat_image_bool_length_respects_disabled_col_tokens():
|
||||
hf_config = SimpleNamespace(
|
||||
image_patch_id=151938,
|
||||
low_res_image_start_token_id=151940,
|
||||
image_start_token_id=151936,
|
||||
image_col_id=151939,
|
||||
image_end_token_id=151937,
|
||||
)
|
||||
image_grids = torch.tensor([[2, 3, 5, 7]], dtype=torch.long)
|
||||
|
||||
image_tokens, num_image_tokens = build_flat_image_bool_length(
|
||||
image_grids,
|
||||
hf_config,
|
||||
image_use_col_tokens=False,
|
||||
use_single_crop_col_tokens=False,
|
||||
use_single_crop_start_token=True,
|
||||
)
|
||||
|
||||
assert num_image_tokens.tolist() == [45]
|
||||
assert len(image_tokens) == 45
|
||||
assert image_tokens[0].item() == hf_config.low_res_image_start_token_id
|
||||
assert (image_tokens == hf_config.image_col_id).sum().item() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_frames_override", "expected_frames"),
|
||||
[(1, 2), (2, 2), (3, 3)],
|
||||
)
|
||||
def test_dummy_video_num_frames_override_honors_min_of_two(
|
||||
num_frames_override, expected_frames
|
||||
):
|
||||
"""A ``num_frames`` override below 2 must be ignored (the model needs at
|
||||
least 2 frames), matching the "cannot be less than 2, will be ignored"
|
||||
warning."""
|
||||
builder = object.__new__(Molmo2DummyInputsBuilder)
|
||||
builder.info = SimpleNamespace(
|
||||
get_hf_processor=lambda: SimpleNamespace(
|
||||
video_processor=SimpleNamespace(size={"width": 64, "height": 64})
|
||||
),
|
||||
get_num_frames_with_most_features=lambda seq_len, mm_counts: 16,
|
||||
get_image_size_with_most_features=lambda: (64, 64),
|
||||
)
|
||||
|
||||
data = builder.get_dummy_mm_data(
|
||||
seq_len=128,
|
||||
mm_counts={"image": 0, "video": 1},
|
||||
mm_options=MultiModalDummyOptions(
|
||||
video=VideoDummyOptions(num_frames=num_frames_override)
|
||||
),
|
||||
)
|
||||
|
||||
video, _metadata = data["video"][0]
|
||||
assert video.shape[0] == expected_frames
|
||||
@@ -1,647 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import Qwen3Config
|
||||
|
||||
from vllm.model_executor.models.interfaces import SupportsLoRA, supports_lora
|
||||
from vllm.model_executor.models.moss_audio import (
|
||||
MOSS_AUDIO_BOS_TOKEN,
|
||||
MOSS_AUDIO_BOS_TOKEN_ID,
|
||||
MOSS_AUDIO_EOS_TOKEN,
|
||||
MOSS_AUDIO_EOS_TOKEN_ID,
|
||||
MOSS_AUDIO_PLACEHOLDER,
|
||||
MOSS_AUDIO_TOKEN,
|
||||
MOSS_AUDIO_TOKEN_ID,
|
||||
MossAudioConfig,
|
||||
MossAudioDummyInputsBuilder,
|
||||
MossAudioEncoder,
|
||||
MossAudioEncoderConfig,
|
||||
MossAudioModel,
|
||||
MossAudioMultiModalProcessor,
|
||||
MossAudioProcessingInfo,
|
||||
MossAudioProcessor,
|
||||
MossQwen3ForCausalLM,
|
||||
MossQwen3Model,
|
||||
)
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
def encode(self, text, **kwargs):
|
||||
return [ord(char) for char in text]
|
||||
|
||||
def decode(self, token_ids, **kwargs):
|
||||
return "".join(chr(token_id) for token_id in token_ids)
|
||||
|
||||
def batch_decode(self, batch_token_ids, **kwargs):
|
||||
return [self.decode(token_ids, **kwargs) for token_ids in batch_token_ids]
|
||||
|
||||
|
||||
class _MMConfig:
|
||||
enable_mm_embeds = False
|
||||
mm_processor_cache_gb = 1
|
||||
mm_hasher_algorithm = "blake3"
|
||||
|
||||
def merge_mm_processor_kwargs(self, kwargs):
|
||||
return dict(kwargs)
|
||||
|
||||
def get_limit_per_prompt(self, modality):
|
||||
del modality
|
||||
return 3
|
||||
|
||||
|
||||
class _ModelConfig:
|
||||
def __init__(self):
|
||||
self.model = "OpenMOSS-Team/MOSS-Audio-4B-Instruct"
|
||||
self.revision = None
|
||||
self.max_model_len = 4096
|
||||
self.encoder_config = {}
|
||||
self.dtype = torch.float32
|
||||
self.hf_config = MossAudioConfig(language_config=Qwen3Config())
|
||||
self.multimodal_config = _MMConfig()
|
||||
|
||||
def get_multimodal_config(self):
|
||||
return self.multimodal_config
|
||||
|
||||
def get_inputs_embeds_size(self):
|
||||
return None
|
||||
|
||||
|
||||
class _ProcessingContext:
|
||||
def __init__(self):
|
||||
self.model_config = _ModelConfig()
|
||||
self.tokenizer = _Tokenizer()
|
||||
|
||||
def get_tokenizer(self):
|
||||
return self.tokenizer
|
||||
|
||||
def get_hf_config(self):
|
||||
return self.model_config.hf_config
|
||||
|
||||
def get_mm_config(self):
|
||||
return self.model_config.get_multimodal_config()
|
||||
|
||||
def get_merged_mm_kwargs(self, kwargs):
|
||||
return self.get_mm_config().merge_mm_processor_kwargs(kwargs)
|
||||
|
||||
def call_hf_processor(self, hf_processor, data, kwargs):
|
||||
merged_kwargs = self.get_merged_mm_kwargs(kwargs)
|
||||
merged_kwargs.setdefault("return_tensors", "pt")
|
||||
return hf_processor(**data, **merged_kwargs)
|
||||
|
||||
|
||||
class _TestMossAudioProcessingInfo(MossAudioProcessingInfo):
|
||||
def _get_processor_config_defaults(self):
|
||||
return {}
|
||||
|
||||
|
||||
def _vllm_config(tensor_parallel_size=1, pipeline_parallel_size=1, hf_config=None):
|
||||
if hf_config is None:
|
||||
hf_config = MossAudioConfig(language_config=Qwen3Config())
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
multimodal_config=None,
|
||||
),
|
||||
quant_config=None,
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class _FakeAudioEncoder:
|
||||
dtype = torch.float32
|
||||
|
||||
def __init__(self, deepstack_layers=0):
|
||||
self.deepstack_layers = deepstack_layers
|
||||
self.output_deepstack_hidden_states = None
|
||||
self.input_shape = None
|
||||
self.feature_lens = None
|
||||
|
||||
def __call__(self, audio_data, *, feature_lens, output_deepstack_hidden_states):
|
||||
self.input_shape = tuple(audio_data.shape)
|
||||
self.feature_lens = feature_lens.detach().cpu().clone()
|
||||
self.output_deepstack_hidden_states = output_deepstack_hidden_states
|
||||
lengths = MossAudioEncoder._compute_downsampled_length(feature_lens)
|
||||
hidden_states = torch.ones(1, int(lengths.sum().item()), 8)
|
||||
if not output_deepstack_hidden_states:
|
||||
return hidden_states, None
|
||||
return hidden_states, [
|
||||
hidden_states * scale for scale in range(2, 2 + self.deepstack_layers)
|
||||
]
|
||||
|
||||
|
||||
def _patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=1, tp_rank=0):
|
||||
import vllm.model_executor.layers.linear as linear_layers
|
||||
import vllm.model_executor.models.moss_audio as moss_audio_module
|
||||
import vllm.model_executor.parameter as parameter_module
|
||||
|
||||
for module in (moss_audio_module, linear_layers, parameter_module):
|
||||
monkeypatch.setattr(
|
||||
module, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
linear_layers, "get_tensor_model_parallel_rank", lambda: tp_rank
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
parameter_module, "get_tensor_model_parallel_rank", lambda: tp_rank
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
linear_layers, "tensor_model_parallel_all_reduce", lambda tensor: tensor
|
||||
)
|
||||
|
||||
|
||||
def _build_moss_audio_processor(cache=None):
|
||||
ctx = _ProcessingContext()
|
||||
info = _TestMossAudioProcessingInfo(ctx)
|
||||
return (
|
||||
MossAudioMultiModalProcessor(
|
||||
info,
|
||||
MossAudioDummyInputsBuilder(info),
|
||||
),
|
||||
ctx,
|
||||
)
|
||||
|
||||
|
||||
def _assert_mm_inputs_equal(left, right):
|
||||
assert left["prompt_token_ids"] == right["prompt_token_ids"]
|
||||
assert left["mm_hashes"] == right["mm_hashes"]
|
||||
|
||||
left_placeholder = left["mm_placeholders"]["audio"][0]
|
||||
right_placeholder = right["mm_placeholders"]["audio"][0]
|
||||
assert left_placeholder.offset == right_placeholder.offset
|
||||
assert left_placeholder.length == right_placeholder.length
|
||||
assert left_placeholder.is_embed.tolist() == right_placeholder.is_embed.tolist()
|
||||
|
||||
assert batched_tensors_equal(
|
||||
left["mm_kwargs"].get_data(),
|
||||
right["mm_kwargs"].get_data(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "prefix"),
|
||||
[
|
||||
(
|
||||
f"before {MOSS_AUDIO_PLACEHOLDER} after",
|
||||
[*[ord(char) for char in "before "], MOSS_AUDIO_BOS_TOKEN_ID],
|
||||
),
|
||||
(
|
||||
f"before {MOSS_AUDIO_BOS_TOKEN}{MOSS_AUDIO_TOKEN}"
|
||||
f"{MOSS_AUDIO_TOKEN}{MOSS_AUDIO_EOS_TOKEN} after",
|
||||
[*[ord(char) for char in "before "], MOSS_AUDIO_BOS_TOKEN_ID],
|
||||
),
|
||||
("Describe this audio.", [MOSS_AUDIO_BOS_TOKEN_ID]),
|
||||
],
|
||||
)
|
||||
def test_moss_audio_processor_expands_audio_placeholders(prompt, prefix):
|
||||
raw_mel_len = 17
|
||||
processed = MossAudioProcessor(_Tokenizer())(
|
||||
text=prompt, audio=[torch.zeros(160 * raw_mel_len)]
|
||||
)
|
||||
input_ids = processed["input_ids"][0].tolist()
|
||||
|
||||
assert input_ids[: len(prefix)] == prefix
|
||||
assert input_ids.count(MOSS_AUDIO_BOS_TOKEN_ID) == 1
|
||||
assert input_ids.count(MOSS_AUDIO_EOS_TOKEN_ID) == 1
|
||||
assert input_ids.count(MOSS_AUDIO_TOKEN_ID) == (
|
||||
MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
|
||||
)
|
||||
assert processed["audio_data"].shape == (1, 128, raw_mel_len)
|
||||
assert processed["audio_data_seqlens"].tolist() == [raw_mel_len]
|
||||
|
||||
|
||||
def test_moss_audio_processor_preserves_placeholder_without_audio():
|
||||
processed = MossAudioProcessor(_Tokenizer())(
|
||||
text=f"before {MOSS_AUDIO_PLACEHOLDER} after"
|
||||
)
|
||||
|
||||
assert processed["input_ids"][0].tolist() == [
|
||||
*[ord(char) for char in "before "],
|
||||
MOSS_AUDIO_BOS_TOKEN_ID,
|
||||
MOSS_AUDIO_TOKEN_ID,
|
||||
MOSS_AUDIO_EOS_TOKEN_ID,
|
||||
*[ord(char) for char in " after"],
|
||||
]
|
||||
assert "audio_data" not in processed
|
||||
assert "audio_data_seqlens" not in processed
|
||||
|
||||
|
||||
def test_moss_audio_multimodal_processor_handles_token_and_cache_paths():
|
||||
raw_mel_len = 17
|
||||
audio = np.zeros(160 * raw_mel_len, dtype=np.float32)
|
||||
prompt = f"{MOSS_AUDIO_PLACEHOLDER}\nTranscribe this audio."
|
||||
|
||||
baseline_processor, ctx = _build_moss_audio_processor()
|
||||
mm_items = baseline_processor.info.parse_mm_data({"audio": [audio]})
|
||||
token_prompt = ctx.get_tokenizer().encode(prompt, add_special_tokens=False)
|
||||
|
||||
baseline_text = baseline_processor(
|
||||
prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
baseline_token = baseline_processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
cached_processor, _ = _build_moss_audio_processor()
|
||||
cached_text_miss = cached_processor(
|
||||
prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
cached_text_hit = cached_processor(
|
||||
prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
cached_token_hit = cached_processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
expected_audio_tokens = MossAudioEncoder.compute_num_audio_tokens(raw_mel_len)
|
||||
prompt_token_ids = baseline_text["prompt_token_ids"]
|
||||
assert prompt_token_ids.count(MOSS_AUDIO_TOKEN_ID) == expected_audio_tokens
|
||||
assert baseline_text["mm_placeholders"]["audio"][0].length == (
|
||||
expected_audio_tokens + 2
|
||||
)
|
||||
|
||||
_assert_mm_inputs_equal(baseline_text, baseline_token)
|
||||
_assert_mm_inputs_equal(baseline_text, cached_text_miss)
|
||||
_assert_mm_inputs_equal(baseline_text, cached_text_hit)
|
||||
_assert_mm_inputs_equal(baseline_text, cached_token_hit)
|
||||
|
||||
|
||||
def test_moss_audio_supports_language_model_lora_only():
|
||||
assert supports_lora(MossAudioModel)
|
||||
|
||||
model = object.__new__(MossAudioModel)
|
||||
assert isinstance(model, SupportsLoRA)
|
||||
|
||||
mapping = model.get_mm_mapping()
|
||||
assert mapping.language_model == ["language_model."]
|
||||
assert mapping.tower_model == []
|
||||
assert mapping.connector == []
|
||||
|
||||
|
||||
def test_moss_audio_error_paths():
|
||||
model = object.__new__(MossAudioModel)
|
||||
with pytest.raises(ValueError, match="DeepStack audio token count mismatch"):
|
||||
model._cache_deepstack_input_embeds(
|
||||
inputs_embeds=torch.zeros(4, 8),
|
||||
deepstack_embeddings=((torch.ones(1, 8),),),
|
||||
is_multimodal=torch.tensor([False, True, True, False]),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="too short"):
|
||||
MossAudioProcessor(_Tokenizer())(
|
||||
text=MOSS_AUDIO_PLACEHOLDER, audio=[torch.empty(0)]
|
||||
)
|
||||
with pytest.raises(ValueError, match="too short"):
|
||||
model._parse_and_validate_audio_input(
|
||||
audio_data=torch.zeros(1, 128, 1),
|
||||
audio_data_seqlens=torch.tensor([0], dtype=torch.long),
|
||||
)
|
||||
|
||||
|
||||
def test_moss_audio_validates_tp_config():
|
||||
vllm_config = _vllm_config(tensor_parallel_size=2)
|
||||
vllm_config.model_config.hf_config.adapter_hidden_size = 7
|
||||
|
||||
with pytest.raises(ValueError, match="adapter_hidden_size"):
|
||||
MossAudioModel(vllm_config=vllm_config)
|
||||
|
||||
vllm_config = _vllm_config(tensor_parallel_size=2)
|
||||
vllm_config.model_config.hf_config.audio_config.d_model = 6
|
||||
vllm_config.model_config.hf_config.audio_config.encoder_attention_heads = 3
|
||||
with pytest.raises(ValueError, match="encoder_attention_heads"):
|
||||
MossAudioModel(vllm_config=vllm_config)
|
||||
|
||||
|
||||
def test_moss_audio_rejects_audio_data_list_seqlen_count_mismatch():
|
||||
model = object.__new__(MossAudioModel)
|
||||
|
||||
with pytest.raises(ValueError, match="audio_data batch size"):
|
||||
model._parse_and_validate_audio_input(
|
||||
audio_data=[torch.zeros(128, 8), torch.zeros(128, 11)],
|
||||
audio_data_seqlens=torch.tensor([8], dtype=torch.long),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deepstack_scales", [(), (7, 11)])
|
||||
def test_moss_audio_embed_multimodal_packs_by_audio(deepstack_scales):
|
||||
model = object.__new__(MossAudioModel)
|
||||
model.audio_encoder = _FakeAudioEncoder(len(deepstack_scales))
|
||||
model.audio_adapter = lambda hidden_states: hidden_states * 5
|
||||
model.deepstack_audio_merger_list = [
|
||||
lambda hidden_states, scale=scale: hidden_states * scale
|
||||
for scale in deepstack_scales
|
||||
]
|
||||
model.deepstack_input_embeds = None
|
||||
|
||||
embeddings = model.embed_multimodal(
|
||||
audio_data=torch.zeros(2, 128, 9),
|
||||
audio_data_seqlens=torch.tensor([8, 9], dtype=torch.long),
|
||||
)
|
||||
|
||||
assert model.audio_encoder.output_deepstack_hidden_states is bool(deepstack_scales)
|
||||
assert [embeds.shape for embeds in embeddings] == [
|
||||
torch.Size([1, 8 * (1 + len(deepstack_scales))]),
|
||||
torch.Size([2, 8 * (1 + len(deepstack_scales))]),
|
||||
]
|
||||
if not deepstack_scales:
|
||||
assert model.deepstack_input_embeds is None
|
||||
return
|
||||
|
||||
main_embeddings, deepstack_embeddings = model._split_multimodal_embeddings(
|
||||
embeddings, hidden_size=8
|
||||
)
|
||||
assert [embeds.shape for embeds in main_embeddings] == [
|
||||
torch.Size([1, 8]),
|
||||
torch.Size([2, 8]),
|
||||
]
|
||||
assert [[e.shape for e in layer] for layer in deepstack_embeddings] == [
|
||||
[torch.Size([1, 8]), torch.Size([2, 8])] for _ in deepstack_scales
|
||||
]
|
||||
assert torch.equal(main_embeddings[0], torch.full((1, 8), 5.0))
|
||||
for idx, scale in enumerate(deepstack_scales):
|
||||
assert torch.equal(
|
||||
deepstack_embeddings[idx][0],
|
||||
torch.full((1, 8), float((idx + 2) * scale)),
|
||||
)
|
||||
|
||||
|
||||
def test_moss_audio_embed_input_ids_caches_packed_deepstack():
|
||||
class _FakeLanguageModel:
|
||||
def embed_input_ids(self, input_ids):
|
||||
return torch.zeros(input_ids.shape[0], 8)
|
||||
|
||||
model = object.__new__(MossAudioModel)
|
||||
model.language_model = _FakeLanguageModel()
|
||||
model.deepstack_audio_merger_list = [object(), object()]
|
||||
model.deepstack_input_embeds = None
|
||||
multimodal_embeddings = (
|
||||
torch.cat([torch.full((1, 8), x) for x in (5.0, 14.0, 33.0)], dim=-1),
|
||||
torch.cat([torch.full((2, 8), x) for x in (7.0, 22.0, 44.0)], dim=-1),
|
||||
)
|
||||
is_multimodal = torch.tensor([False, True, True, True, False])
|
||||
|
||||
inputs_embeds = model.embed_input_ids(
|
||||
input_ids=torch.arange(5),
|
||||
multimodal_embeddings=multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
assert torch.equal(inputs_embeds[1], torch.full((8,), 5.0))
|
||||
assert torch.equal(inputs_embeds[2], torch.full((8,), 7.0))
|
||||
assert torch.equal(inputs_embeds[3], torch.full((8,), 7.0))
|
||||
assert model.deepstack_input_embeds is not None
|
||||
tensors = model.deepstack_input_embeds.tensors
|
||||
assert set(tensors) == {"deepstack_input_embeds_0", "deepstack_input_embeds_1"}
|
||||
for tensor in tensors.values():
|
||||
assert tensor[is_multimodal].abs().sum() > 0
|
||||
assert torch.equal(tensor[~is_multimodal], torch.zeros(2, 8))
|
||||
|
||||
|
||||
def _patch_pp_group(monkeypatch, *, first=True, last=True):
|
||||
import vllm.model_executor.models.moss_audio as moss_audio_module
|
||||
|
||||
monkeypatch.setattr(
|
||||
moss_audio_module,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=first, is_last_rank=last),
|
||||
)
|
||||
|
||||
|
||||
def test_moss_audio_pp_forward_routes_deepstack(monkeypatch):
|
||||
for first in (True, False):
|
||||
calls: list[dict[str, object]] = []
|
||||
|
||||
def fake_lm(*args, _calls=calls, **kwargs):
|
||||
del args
|
||||
_calls.append(kwargs)
|
||||
return torch.ones(1, 1)
|
||||
|
||||
_patch_pp_group(monkeypatch, first=first)
|
||||
model = object.__new__(MossAudioModel)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.language_model = fake_lm
|
||||
cached = IntermediateTensors({"deepstack_input_embeds_0": torch.ones(3, 8)})
|
||||
inter = IntermediateTensors(
|
||||
{
|
||||
"hidden_states": torch.ones(3, 8),
|
||||
"residual": torch.zeros(3, 8),
|
||||
"deepstack_input_embeds_0": torch.full((3, 8), 5.0),
|
||||
}
|
||||
)
|
||||
inputs_embeds = torch.full((3, 8), 9.0)
|
||||
model.deepstack_input_embeds = cached
|
||||
|
||||
model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.arange(3),
|
||||
intermediate_tensors=None if first else inter,
|
||||
inputs_embeds=inputs_embeds if first else None,
|
||||
)
|
||||
|
||||
kwargs = calls[0]
|
||||
assert kwargs["inputs_embeds"] is (inputs_embeds if first else None)
|
||||
assert kwargs["deepstack_input_embeds"] is (cached if first else inter)
|
||||
assert model.deepstack_input_embeds is None
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_lm_non_first_rank(*args, **kwargs):
|
||||
del args
|
||||
calls.append(kwargs)
|
||||
return torch.ones(1, 1)
|
||||
|
||||
_patch_pp_group(monkeypatch, first=False)
|
||||
model = object.__new__(MossAudioModel)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.language_model = fake_lm_non_first_rank
|
||||
model.deepstack_input_embeds = IntermediateTensors({})
|
||||
inter = IntermediateTensors(
|
||||
{
|
||||
"hidden_states": torch.ones(3, 8),
|
||||
"residual": torch.zeros(3, 8),
|
||||
}
|
||||
)
|
||||
|
||||
model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.arange(3),
|
||||
intermediate_tensors=inter,
|
||||
inputs_embeds=torch.ones(3, 8),
|
||||
)
|
||||
assert calls[0]["inputs_embeds"] is None
|
||||
assert calls[0]["deepstack_input_embeds"] is inter
|
||||
|
||||
|
||||
def test_moss_qwen3_deepstack_keys_for_pp(monkeypatch):
|
||||
class AddOne(torch.nn.Module):
|
||||
def forward(self, positions, hidden_states, residual):
|
||||
del positions, residual
|
||||
return hidden_states + 1, torch.zeros_like(hidden_states)
|
||||
|
||||
def make_model(num_layers, deepstack_layers=None):
|
||||
model = object.__new__(MossQwen3Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.start_layer, model.end_layer = 0, num_layers
|
||||
model.layers = torch.nn.ModuleList([AddOne() for _ in range(num_layers)])
|
||||
model.norm = lambda hidden_states, residual: (hidden_states, residual)
|
||||
model._maybe_add_hidden_state = lambda aux, *args: aux
|
||||
model.deepstack_inject_layer_indices = (
|
||||
range(0) if deepstack_layers is None else deepstack_layers
|
||||
)
|
||||
return model
|
||||
|
||||
_patch_pp_group(monkeypatch, first=True, last=True)
|
||||
output = make_model(3).forward(
|
||||
input_ids=None,
|
||||
positions=torch.arange(2),
|
||||
inputs_embeds=torch.zeros(2, 4),
|
||||
deepstack_input_embeds=IntermediateTensors(
|
||||
{
|
||||
"deepstack_input_embeds_2": torch.full((2, 4), 5.0),
|
||||
}
|
||||
),
|
||||
)
|
||||
assert torch.equal(output, torch.full((2, 4), 8.0))
|
||||
|
||||
_patch_pp_group(monkeypatch, first=True, last=False)
|
||||
deepstack = IntermediateTensors(
|
||||
{
|
||||
"deepstack_input_embeds_0": torch.full((2, 4), 7.0),
|
||||
"deepstack_input_embeds_3": torch.full((2, 4), 11.0),
|
||||
}
|
||||
)
|
||||
output = make_model(2, range(4)).forward(
|
||||
input_ids=None,
|
||||
positions=torch.arange(2),
|
||||
inputs_embeds=torch.zeros(2, 4),
|
||||
deepstack_input_embeds=deepstack,
|
||||
)
|
||||
assert isinstance(output, IntermediateTensors)
|
||||
assert set(output.tensors) == {
|
||||
"hidden_states",
|
||||
"residual",
|
||||
"deepstack_input_embeds_2",
|
||||
"deepstack_input_embeds_3",
|
||||
}
|
||||
assert torch.equal(output["hidden_states"], torch.full((2, 4), 9.0))
|
||||
assert torch.equal(output["deepstack_input_embeds_2"], torch.zeros(2, 4))
|
||||
assert output["deepstack_input_embeds_3"] is deepstack["deepstack_input_embeds_3"]
|
||||
|
||||
inner_model = make_model(0, range(2))
|
||||
inner_model.make_empty_intermediate_tensors = lambda batch, dtype, device: (
|
||||
IntermediateTensors(
|
||||
{
|
||||
"hidden_states": torch.zeros(batch, 4, dtype=dtype, device=device),
|
||||
"residual": torch.zeros(batch, 4, dtype=dtype, device=device),
|
||||
}
|
||||
)
|
||||
)
|
||||
language_model = object.__new__(MossQwen3ForCausalLM)
|
||||
torch.nn.Module.__init__(language_model)
|
||||
language_model.model = inner_model
|
||||
language_model.config = SimpleNamespace(hidden_size=4)
|
||||
language_model.deepstack_inject_layer_indices = range(2)
|
||||
|
||||
tensors = MossQwen3ForCausalLM.make_empty_intermediate_tensors(
|
||||
language_model,
|
||||
batch_size=3,
|
||||
dtype=torch.float16,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
assert set(tensors.tensors) == {
|
||||
"hidden_states",
|
||||
"residual",
|
||||
"deepstack_input_embeds_0",
|
||||
"deepstack_input_embeds_1",
|
||||
}
|
||||
assert tensors["deepstack_input_embeds_0"].shape == (3, 4)
|
||||
assert tensors["deepstack_input_embeds_0"].dtype == torch.float16
|
||||
|
||||
_patch_pp_group(monkeypatch, first=True, last=False)
|
||||
forward_tensors = inner_model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.arange(3),
|
||||
inputs_embeds=torch.ones(3, 4, dtype=torch.float16),
|
||||
deepstack_input_embeds=None,
|
||||
)
|
||||
assert isinstance(forward_tensors, IntermediateTensors)
|
||||
assert set(forward_tensors.tensors) == set(tensors.tensors)
|
||||
|
||||
|
||||
def test_moss_audio_encoder_loads_realistic_attention_weight_names(monkeypatch):
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.device import DeviceConfig
|
||||
|
||||
_patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=2)
|
||||
config = MossAudioEncoderConfig(
|
||||
d_model=8,
|
||||
output_dim=8,
|
||||
num_mel_bins=8,
|
||||
encoder_layers=1,
|
||||
encoder_attention_heads=2,
|
||||
encoder_ffn_dim=16,
|
||||
downsample_hidden_size=2,
|
||||
deepstack_encoder_layer_indexes=[],
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
|
||||
encoder = MossAudioEncoder(config)
|
||||
|
||||
attention = encoder.layers[0].self_attn
|
||||
assert all(hasattr(attention, name) for name in ("q_proj", "k_proj", "v_proj"))
|
||||
assert hasattr(attention, "out_proj")
|
||||
assert not hasattr(attention, "qkv")
|
||||
assert attention.k_proj.bias is None
|
||||
|
||||
weight_names = [
|
||||
"layers.0.self_attn.q_proj.weight",
|
||||
"layers.0.self_attn.q_proj.bias",
|
||||
"layers.0.self_attn.k_proj.weight",
|
||||
"layers.0.self_attn.v_proj.weight",
|
||||
"layers.0.self_attn.v_proj.bias",
|
||||
"layers.0.self_attn.out_proj.weight",
|
||||
"layers.0.self_attn.out_proj.bias",
|
||||
"conv1.weight",
|
||||
"conv1.bias",
|
||||
]
|
||||
params = dict(encoder.named_parameters(remove_duplicate=False))
|
||||
assert "layers.0.self_attn.k_proj.bias" not in params
|
||||
weights = {
|
||||
name: torch.full_like(params[name], fill_value=float(i + 1))
|
||||
for i, name in enumerate(weight_names)
|
||||
}
|
||||
|
||||
loaded = AutoWeightsLoader(encoder).load_weights(weights.items())
|
||||
|
||||
assert "load_weights" not in MossAudioEncoder.__dict__
|
||||
assert loaded == set(weight_names)
|
||||
assert not any(".qkv." in name for name in loaded)
|
||||
assert torch.equal(
|
||||
params["layers.0.self_attn.q_proj.weight"],
|
||||
weights["layers.0.self_attn.q_proj.weight"],
|
||||
)
|
||||
@@ -1,108 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.model_executor.models.moss_transcribe_diarize import (
|
||||
MossTranscribeDiarizeForConditionalGeneration,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_preserves_moss_segments():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0.48][S01]Welcome[1.66][12.26][S02]Ready[13.81]"
|
||||
)
|
||||
|
||||
assert [
|
||||
(segment.start, segment.end, segment.speaker, segment.text)
|
||||
for segment in segments
|
||||
] == [
|
||||
(0.48, 1.66, "S01", "Welcome"),
|
||||
(12.26, 13.81, "S02", "Ready"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_preserves_overlapping_segments():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01]First speaker[2][1][S02]Second speaker[3]"
|
||||
)
|
||||
|
||||
assert [
|
||||
(segment.start, segment.end, segment.speaker, segment.text)
|
||||
for segment in segments
|
||||
] == [
|
||||
(0.0, 2.0, "S01", "First speaker"),
|
||||
(1.0, 3.0, "S02", "Second speaker"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_preserves_numeric_text_markers():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01]The [2024] report is ready.[4]"
|
||||
)
|
||||
|
||||
assert [segment.text for segment in segments] == ["The [2024] report is ready."]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_ignores_whitespace_between_segments():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01]Hello[1]\n [2][S02]Hi[3]"
|
||||
)
|
||||
|
||||
assert [(segment.start, segment.end, segment.text) for segment in segments] == [
|
||||
(0.0, 1.0, "Hello"),
|
||||
(2.0, 3.0, "Hi"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_ignores_noise_before_a_segment():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"noise [bad][0.1][S01]Hello[0.9]"
|
||||
)
|
||||
|
||||
assert [(segment.start, segment.end, segment.text) for segment in segments] == [
|
||||
(0.1, 0.9, "Hello"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_preserves_timestamps_before_the_end():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[2][S01]The earlier timestamp is [1] not the end[3]"
|
||||
)
|
||||
|
||||
assert [segment.text for segment in segments] == [
|
||||
"The earlier timestamp is [1] not the end",
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_skips_empty_segments():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01][1][2][S02]Complete[3]"
|
||||
)
|
||||
|
||||
assert [(segment.speaker, segment.text) for segment in segments] == [
|
||||
("S02", "Complete"),
|
||||
]
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_fails_closed_for_incomplete_output():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01]Complete[1][2][S02]Incomplete"
|
||||
)
|
||||
|
||||
assert segments == []
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_fails_closed_for_trailing_text():
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
"[0][S01]Complete[1] trailing text"
|
||||
)
|
||||
|
||||
assert segments == []
|
||||
|
||||
|
||||
def test_parse_diarized_transcript_preserves_overlong_timestamp_markers():
|
||||
marker = f"[{'1' * 33}]"
|
||||
segments = MossTranscribeDiarizeForConditionalGeneration.parse_diarized_transcript(
|
||||
f"[0][S01]Value {marker}[1]"
|
||||
)
|
||||
|
||||
assert [segment.text for segment in segments] == [f"Value {marker}"]
|
||||
@@ -1,205 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for OpenVLA multimodal preprocessing."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import LlamaConfig
|
||||
|
||||
from vllm.model_executor.models.openvla import (
|
||||
OpenVLAForActionPrediction,
|
||||
OpenVLAMultiModalProcessor,
|
||||
OpenVLAProcessingInfo,
|
||||
)
|
||||
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
|
||||
from vllm.transformers_utils.configs.openvla import OpenVLAConfig
|
||||
from vllm.transformers_utils.processors.openvla import (
|
||||
IMAGENET_MEAN,
|
||||
IMAGENET_STD,
|
||||
SIGLIP_MEAN,
|
||||
SIGLIP_STD,
|
||||
OpenVLAImageProcessor,
|
||||
OpenVLAProcessor,
|
||||
preprocess_openvla_image,
|
||||
to_rgb_image,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _FakeTokenizer:
|
||||
bos_token_id = 1
|
||||
init_kwargs: dict[str, object] = {}
|
||||
|
||||
def encode(self, prompt: str, **kwargs: object) -> list[int]:
|
||||
assert prompt == "In: test\nOut:"
|
||||
if kwargs == {"add_special_tokens": True}:
|
||||
return [self.bos_token_id, 10, 11]
|
||||
assert kwargs == {"add_special_tokens": False}
|
||||
return [10, 11]
|
||||
|
||||
def __call__(self, text: str, **kwargs: object) -> dict[str, list[list[int]]]:
|
||||
return {"input_ids": [self.encode(text, **kwargs)]}
|
||||
|
||||
|
||||
class _FakeProcessingInfo:
|
||||
def __init__(self) -> None:
|
||||
self.config = OpenVLAConfig()
|
||||
|
||||
def get_hf_config(self) -> OpenVLAConfig:
|
||||
return self.config
|
||||
|
||||
def get_tokenizer(self) -> _FakeTokenizer:
|
||||
return _FakeTokenizer()
|
||||
|
||||
def get_num_image_tokens(self, *, image_width: int, image_height: int) -> int:
|
||||
assert image_width > 0
|
||||
assert image_height > 0
|
||||
return 256
|
||||
|
||||
|
||||
class _FakeOpenVLAProcessingInfo(OpenVLAProcessingInfo):
|
||||
def get_hf_config(self) -> OpenVLAConfig:
|
||||
return OpenVLAConfig()
|
||||
|
||||
|
||||
def _make_processor() -> OpenVLAMultiModalProcessor:
|
||||
processor = OpenVLAMultiModalProcessor.__new__(OpenVLAMultiModalProcessor)
|
||||
processor.info = _FakeProcessingInfo()
|
||||
return processor
|
||||
|
||||
|
||||
def test_openvla_config_converts_text_config_dict() -> None:
|
||||
config = OpenVLAConfig(
|
||||
text_config={
|
||||
"vocab_size": 123,
|
||||
"hidden_size": 64,
|
||||
"intermediate_size": 128,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 4,
|
||||
},
|
||||
)
|
||||
|
||||
assert isinstance(config.text_config, LlamaConfig)
|
||||
assert config.text_config.vocab_size == 123
|
||||
assert config.text_config.hidden_size == 64
|
||||
assert config.text_config.architectures == ["LlamaForCausalLM"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("image", "expected_size", "expected_pixel"),
|
||||
[
|
||||
(
|
||||
torch.tensor(
|
||||
[
|
||||
[[1.0, 1.0], [1.0, 1.0]],
|
||||
[[0.0, 0.0], [0.0, 0.0]],
|
||||
[[0.0, 0.0], [0.0, 0.0]],
|
||||
]
|
||||
),
|
||||
(2, 2),
|
||||
(255, 0, 0),
|
||||
),
|
||||
(
|
||||
np.full((4, 5, 1), 128, dtype=np.uint8),
|
||||
(5, 4),
|
||||
(128, 128, 128),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_openvla_to_rgb_image(
|
||||
image: torch.Tensor | np.ndarray,
|
||||
expected_size: tuple[int, int],
|
||||
expected_pixel: tuple[int, int, int],
|
||||
) -> None:
|
||||
rgb_image = to_rgb_image(image)
|
||||
|
||||
assert rgb_image.mode == "RGB"
|
||||
assert rgb_image.size == expected_size
|
||||
assert rgb_image.getpixel((0, 0)) == expected_pixel
|
||||
|
||||
|
||||
def test_openvla_preprocess_image_matches_expected_normalization() -> None:
|
||||
image = Image.fromarray(
|
||||
np.arange(12 * 10 * 3, dtype=np.uint8).reshape(10, 12, 3),
|
||||
mode="RGB",
|
||||
)
|
||||
|
||||
pixel_values = preprocess_openvla_image(image, image_size=224)
|
||||
|
||||
resized = image.resize((224, 224), Image.Resampling.BICUBIC)
|
||||
raw = np.asarray(resized, dtype=np.float32) / 255.0
|
||||
expected_dinov2 = ((raw - IMAGENET_MEAN) / IMAGENET_STD).transpose(2, 0, 1)
|
||||
expected_siglip = ((raw - SIGLIP_MEAN) / SIGLIP_STD).transpose(2, 0, 1)
|
||||
expected = np.concatenate([expected_dinov2, expected_siglip], axis=0)
|
||||
|
||||
assert pixel_values.shape == (6, 224, 224)
|
||||
assert pixel_values.dtype == torch.float32
|
||||
torch.testing.assert_close(pixel_values, torch.from_numpy(expected))
|
||||
|
||||
|
||||
def test_openvla_processor_outputs_pixel_values() -> None:
|
||||
processor = OpenVLAProcessor(
|
||||
image_processor=OpenVLAImageProcessor(image_size=224),
|
||||
tokenizer=_FakeTokenizer(),
|
||||
)
|
||||
image = Image.new("RGB", (8, 8), color=(255, 0, 0))
|
||||
|
||||
batch = processor(
|
||||
text="In: test\nOut:",
|
||||
images=image,
|
||||
text_kwargs={"add_special_tokens": True},
|
||||
)
|
||||
|
||||
assert batch["input_ids"] == [[1, 10, 11]]
|
||||
assert batch["pixel_values"].shape == (1, 6, 224, 224)
|
||||
assert batch["pixel_values"].dtype == torch.float32
|
||||
|
||||
|
||||
def test_openvla_image_processor_outputs_pixel_values() -> None:
|
||||
processor = OpenVLAImageProcessor(image_size=224)
|
||||
image = Image.new("RGB", (8, 8), color=(255, 0, 0))
|
||||
|
||||
output = processor(images=image)
|
||||
|
||||
assert output["pixel_values"].shape == (1, 6, 224, 224)
|
||||
assert output["pixel_values"].dtype == torch.float32
|
||||
|
||||
|
||||
def test_openvla_processing_info_token_counts() -> None:
|
||||
info = _FakeOpenVLAProcessingInfo.__new__(_FakeOpenVLAProcessingInfo)
|
||||
|
||||
assert info.get_supported_mm_limits() == {"image": 1}
|
||||
assert info.get_num_image_tokens(image_width=640, image_height=480) == 256
|
||||
assert info.get_image_size_with_most_features().width == 224
|
||||
assert info.get_image_size_with_most_features().height == 224
|
||||
assert info.get_mm_max_tokens_per_item(seq_len=2048, mm_counts={"image": 1}) == {
|
||||
"image": 256
|
||||
}
|
||||
|
||||
|
||||
def test_openvla_prompt_update_inserts_image_tokens_after_bos() -> None:
|
||||
processor = _make_processor()
|
||||
image = Image.new("RGB", (640, 480), color=(255, 255, 255))
|
||||
mm_items = MultiModalDataItems({"image": ImageProcessorItems([image])})
|
||||
|
||||
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
|
||||
resolved = prompt_update.resolve(0)
|
||||
content = resolved.content
|
||||
|
||||
assert resolved.modality == "image"
|
||||
assert [
|
||||
(match.start_idx, match.end_idx)
|
||||
for match in resolved.iter_token_matches([1, 10, 11])
|
||||
] == [(1, 1)]
|
||||
assert content.full == [32000] * 256
|
||||
|
||||
is_embed = content.is_embed(content.full)
|
||||
assert is_embed.dtype == torch.bool
|
||||
assert is_embed.tolist() == [True] * 256
|
||||
|
||||
|
||||
def test_openvla_placeholder_string() -> None:
|
||||
assert OpenVLAForActionPrediction.get_placeholder_str("image", 0) is None
|
||||
@@ -1,58 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for phi3v's multimodal preprocessing kwargs."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["microsoft/Phi-3.5-vision-instruct"])
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||
[
|
||||
({"num_crops": 4}, 757),
|
||||
({"num_crops": 16}, 1921),
|
||||
# the default num_crops of phi-3.5-vision is 4
|
||||
({}, 757),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, int],
|
||||
expected_toks_per_img: int,
|
||||
num_imgs: int,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
"""Ensure Phi3VMultiModalProcessor handles num_crops properly."""
|
||||
# Avoid initializing CUDA early
|
||||
from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
|
||||
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
|
||||
prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
|
||||
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
|
||||
assert img_tok_count == expected_toks_per_img * num_imgs
|
||||
@@ -1,64 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for phi4mm's multimodal preprocessing kwargs."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["microsoft/Phi-4-multimodal-instruct"])
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img"),
|
||||
[
|
||||
({"dynamic_hd": 4}, 1329),
|
||||
({"dynamic_hd": 16}, 4433),
|
||||
# the default num_crops of phi-4-multimodal is 36
|
||||
({}, 9585),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, int],
|
||||
expected_toks_per_img: int,
|
||||
num_imgs: int,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
"""Ensure Phi4MMMultiModalProcessor handles dynamic_hd properly."""
|
||||
# Avoid initializing CUDA early
|
||||
from vllm.model_executor.models.phi4mm import _IMAGE_PLACEHOLDER_TOKEN_ID
|
||||
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
|
||||
prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
|
||||
|
||||
image_size = ctx.get_hf_config().embd_layer["image_embd_layer"]["crop_size"]
|
||||
dummy_image_size = (image_size * 7, image_size * 7)
|
||||
dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
|
||||
mm_data = {"image": [dummy_image] * num_imgs}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(
|
||||
_IMAGE_PLACEHOLDER_TOKEN_ID
|
||||
)
|
||||
assert img_tok_count == expected_toks_per_img * num_imgs
|
||||
@@ -1,457 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for Qwen2.5-Omni embed_input_ids to verify embeddings are
|
||||
correctly assigned to audio/image/video token positions.
|
||||
|
||||
Regression test for: https://github.com/vllm-project/vllm/issues/34506
|
||||
- Non-interleaved mixed modalities (audio + image + video) should correctly
|
||||
assign audio embeddings to audio positions, image to image, video to video.
|
||||
- Interleaved (use_audio_in_video) should also work correctly.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||
check_interleaved_audio_video,
|
||||
merge_interleaved_embeddings,
|
||||
)
|
||||
from vllm.multimodal.utils import set_mm_embedding_modality
|
||||
|
||||
# Fake token IDs
|
||||
AUDIO_TOKEN_ID = 1001
|
||||
IMAGE_TOKEN_ID = 1002
|
||||
VIDEO_TOKEN_ID = 1003
|
||||
TEXT_TOKEN_ID = 0
|
||||
|
||||
|
||||
def _mm_embed(shape: tuple[int, ...], value: float, modality: str) -> torch.Tensor:
|
||||
return set_mm_embedding_modality(torch.full(shape, value), modality)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_token_seq(
|
||||
audio_n: int, image_n: int, video_n: int, text_prefix: int = 3, text_sep: int = 2
|
||||
):
|
||||
"""Build a flat token sequence:
|
||||
[text_prefix] [AUDIO * audio_n] [text_sep] [IMAGE * image_n]
|
||||
[text_sep] [VIDEO * video_n] [text_sep]
|
||||
Returns (input_ids tensor, is_multimodal mask, positions dict).
|
||||
"""
|
||||
tokens = (
|
||||
[TEXT_TOKEN_ID] * text_prefix
|
||||
+ [AUDIO_TOKEN_ID] * audio_n
|
||||
+ [TEXT_TOKEN_ID] * text_sep
|
||||
+ [IMAGE_TOKEN_ID] * image_n
|
||||
+ [TEXT_TOKEN_ID] * text_sep
|
||||
+ [VIDEO_TOKEN_ID] * video_n
|
||||
+ [TEXT_TOKEN_ID] * text_sep
|
||||
)
|
||||
input_ids = torch.tensor(tokens)
|
||||
is_multimodal = (
|
||||
(input_ids == AUDIO_TOKEN_ID)
|
||||
| (input_ids == IMAGE_TOKEN_ID)
|
||||
| (input_ids == VIDEO_TOKEN_ID)
|
||||
)
|
||||
return input_ids, is_multimodal
|
||||
|
||||
|
||||
def make_interleaved_seq(
|
||||
video_chunks: list[int], audio_chunks: list[int], text_prefix: int = 2
|
||||
):
|
||||
"""Build an interleaved sequence like use_audio_in_video:
|
||||
[text] [V*v0] [A*a0] [V*v1] [A*a1] ...
|
||||
"""
|
||||
tokens = [TEXT_TOKEN_ID] * text_prefix
|
||||
for v, a in zip(video_chunks, audio_chunks):
|
||||
tokens += [VIDEO_TOKEN_ID] * v + [AUDIO_TOKEN_ID] * a
|
||||
input_ids = torch.tensor(tokens)
|
||||
is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID)
|
||||
return input_ids, is_multimodal
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for check_interleaved_audio_video
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCheckInterleavedAudioVideo:
|
||||
def test_non_interleaved_audio_then_video(self):
|
||||
"""Audio entirely before video → not interleaved."""
|
||||
input_ids, is_multimodal = make_token_seq(5, 0, 4)
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
assert not check_interleaved_audio_video(
|
||||
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
|
||||
)
|
||||
|
||||
def test_non_interleaved_with_image(self):
|
||||
"""Audio + image + video (the mixed_modalities case) → not interleaved."""
|
||||
input_ids, is_multimodal = make_token_seq(5, 4, 6)
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
assert not check_interleaved_audio_video(
|
||||
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
|
||||
)
|
||||
|
||||
def test_no_audio(self):
|
||||
"""Video only → not interleaved."""
|
||||
input_ids, is_multimodal = make_token_seq(0, 0, 6)
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
assert not check_interleaved_audio_video(
|
||||
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
|
||||
)
|
||||
|
||||
def test_interleaved(self):
|
||||
"""V A V A interleaved → True."""
|
||||
input_ids, is_multimodal = make_interleaved_seq([4, 4], [3, 3])
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
assert check_interleaved_audio_video(
|
||||
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
|
||||
)
|
||||
|
||||
def test_multi_video_with_boundary_tokens(self):
|
||||
"""Two interleaved videos separated by boundary tokens → still True.
|
||||
|
||||
use_audio_in_video expands each video into a local V/A span bounded by
|
||||
non-pad tokens. A global density check would fail across those gaps.
|
||||
"""
|
||||
# [text][V A V A][text boundary][V A V A]
|
||||
first_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=1)
|
||||
second_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=0)
|
||||
boundary = torch.tensor([TEXT_TOKEN_ID, TEXT_TOKEN_ID])
|
||||
input_ids = torch.cat([first_ids, boundary, second_ids])
|
||||
is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID)
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
assert check_interleaved_audio_video(
|
||||
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
|
||||
)
|
||||
|
||||
def test_batched_non_interleaved_no_false_positive(self):
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/35394.
|
||||
|
||||
5 identical non-interleaved mixed-modality requests batched together:
|
||||
each has [audio][image][video] in separate blocks with text between them.
|
||||
Across the batch, audio from request N falls between video blocks of
|
||||
request N and request N+1, causing the global ranges to overlap.
|
||||
check_interleaved_audio_video must return False (not a false positive).
|
||||
"""
|
||||
# Build one request: [text][audio*5][text][image*4][text][video*6][text]
|
||||
single_ids, _ = make_token_seq(5, 4, 6)
|
||||
# Batch 5 identical requests (separated by text tokens to simulate padding)
|
||||
sep = torch.tensor([TEXT_TOKEN_ID] * 3)
|
||||
batched_ids = torch.cat([single_ids, sep] * 5)
|
||||
is_multimodal = (
|
||||
(batched_ids == AUDIO_TOKEN_ID)
|
||||
| (batched_ids == IMAGE_TOKEN_ID)
|
||||
| (batched_ids == VIDEO_TOKEN_ID)
|
||||
)
|
||||
is_video = is_multimodal & (batched_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (batched_ids == AUDIO_TOKEN_ID)
|
||||
assert not check_interleaved_audio_video(
|
||||
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
|
||||
), "Batched non-interleaved requests should not be detected as interleaved"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for embed_input_ids via a minimal mock
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_mock_model(hidden: int = 8):
|
||||
"""Return a minimal mock of Qwen2_5OmniThinkerForConditionalGeneration
|
||||
that has enough structure to run embed_input_ids.
|
||||
"""
|
||||
from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||
Qwen2_5OmniThinkerForConditionalGeneration,
|
||||
)
|
||||
|
||||
model = Mock(spec=Qwen2_5OmniThinkerForConditionalGeneration)
|
||||
|
||||
# Config with token IDs
|
||||
cfg = Mock()
|
||||
cfg.video_token_index = VIDEO_TOKEN_ID
|
||||
cfg.audio_token_index = AUDIO_TOKEN_ID
|
||||
model.config = cfg
|
||||
|
||||
# embed_input_ids: simply embed each token as a one-hot-like vector
|
||||
# token_id * ones so we can verify which embedding ends up where.
|
||||
def fake_lm_embed(ids: torch.Tensor) -> torch.Tensor:
|
||||
# Use .clone() so the tensor is contiguous (expand() creates a strided
|
||||
# view with shared memory, which masked_scatter_ cannot handle).
|
||||
return ids.float().unsqueeze(-1).expand(-1, hidden).clone()
|
||||
|
||||
lang_model = Mock()
|
||||
lang_model.embed_input_ids = fake_lm_embed
|
||||
model.get_language_model = Mock(return_value=lang_model)
|
||||
|
||||
# _embed_text_input_ids: delegate to SupportsMultiModal's implementation
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
|
||||
model._embed_text_input_ids = (
|
||||
lambda *a, **kw: SupportsMultiModal._embed_text_input_ids(model, *a, **kw)
|
||||
)
|
||||
|
||||
# super().embed_input_ids → use SupportsMultiModal.embed_input_ids
|
||||
def fake_super_embed(
|
||||
ids,
|
||||
mm_embs=None,
|
||||
*,
|
||||
is_multimodal=None,
|
||||
):
|
||||
return SupportsMultiModal.embed_input_ids(
|
||||
model,
|
||||
ids,
|
||||
mm_embs,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
# Bind embed_input_ids as the real method
|
||||
model.embed_input_ids = (
|
||||
lambda *a, **kw: Qwen2_5OmniThinkerForConditionalGeneration.embed_input_ids(
|
||||
model, *a, **kw
|
||||
)
|
||||
)
|
||||
|
||||
# Store super-embed for use inside the method
|
||||
model._super_embed_input_ids = fake_super_embed
|
||||
|
||||
return model, hidden
|
||||
|
||||
|
||||
def build_mm_embeds(
|
||||
audio_n, image_n, video_n, hidden, audio_val=10.0, image_val=20.0, video_val=30.0
|
||||
):
|
||||
"""Build multimodal_embeddings list in position order (audio, image, video).
|
||||
Each embedding is filled with a distinct constant so we can verify placement.
|
||||
"""
|
||||
embs = []
|
||||
if audio_n:
|
||||
embs.append(torch.full((audio_n, hidden), audio_val))
|
||||
if image_n:
|
||||
embs.append(torch.full((image_n, hidden), image_val))
|
||||
if video_n:
|
||||
embs.append(torch.full((video_n, hidden), video_val))
|
||||
return embs
|
||||
|
||||
|
||||
class TestEmbedInputIds:
|
||||
def _run(self, audio_n, image_n, video_n, hidden=8):
|
||||
"""Run embed_input_ids for a non-interleaved mixed-modality sequence.
|
||||
Returns (result_embeds, input_ids, is_multimodal).
|
||||
"""
|
||||
input_ids, is_multimodal = make_token_seq(audio_n, image_n, video_n)
|
||||
mm_embeds = build_mm_embeds(audio_n, image_n, video_n, hidden)
|
||||
|
||||
model, _ = make_mock_model(hidden)
|
||||
result = model.embed_input_ids(
|
||||
input_ids, mm_embeds, is_multimodal=is_multimodal
|
||||
)
|
||||
return result, input_ids, is_multimodal
|
||||
|
||||
def test_audio_only(self):
|
||||
"""Audio-only: audio positions get audio embeddings."""
|
||||
audio_n, hidden = 5, 8
|
||||
audio_val = 10.0
|
||||
result, input_ids, is_multimodal = self._run(audio_n, 0, 0, hidden)
|
||||
|
||||
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
|
||||
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
|
||||
"Audio positions should get audio embeddings"
|
||||
)
|
||||
|
||||
def test_video_only(self):
|
||||
"""Video-only: video positions get video embeddings."""
|
||||
video_n, hidden = 6, 8
|
||||
video_val = 30.0
|
||||
result, input_ids, is_multimodal = self._run(0, 0, video_n, hidden)
|
||||
|
||||
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
|
||||
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
|
||||
"Video positions should get video embeddings"
|
||||
)
|
||||
|
||||
def test_mixed_modalities_audio_goes_to_audio_pos(self):
|
||||
"""Regression test for GitHub issue #34506:
|
||||
With audio + image + video (non-interleaved), audio positions must
|
||||
receive audio embeddings (not image or video embeddings).
|
||||
"""
|
||||
audio_n, image_n, video_n, hidden = 5, 4, 6, 8
|
||||
audio_val, image_val, video_val = 10.0, 20.0, 30.0
|
||||
|
||||
result, input_ids, is_multimodal = self._run(audio_n, image_n, video_n, hidden)
|
||||
|
||||
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
|
||||
image_pos = (input_ids == IMAGE_TOKEN_ID).nonzero(as_tuple=True)[0]
|
||||
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
|
||||
|
||||
mean_a = result[audio_pos].mean().item()
|
||||
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
|
||||
f"Audio emb wrong: expected {audio_val}, got mean={mean_a:.1f}"
|
||||
)
|
||||
|
||||
mean_i = result[image_pos].mean().item()
|
||||
assert result[image_pos].allclose(torch.full((image_n, hidden), image_val)), (
|
||||
f"Image emb wrong: expected {image_val}, got mean={mean_i:.1f}"
|
||||
)
|
||||
|
||||
mean_v = result[video_pos].mean().item()
|
||||
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
|
||||
f"Video emb wrong: expected {video_val}, got mean={mean_v:.1f}"
|
||||
)
|
||||
|
||||
def test_text_positions_unchanged(self):
|
||||
"""Text positions should keep their text embeddings."""
|
||||
audio_n, image_n, video_n, hidden = 3, 2, 4, 8
|
||||
result, input_ids, is_multimodal = self._run(audio_n, image_n, video_n, hidden)
|
||||
|
||||
text_pos = (~is_multimodal).nonzero(as_tuple=True)[0]
|
||||
# Text tokens have value TEXT_TOKEN_ID=0, so embed → 0.0
|
||||
assert result[text_pos].allclose(torch.zeros(len(text_pos), hidden)), (
|
||||
"Text positions should keep text embeddings"
|
||||
)
|
||||
|
||||
def test_interleaved_use_audio_in_video(self):
|
||||
"""Interleaved (use_audio_in_video): video chunks interleaved with audio.
|
||||
Video embeddings must go to video positions, audio to audio positions.
|
||||
"""
|
||||
hidden = 8
|
||||
audio_val, video_val = 10.0, 30.0
|
||||
# Two video chunks of 4, two audio chunks of 3
|
||||
video_chunks = [4, 4]
|
||||
audio_chunks = [3, 3]
|
||||
input_ids, is_multimodal = make_interleaved_seq(video_chunks, audio_chunks)
|
||||
|
||||
video_n = sum(video_chunks) # 8
|
||||
audio_n = sum(audio_chunks) # 6
|
||||
|
||||
# mm_embeds come in [video, audio] order (video feature first in
|
||||
# mm_features when positions are the same for use_audio_in_video)
|
||||
mm_embeds = [
|
||||
_mm_embed((video_n, hidden), video_val, "video"),
|
||||
_mm_embed((audio_n, hidden), audio_val, "audio"),
|
||||
]
|
||||
|
||||
model, _ = make_mock_model(hidden)
|
||||
# Modalities are attached on the embedding tensors (as in encoder gather).
|
||||
result = model.embed_input_ids(
|
||||
input_ids,
|
||||
mm_embeds,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
|
||||
audio_pos = (input_ids == AUDIO_TOKEN_ID).nonzero(as_tuple=True)[0]
|
||||
|
||||
assert result[video_pos].allclose(torch.full((video_n, hidden), video_val)), (
|
||||
"Interleaved: video positions should get video embeddings"
|
||||
)
|
||||
|
||||
assert result[audio_pos].allclose(torch.full((audio_n, hidden), audio_val)), (
|
||||
"Interleaved: audio positions should get audio embeddings"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for merge_interleaved_embeddings helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMergeInterleavedEmbeddings:
|
||||
def test_basic_interleaved(self):
|
||||
"""Video chunks + audio chunks scattered to correct positions."""
|
||||
hidden = 4
|
||||
input_ids, is_multimodal = make_interleaved_seq([3, 3], [2, 2])
|
||||
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
num_video = is_video.sum().item() # 6
|
||||
num_audio = is_audio.sum().item() # 4
|
||||
|
||||
inputs_embeds = torch.zeros(len(input_ids), hidden)
|
||||
mm_embeds = [
|
||||
_mm_embed((num_video, hidden), 30.0, "video"),
|
||||
_mm_embed((num_audio, hidden), 10.0, "audio"),
|
||||
]
|
||||
|
||||
result = merge_interleaved_embeddings(
|
||||
inputs_embeds,
|
||||
mm_embeds,
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
)
|
||||
|
||||
video_pos = is_video.nonzero(as_tuple=True)[0]
|
||||
audio_pos = is_audio.nonzero(as_tuple=True)[0]
|
||||
assert result[video_pos].allclose(torch.full((num_video, hidden), 30.0))
|
||||
assert result[audio_pos].allclose(torch.full((num_audio, hidden), 10.0))
|
||||
|
||||
def test_image_and_video_mixed(self):
|
||||
"""Image embeddings must not be misclassified as video."""
|
||||
hidden = 4
|
||||
# [text][I I][V A V A]
|
||||
tokens = (
|
||||
[TEXT_TOKEN_ID] * 2
|
||||
+ [IMAGE_TOKEN_ID] * 2
|
||||
+ [VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID]
|
||||
)
|
||||
input_ids = torch.tensor(tokens)
|
||||
is_multimodal = (
|
||||
(input_ids == IMAGE_TOKEN_ID)
|
||||
| (input_ids == VIDEO_TOKEN_ID)
|
||||
| (input_ids == AUDIO_TOKEN_ID)
|
||||
)
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
is_image = is_multimodal & (input_ids == IMAGE_TOKEN_ID)
|
||||
|
||||
inputs_embeds = torch.zeros(len(input_ids), hidden)
|
||||
mm_embeds = [
|
||||
_mm_embed((2, hidden), 20.0, "image"),
|
||||
_mm_embed((2, hidden), 30.0, "video"),
|
||||
_mm_embed((2, hidden), 10.0, "audio"),
|
||||
]
|
||||
result = merge_interleaved_embeddings(
|
||||
inputs_embeds,
|
||||
mm_embeds,
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
)
|
||||
assert result[is_image.nonzero(as_tuple=True)[0]].allclose(
|
||||
torch.full((2, hidden), 20.0)
|
||||
)
|
||||
assert result[is_video.nonzero(as_tuple=True)[0]].allclose(
|
||||
torch.full((2, hidden), 30.0)
|
||||
)
|
||||
assert result[is_audio.nonzero(as_tuple=True)[0]].allclose(
|
||||
torch.full((2, hidden), 10.0)
|
||||
)
|
||||
|
||||
def test_missing_modality_raises(self):
|
||||
hidden = 2
|
||||
input_ids, is_multimodal = make_interleaved_seq([2], [2])
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
with pytest.raises(ValueError, match="Missing modality"):
|
||||
merge_interleaved_embeddings(
|
||||
torch.zeros(len(input_ids), hidden),
|
||||
[torch.zeros(2, hidden), torch.zeros(2, hidden)],
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -1,484 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.multimodal.processing.context import InputProcessingContext
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_kwargs", "expected"),
|
||||
[
|
||||
(
|
||||
{"size": {"shortest_edge": 64, "longest_edge": 1024}},
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 64,
|
||||
"max_pixels": 1024,
|
||||
},
|
||||
),
|
||||
(
|
||||
{"min_pixels": 64, "max_pixels": 1024},
|
||||
{
|
||||
"min_pixels": 64,
|
||||
"max_pixels": 1024,
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 2048,
|
||||
},
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 2048,
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"min_pixels": None,
|
||||
"size": {"shortest_edge": 0},
|
||||
"images_kwargs": {"max_pixels": 4096},
|
||||
"videos_kwargs": {"size": {"longest_edge": 8192}},
|
||||
},
|
||||
{
|
||||
"min_pixels": None,
|
||||
"size": {"shortest_edge": 0},
|
||||
"images_kwargs": {
|
||||
"max_pixels": 4096,
|
||||
"size": {"longest_edge": 4096},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"size": {"longest_edge": 8192},
|
||||
"max_pixels": 8192,
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
{"min_pixels": 0},
|
||||
{"min_pixels": 0, "size": {"shortest_edge": 0}},
|
||||
),
|
||||
(
|
||||
{"size": None, "min_pixels": 64},
|
||||
{"size": {"shortest_edge": 64}, "min_pixels": 64},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_complete_mm_processor_size_aliases(
|
||||
mm_kwargs: dict[str, object],
|
||||
expected: dict[str, object],
|
||||
) -> None:
|
||||
original = deepcopy(mm_kwargs)
|
||||
|
||||
actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
|
||||
|
||||
assert actual == expected
|
||||
assert mm_kwargs == original
|
||||
|
||||
|
||||
def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
"images_kwargs": {"min_pixels": 64},
|
||||
},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
|
||||
ctx = InputProcessingContext(model_config, tokenizer=None)
|
||||
info = Qwen2VLProcessingInfo(ctx)
|
||||
monkeypatch.setattr(
|
||||
info,
|
||||
"get_supported_mm_processor_kwargs",
|
||||
lambda: {"images_kwargs": {"size", "min_pixels"}},
|
||||
)
|
||||
|
||||
assert info._merge_and_resolve_mm_processor_kwargs(
|
||||
{"size": {"shortest_edge": 128}}
|
||||
) == {
|
||||
"images_kwargs": {
|
||||
"size": {"shortest_edge": 128},
|
||||
"min_pixels": 128,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_kwargs", "default_size", "expected"),
|
||||
[
|
||||
(
|
||||
{},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
),
|
||||
(
|
||||
{"size": {"shortest_edge": 64}},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 64, "longest_edge": 1024},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 2048},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 4096,
|
||||
},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 128, "longest_edge": 4096},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64},
|
||||
"max_pixels": 4096,
|
||||
},
|
||||
None,
|
||||
{"shortest_edge": 64, "longest_edge": 4096},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 2048},
|
||||
"min_pixels": None,
|
||||
"max_pixels": 0,
|
||||
},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 64, "longest_edge": 0},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_vision_size(
|
||||
mm_kwargs: dict[str, object],
|
||||
default_size: dict[str, object] | None,
|
||||
expected: dict[str, object],
|
||||
) -> None:
|
||||
original_mm_kwargs = {
|
||||
key: dict(value) if isinstance(value, dict) else value
|
||||
for key, value in mm_kwargs.items()
|
||||
}
|
||||
original_default_size = dict(default_size) if default_size is not None else None
|
||||
|
||||
actual = Qwen2VLProcessingInfo._get_vision_size(
|
||||
mm_kwargs,
|
||||
default_size=default_size,
|
||||
)
|
||||
|
||||
assert actual == expected
|
||||
assert mm_kwargs == original_mm_kwargs
|
||||
assert default_size == original_default_size
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("modality", "scope"),
|
||||
[
|
||||
("image", "images_kwargs"),
|
||||
("video", "videos_kwargs"),
|
||||
],
|
||||
)
|
||||
def test_get_vision_info_reads_merged_modality_scope(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
modality: str,
|
||||
scope: str,
|
||||
) -> None:
|
||||
vision_config = SimpleNamespace(
|
||||
patch_size=14,
|
||||
spatial_merge_size=2,
|
||||
temporal_patch_size=2,
|
||||
)
|
||||
ctx = SimpleNamespace(
|
||||
get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
|
||||
)
|
||||
info = Qwen2VLProcessingInfo(ctx)
|
||||
merged = {
|
||||
"images_kwargs": {"size": {"shortest_edge": 111}},
|
||||
"videos_kwargs": {"size": {"shortest_edge": 222}},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
info,
|
||||
"_merge_and_resolve_mm_processor_kwargs",
|
||||
lambda _: merged,
|
||||
)
|
||||
|
||||
seen: list[dict[str, object]] = []
|
||||
|
||||
def get_size(
|
||||
mm_kwargs: dict[str, object],
|
||||
default_size: dict[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
seen.append(mm_kwargs)
|
||||
return dict(default_size or {})
|
||||
|
||||
monkeypatch.setattr(info, "_get_vision_size", get_size)
|
||||
image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
|
||||
|
||||
info._get_vision_info(
|
||||
image_width=28,
|
||||
image_height=28,
|
||||
num_frames=2,
|
||||
do_resize=False,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs={},
|
||||
modality=modality,
|
||||
)
|
||||
|
||||
assert seen == [merged[scope]]
|
||||
|
||||
|
||||
def test_jina_vl_processing_order() -> None:
|
||||
"""Jina's document-first prompt keeps cached features and hashes aligned."""
|
||||
ctx = build_model_context(
|
||||
"jinaai/jina-reranker-m0",
|
||||
runner="pooling",
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
query_image = Image.new("RGB", (128, 160), color=(255, 0, 0))
|
||||
document_image = Image.new("RGB", (192, 128), color=(0, 255, 0))
|
||||
|
||||
def process(images: list[Image.Image]):
|
||||
return processor(
|
||||
placeholder * len(images),
|
||||
mm_items=processor.info.parse_mm_data({"image": images}),
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
query = process([query_image])
|
||||
document = process([document_image])
|
||||
pair = process([query_image, document_image])
|
||||
|
||||
pair_items = pair["mm_kwargs"]["image"]
|
||||
assert pair["mm_hashes"]["image"] == [
|
||||
document["mm_hashes"]["image"][0],
|
||||
query["mm_hashes"]["image"][0],
|
||||
]
|
||||
assert batched_tensors_equal(
|
||||
pair_items[0].get_data(),
|
||||
document["mm_kwargs"]["image"][0].get_data(),
|
||||
)
|
||||
assert batched_tensors_equal(
|
||||
pair_items[1].get_data(),
|
||||
query["mm_kwargs"]["image"][0].get_data(),
|
||||
)
|
||||
assert [item.length for item in pair["mm_placeholders"]["image"]] == [
|
||||
document["mm_placeholders"]["image"][0].length,
|
||||
query["mm_placeholders"]["image"][0].length,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"),
|
||||
[
|
||||
({}, 1426, (5704, 1176)),
|
||||
({"min_pixels": 64**2, "max_pixels": 512**2}, 330, (1320, 1176)),
|
||||
(
|
||||
{
|
||||
"size": {
|
||||
"shortest_edge": 64**2,
|
||||
"longest_edge": 512**2,
|
||||
},
|
||||
},
|
||||
330,
|
||||
(1320, 1176),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [True, False])
|
||||
def test_processor_override(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
expected_toks_per_img: int,
|
||||
expected_pixels_shape: tuple[int, int],
|
||||
num_imgs: int,
|
||||
kwargs_on_init: bool,
|
||||
):
|
||||
"""Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
|
||||
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
|
||||
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
# Ensure we have the right number of placeholders per num_crops size
|
||||
hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token)
|
||||
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
|
||||
pixel_shape = processed_inputs["mm_kwargs"].get_data()["pixel_values"].shape
|
||||
|
||||
assert img_tok_count == expected_toks_per_img * num_imgs
|
||||
assert pixel_shape[0] == expected_pixels_shape[0] * num_imgs
|
||||
assert pixel_shape[1] == expected_pixels_shape[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
|
||||
@pytest.mark.parametrize(
|
||||
"mm_processor_kwargs",
|
||||
[
|
||||
{"min_pixels": 28 * 28, "max_pixels": 1280 * 28 * 28},
|
||||
{"min_pixels": 28 * 28, "max_pixels": 1283 * 28 * 28},
|
||||
{"size": {"shortest_edge": 28 * 28, "longest_edge": 1280 * 28 * 28}},
|
||||
{"size": {"shortest_edge": 28 * 28, "longest_edge": 1283 * 28 * 28}},
|
||||
],
|
||||
)
|
||||
def test_get_image_size_with_most_features(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
|
||||
merge_size = processor.info.get_hf_config().vision_config.spatial_merge_size
|
||||
|
||||
max_image_size = processor.info.get_image_size_with_most_features()
|
||||
max_tokens = processor.info.get_num_image_tokens(
|
||||
image_width=max_image_size.width,
|
||||
image_height=max_image_size.height,
|
||||
image_processor=hf_processor.image_processor,
|
||||
mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
prompt = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
for asset in image_assets:
|
||||
mm_data = {"image": [asset.pil_image]}
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
grid_thw = processed_inputs["mm_kwargs"].get_data()["image_grid_thw"].tolist()
|
||||
t, h, w = grid_thw[0]
|
||||
tokens = (t * h * w) // (merge_size**2)
|
||||
assert tokens < max_tokens
|
||||
|
||||
|
||||
def _build_qwen2_5_vl_video_mm_data(num_frames: int, fps: float) -> dict[str, Any]:
|
||||
video = np.zeros((num_frames, 56, 56, 3), dtype=np.uint8)
|
||||
metadata = {
|
||||
"fps": fps,
|
||||
"duration": num_frames / fps,
|
||||
"total_num_frames": num_frames,
|
||||
"frames_indices": list(range(num_frames)),
|
||||
"video_backend": "opencv",
|
||||
"do_sample_frames": False,
|
||||
}
|
||||
return {"video": [(video, metadata)]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2.5-VL-3B-Instruct"])
|
||||
def test_qwen2_5_vl_video_fps_to_second_per_grid_ts(model_id: str) -> None:
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"image": 0, "video": 1})
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
temporal_patch_size = (
|
||||
processor.info.get_hf_processor().video_processor.temporal_patch_size
|
||||
)
|
||||
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
for fps in (1.0, 30.0):
|
||||
mm_data = _build_qwen2_5_vl_video_mm_data(num_frames=32, fps=fps)
|
||||
processed = processor(prompt, mm_items=processor.info.parse_mm_data(mm_data))
|
||||
spg = processed["mm_kwargs"]["video"][0].get_data()["second_per_grid_ts"]
|
||||
assert float(spg) == pytest.approx(temporal_patch_size / fps, rel=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize(
|
||||
"model_id", ["Qwen/Qwen2-VL-2B-Instruct", "Qwen/Qwen2.5-VL-3B-Instruct"]
|
||||
)
|
||||
@pytest.mark.parametrize("num_imgs", [1, 2])
|
||||
def test_mm_device_do_normalize(
|
||||
image_assets: ImageTestAssets, model_id: str, num_imgs: int
|
||||
) -> None:
|
||||
"""Device-side normalisation must reproduce the on-CPU processor result.
|
||||
|
||||
Runs on any platform: the CPU platform exercises the ``forward_native``
|
||||
fallback, accelerators exercise the fused kernel.
|
||||
"""
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": num_imgs},
|
||||
)
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Build the image str / prompt based on the number of images we pass
|
||||
prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
|
||||
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
|
||||
|
||||
processed_inputs_with_normalize = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
)
|
||||
pixel_values_with_normalize = processed_inputs_with_normalize[
|
||||
"mm_kwargs"
|
||||
].get_data()["pixel_values"]
|
||||
dtype = pixel_values_with_normalize.dtype
|
||||
|
||||
processed_inputs_without_normalize = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={"do_normalize": False, "do_rescale": False},
|
||||
)
|
||||
pixel_values_without_normalize = processed_inputs_without_normalize[
|
||||
"mm_kwargs"
|
||||
].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
|
||||
# With normalisation disabled, the processor emits raw uint8 pixels,
|
||||
# matching the production mm_device_do_normalize path.
|
||||
assert pixel_values_without_normalize.dtype == torch.uint8
|
||||
pixel_values_do_input_norm = input_norm(
|
||||
pixel_values_without_normalize.to(device), dtype
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
pixel_values_with_normalize.to(device), pixel_values_do_input_norm
|
||||
)
|
||||
@@ -1,604 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for Qwen3-VL multimodal processing."""
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.processing.context import BaseProcessingInfo
|
||||
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import build_model_context
|
||||
|
||||
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
|
||||
|
||||
|
||||
def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
|
||||
assert "size" in native_video_kwargs
|
||||
assert "min_pixels" not in native_video_kwargs
|
||||
assert "max_pixels" not in native_video_kwargs
|
||||
|
||||
info = Qwen3VLProcessingInfo(SimpleNamespace())
|
||||
parent_supported = {
|
||||
"text_kwargs": {"padding"},
|
||||
"images_kwargs": {"size", "min_pixels", "max_pixels"},
|
||||
"videos_kwargs": set(native_video_kwargs),
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
BaseProcessingInfo,
|
||||
"get_supported_mm_processor_kwargs",
|
||||
lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
|
||||
)
|
||||
|
||||
supported = info.supported_mm_processor_kwargs
|
||||
|
||||
assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
|
||||
assert parent_supported["videos_kwargs"] == native_video_kwargs
|
||||
|
||||
|
||||
def _build_video_mm_data(
|
||||
num_frames: int,
|
||||
width: int = 128,
|
||||
height: int = 128,
|
||||
original_fps: float = 30.0,
|
||||
) -> dict[str, Any]:
|
||||
"""Create synthetic video data with metadata indicating that
|
||||
HF processor should re-sample frames (do_sample_frames=True).
|
||||
|
||||
``total_num_frames`` is set equal to the ndarray frame count so
|
||||
that HF's ``sample_frames`` indices stay within bounds of the
|
||||
actual tensor that is passed."""
|
||||
video = np.zeros((num_frames, height, width, 3), dtype=np.uint8)
|
||||
metadata = {
|
||||
"fps": original_fps,
|
||||
"duration": num_frames / original_fps,
|
||||
"total_num_frames": num_frames,
|
||||
"frames_indices": list(range(num_frames)),
|
||||
"video_backend": "opencv",
|
||||
"do_sample_frames": True,
|
||||
}
|
||||
return {"video": [(video, metadata)]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_partial_scoped_image_size(model_id: str) -> None:
|
||||
"""Partial image ``size`` must be completed and applied before the HF call.
|
||||
|
||||
Keep video compatibility and per-video kwargs in an image-only call as well:
|
||||
Qwen3 must sanitize the video scope before HF validates it. Regression for
|
||||
#56363.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1, "video": 0},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
image = np.zeros((128, 128, 3), dtype=np.uint8)
|
||||
min_pixels = 256 * 32 * 32
|
||||
default_size = dict(processor.info.get_image_processor().size)
|
||||
complete_size = default_size | {"shortest_edge": min_pixels}
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data({"image": [image]})
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"image_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
complete_grid = process({"images_kwargs": {"size": complete_size}})
|
||||
partial_grid = process(
|
||||
{
|
||||
"images_kwargs": {
|
||||
"size": {"shortest_edge": min_pixels},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"min_pixels": 4096,
|
||||
"max_pixels": 469_762_048,
|
||||
"fps": [2.0],
|
||||
"num_frames": [8],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(partial_grid, complete_grid)
|
||||
assert not torch.equal(partial_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_partial_scoped_video_size(model_id: str) -> None:
|
||||
"""Partial scoped video ``size`` must be completed and applied before HF."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
min_pixels = 512 * 32 * 32
|
||||
default_size = dict(processor.info.get_video_processor().size)
|
||||
complete_size = default_size | {"shortest_edge": min_pixels}
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"video_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
complete_grid = process({"videos_kwargs": {"size": complete_size}})
|
||||
partial_grid = process(
|
||||
{
|
||||
"videos_kwargs": {
|
||||
"size": {"shortest_edge": min_pixels},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(partial_grid, complete_grid)
|
||||
assert not torch.equal(partial_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
|
||||
"""Flat min/max pixel aliases keep their historical Qwen3 video effect."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
pixel_budget = 512 * 32 * 32
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"video_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
scoped_grid = process(
|
||||
{
|
||||
"videos_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": pixel_budget,
|
||||
"longest_edge": pixel_budget,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
flat_grid = process(
|
||||
{
|
||||
"min_pixels": pixel_budget,
|
||||
"max_pixels": pixel_budget,
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(flat_grid, scoped_grid)
|
||||
assert not torch.equal(flat_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"num_frames",
|
||||
[8, 16],
|
||||
)
|
||||
def test_processor_num_frames_timestamp(
|
||||
model_id: str,
|
||||
num_frames: int,
|
||||
) -> None:
|
||||
"""Regression test: using ``num_frames`` (without ``fps``) must not
|
||||
cause a timestamp / token-count mismatch.
|
||||
|
||||
Before the fix, ``_get_video_second_idx`` ignored the explicit
|
||||
``num_frames`` and fell back to an fps-based calculation, which
|
||||
produced a different number of timestamp entries and ultimately led
|
||||
to shape mismatches in downstream token construction.
|
||||
|
||||
We deliberately choose ``num_frames`` values (8, 16) that differ
|
||||
from what the default fps-based path would compute (which clamps
|
||||
to ``min_frames=4`` for a short video at 30 fps), so this test
|
||||
would fail without the fix.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
mm_data = _build_video_mm_data(num_frames=num_frames)
|
||||
|
||||
# Process with explicit num_frames (no fps) -- this is the path
|
||||
# that was broken before the fix.
|
||||
hf_mm_kwargs: dict[str, Any] = {"num_frames": num_frames}
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_mm_kwargs,
|
||||
)
|
||||
|
||||
# Basic sanity: the processor must produce video tokens.
|
||||
token_ids = processed["prompt_token_ids"]
|
||||
assert len(token_ids) > 0, "Processor produced empty token list"
|
||||
|
||||
# Verify that video placeholders were actually inserted.
|
||||
assert "mm_placeholders" in processed
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == 1, (
|
||||
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize("num_videos", [2, 4])
|
||||
def test_processor_multi_video(
|
||||
model_id: str,
|
||||
num_videos: int,
|
||||
) -> None:
|
||||
"""Verify that multi-video processing produces correct placeholders.
|
||||
|
||||
This exercises the token-level replacement path in
|
||||
``_apply_hf_processor_main`` which avoids the quadratic text-level
|
||||
prompt expansion.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": num_videos},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * num_videos
|
||||
mm_data = {"video": [_build_video_mm_data(num_frames=8)["video"][0]] * num_videos}
|
||||
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={"num_frames": 8},
|
||||
)
|
||||
|
||||
token_ids = processed["prompt_token_ids"]
|
||||
assert len(token_ids) > 0
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == num_videos, (
|
||||
f"Expected {num_videos} video placeholders, got {len(video_phs)}"
|
||||
)
|
||||
|
||||
# All placeholders should have the same length (same video params)
|
||||
# and must not overlap.
|
||||
lengths = {ph.length for ph in video_phs}
|
||||
assert len(lengths) == 1, f"Placeholder lengths differ: {lengths}"
|
||||
for i in range(1, len(video_phs)):
|
||||
prev_end = video_phs[i - 1].offset + video_phs[i - 1].length
|
||||
assert video_phs[i].offset >= prev_end, (
|
||||
f"Placeholder {i} overlaps with placeholder {i - 1}"
|
||||
)
|
||||
|
||||
|
||||
# Qwen3-VL / Qwen3.8 "Long Video Understanding" pixel budget from the
|
||||
# model card. Used to check --mm-processor-kwargs scoping (#52834).
|
||||
_LONG_VIDEO_SIZE = {"longest_edge": 469762048, "shortest_edge": 4096}
|
||||
|
||||
|
||||
def _probe_mm_token_budgets(
|
||||
model_id: str, mm_processor_kwargs: dict[str, Any] | None
|
||||
) -> tuple[int, int]:
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
)
|
||||
info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info
|
||||
video = info.get_max_video_tokens(
|
||||
seq_len=500000, mm_counts={"video": 1, "image": 1}
|
||||
)
|
||||
return video, info.get_max_image_tokens()
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
|
||||
model_id: str,
|
||||
) -> None:
|
||||
"""``videos_kwargs.size`` must raise only the video token budget.
|
||||
|
||||
A flat ``size`` override still applies to both modalities (the previous
|
||||
shared-namespace behavior). Regression for #52834.
|
||||
"""
|
||||
stock_video, stock_image = _probe_mm_token_budgets(model_id, None)
|
||||
scoped_video, scoped_image = _probe_mm_token_budgets(
|
||||
model_id, {"videos_kwargs": {"size": _LONG_VIDEO_SIZE}}
|
||||
)
|
||||
flat_video, flat_image = _probe_mm_token_budgets(
|
||||
model_id, {"size": _LONG_VIDEO_SIZE}
|
||||
)
|
||||
|
||||
assert scoped_video == flat_video
|
||||
assert scoped_video > stock_video
|
||||
assert scoped_image == stock_image
|
||||
assert flat_image > stock_image
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
("hf_mm_kwargs", "scalar_kwargs"),
|
||||
[
|
||||
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
|
||||
({"fps": [2.0, 4.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
|
||||
({"num_frames": [8, 16, 32]}, [{"num_frames": 8}, {"num_frames": 16}]),
|
||||
({"fps": [2.0, 4.0, 8.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_kwargs", [False, True])
|
||||
@pytest.mark.parametrize("use_uuids", [False, True])
|
||||
@pytest.mark.parametrize("scoped", [False, True])
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_processor_multi_video_list_kwargs(
|
||||
model_id: str,
|
||||
hf_mm_kwargs: dict[str, Any],
|
||||
scalar_kwargs: list[dict[str, Any]],
|
||||
model_kwargs: bool,
|
||||
use_uuids: bool,
|
||||
scoped: bool,
|
||||
) -> None:
|
||||
"""Per-video options must survive duplicate items and partial cache hits."""
|
||||
if scoped:
|
||||
hf_mm_kwargs = {"videos_kwargs": hf_mm_kwargs}
|
||||
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 2},
|
||||
mm_processor_kwargs=hf_mm_kwargs if model_kwargs else None,
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
|
||||
videos = [
|
||||
_build_video_mm_data(num_frames=32, original_fps=8.0)["video"][0]
|
||||
for _ in range(3)
|
||||
]
|
||||
for i, (frames, _) in enumerate(videos):
|
||||
frames.fill(i)
|
||||
|
||||
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
single = processor(
|
||||
single_prompt,
|
||||
mm_items=processor.info.parse_mm_data({"video": [videos[0]]}),
|
||||
hf_processor_mm_kwargs=hf_kwargs,
|
||||
)
|
||||
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
|
||||
|
||||
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
|
||||
assert not torch.equal(expected_grids[0], expected_grids[1])
|
||||
|
||||
def run(indices, *, cache=None, cached_only=False):
|
||||
return processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(
|
||||
{"video": [None if cached_only else videos[i] for i in indices]}
|
||||
),
|
||||
mm_uuid_items=(
|
||||
{"video": [f"video-{i}" for i in indices]} if use_uuids else None
|
||||
),
|
||||
hf_processor_mm_kwargs={} if model_kwargs else hf_mm_kwargs,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
for indices in ([0, 0], [0, 1], [0, 2], [2, 0]):
|
||||
expected = run(indices)
|
||||
actual = run(indices, cache=cache)
|
||||
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
|
||||
assert actual["mm_placeholders"] == expected["mm_placeholders"]
|
||||
assert actual["mm_kwargs"] == expected["mm_kwargs"]
|
||||
assert actual["mm_hashes"] == expected["mm_hashes"]
|
||||
assert len(set(actual["mm_hashes"]["video"])) == 2
|
||||
multi_grids = actual["mm_kwargs"].get_data()["video_grid_thw"]
|
||||
assert torch.equal(multi_grids[0], expected_grids[0])
|
||||
assert torch.equal(multi_grids[1], expected_grids[1])
|
||||
|
||||
if use_uuids:
|
||||
cached_only = run([2, 0], cache=cache, cached_only=True)
|
||||
assert cached_only["prompt_token_ids"] == actual["prompt_token_ids"]
|
||||
assert cached_only["mm_kwargs"] == actual["mm_kwargs"]
|
||||
assert hf_mm_kwargs == hf_mm_kwargs_before
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("key", "value"), [("fps", 2.0), ("num_frames", 8)])
|
||||
@pytest.mark.parametrize("length", [0, 1])
|
||||
@pytest.mark.parametrize("use_cache", [False, True])
|
||||
@pytest.mark.parametrize("scoped", [False, True])
|
||||
def test_processor_rejects_short_per_video_options(
|
||||
key: str, value: float | int, length: int, use_cache: bool, scoped: bool
|
||||
) -> None:
|
||||
"""Reject malformed option lists with and without a warm processor cache."""
|
||||
ctx = build_model_context(
|
||||
MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 0, "video": 2},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config) if use_cache else None
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
|
||||
videos = _build_video_mm_data(num_frames=32, original_fps=8.0)["video"] * 2
|
||||
|
||||
def process(values: list[float | int]) -> None:
|
||||
kwargs = {key: values}
|
||||
processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"video": videos}),
|
||||
hf_processor_mm_kwargs={"videos_kwargs": kwargs} if scoped else kwargs,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
if use_cache:
|
||||
process([value, value])
|
||||
|
||||
with pytest.raises(ValueError, match=f"{key} must have at least 2 entries"):
|
||||
process([value] * length)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kwargs", [{"fps": [2.0, 4.0]}, {"num_frames": [8, 16]}])
|
||||
def test_processor_profiles_fewer_videos_than_configured_options(
|
||||
kwargs: dict[str, list[float] | list[int]],
|
||||
) -> None:
|
||||
"""Profiling may use fewer videos than the configured per-video list."""
|
||||
ctx = build_model_context(
|
||||
MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 0, "video": 2},
|
||||
mm_processor_kwargs=kwargs,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
profiled = processor.get_dummy_mm_inputs({"video": 1})
|
||||
|
||||
assert len(profiled["mm_placeholders"]["video"]) == 1
|
||||
assert profiled["mm_kwargs"].get_data()["video_grid_thw"].shape[0] == 1
|
||||
|
||||
|
||||
def _build_video_embeds_mm_data(
|
||||
hidden_size: int,
|
||||
grid_thw: tuple[int, int, int] = (2, 4, 4),
|
||||
) -> dict[str, Any]:
|
||||
"""Create an embeds-only video item as an EPD consumer receives it:
|
||||
pre-computed embeddings plus the metadata published by the encoder."""
|
||||
import torch
|
||||
|
||||
t, h, w = grid_thw
|
||||
num_tokens = t * h * w // 4 # spatial_merge_size ** 2
|
||||
return {
|
||||
"video": {
|
||||
"video_embeds": torch.zeros(num_tokens, hidden_size),
|
||||
"video_grid_thw": torch.tensor([grid_thw]),
|
||||
"timestamps": torch.tensor([[float(i) for i in range(t)]]),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
|
||||
"""Embeds-only video input must size the placeholder range from the
|
||||
grid and the real timestamps published by the encoder (EC consumer
|
||||
path); synthesized or missing timestamps would change the token count
|
||||
and break embedding merging downstream."""
|
||||
# `build_model_context` forces enable_mm_embeds off, so build the
|
||||
# config directly; keep the same online-availability skip behavior.
|
||||
HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip")
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
runner="generate",
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
enable_mm_embeds=True,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
|
||||
grid_thw = (2, 4, 4)
|
||||
mm_data = _build_video_embeds_mm_data(
|
||||
model_config.get_inputs_embeds_size(), grid_thw
|
||||
)
|
||||
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == 1, (
|
||||
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
|
||||
)
|
||||
|
||||
t, h, w = grid_thw
|
||||
tokens_per_frame = h * w // 4
|
||||
timestamp_tokens = sum(
|
||||
len(tokenizer.encode(f"<{float(i):.1f} seconds>", add_special_tokens=False))
|
||||
for i in range(t)
|
||||
)
|
||||
# per frame: timestamp tokens + vision_start + video tokens + vision_end
|
||||
expected_len = timestamp_tokens + t * (tokens_per_frame + 2)
|
||||
assert video_phs[0].length == expected_len
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_video_embeds_missing_timestamps(model_id: str) -> None:
|
||||
"""Timestamps are required metadata for video embeds: they size the
|
||||
placeholder range, so omitting them must fail loudly at parse time
|
||||
instead of silently producing a wrong prompt."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
mm_data = _build_video_embeds_mm_data(ctx.model_config.get_inputs_embeds_size())
|
||||
del mm_data["video"]["timestamps"]
|
||||
|
||||
with pytest.raises(ValueError, match="timestamps"):
|
||||
processor.info.parse_mm_data(mm_data)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None:
|
||||
"""Regression test for memory profiling with ``cap_pixels_per_frame``.
|
||||
|
||||
With the HF per-frame pixel cap enabled (transformers#48071) and a
|
||||
raised video budget, a 2-frame profiling dummy would be processed at
|
||||
only 2 * cap pixels, underestimating the largest possible video (a
|
||||
fully sampled one still fills the whole ``longest_edge`` budget). The
|
||||
dummy builder must spread the budget over enough frames that the cap
|
||||
is not binding.
|
||||
"""
|
||||
# 16 capped frames' worth of budget on top of the default per-frame
|
||||
# ceiling (max_video_tokens=768 at patch 16, merge 2 -> 786,432
|
||||
# pixels per frame).
|
||||
per_frame_cap = 768 * (16 * 2) ** 2
|
||||
budget = per_frame_cap * 16
|
||||
size = {"longest_edge": budget, "shortest_edge": 4096}
|
||||
|
||||
capped_ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={
|
||||
"videos_kwargs": {
|
||||
"size": size,
|
||||
"cap_pixels_per_frame": True,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
|
||||
capped_dummy = capped.dummy_inputs.get_dummy_mm_data(
|
||||
1024, {"video": 1}, MultiModalDummyOptions()
|
||||
)
|
||||
capped_frames = capped_dummy["video"][0][0].shape[0]
|
||||
assert capped_frames == 16, (
|
||||
f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}"
|
||||
)
|
||||
|
||||
uncapped_ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"videos_kwargs": {"size": size}},
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
|
||||
uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(
|
||||
1024, {"video": 1}, MultiModalDummyOptions()
|
||||
)
|
||||
uncapped_frames = uncapped_dummy["video"][0][0].shape[0]
|
||||
assert uncapped_frames == 2, (
|
||||
f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}"
|
||||
)
|
||||
@@ -1,58 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Step3-VL precomputed image embedding inputs."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.step3_vl import (
|
||||
Step3VLForConditionalGeneration,
|
||||
Step3VLImageEmbeddingInputs,
|
||||
)
|
||||
|
||||
|
||||
class _FakeStep3VL:
|
||||
@staticmethod
|
||||
def _process_image_features(image_features: torch.Tensor) -> torch.Tensor:
|
||||
return image_features
|
||||
|
||||
|
||||
def test_image_embedding_inputs_construction():
|
||||
"""Step3VLImageEmbeddingInputs should store embeddings in the data field."""
|
||||
image_embeds = torch.randn(2, 16, 64)
|
||||
|
||||
inputs = Step3VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
data=image_embeds,
|
||||
)
|
||||
|
||||
assert inputs["type"] == "image_embeds"
|
||||
assert torch.equal(inputs["data"], image_embeds)
|
||||
assert torch.equal(inputs.data, image_embeds)
|
||||
|
||||
|
||||
def test_image_embedding_inputs_validation_rejects_wrong_rank():
|
||||
"""Validation should reject tensors with wrong rank."""
|
||||
with pytest.raises(ValueError, match="rank"):
|
||||
Step3VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
data=torch.randn(16, 64),
|
||||
)
|
||||
|
||||
|
||||
def test_process_image_embeds_does_not_require_pixel_input_fields():
|
||||
"""The image_embeds branch should not reference patch pixel metadata."""
|
||||
image_embeds = torch.randn(2, 4, 8)
|
||||
image_input = Step3VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
data=image_embeds,
|
||||
)
|
||||
|
||||
outputs = Step3VLForConditionalGeneration._process_image_input(
|
||||
_FakeStep3VL(),
|
||||
image_input,
|
||||
)
|
||||
|
||||
assert len(outputs) == 2
|
||||
assert torch.equal(outputs[0], image_embeds[0])
|
||||
assert torch.equal(outputs[1], image_embeds[1])
|
||||
Reference in New Issue
Block a user