[Bugfix][Multimodal] Fix flat/scoped mm_processor_kwargs merge and resolution (#56372)

Signed-off-by: Daniel M. García-Ocaña Hernández <danielgarciaocana@gmail.com>
This commit is contained in:
Daniel M. García-Ocaña Hernández
2026-09-29 03:58:39 +00:00
committed by GitHub
parent 491f44adfa
commit 295a0b8d64
11 changed files with 1606 additions and 192 deletions
+244
View File
@@ -188,6 +188,250 @@ def test_mm_processor_device_type_normalizes(device: object, expected: str | Non
assert config.get_mm_processor_device_type() == expected
def test_merge_mm_processor_kwargs_scoped_precedence():
configured = {
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
"do_resize": True,
"padding": False,
"images_kwargs": {
"size": {"longest_edge": 1024},
"do_resize": False,
},
"videos_kwargs": {
"size": {"longest_edge": 2048},
},
}
inference = {
"size": {"shortest_edge": 128},
"do_resize": True,
"images_kwargs": {
"size": {"shortest_edge": 256},
},
}
config = MultiModalConfig(
mm_processor_kwargs=configured,
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs(inference) == {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
"do_resize": True,
"padding": False,
"images_kwargs": {
"size": {
"shortest_edge": 256,
"longest_edge": 1024,
},
"do_resize": True,
},
"videos_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 2048,
},
},
}
def test_merge_mm_processor_kwargs_mapping_vs_replacement():
config = MultiModalConfig(
mm_processor_kwargs={
"size": {"shortest_edge": 64},
"images_kwargs": {"size": {"longest_edge": 1024}},
},
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs({"size": 7}) == {
"size": 7,
"images_kwargs": {"size": 7},
}
assert config.merge_mm_processor_kwargs({"size": {}}) == {
"size": {"shortest_edge": 64},
"images_kwargs": {
"size": {
"shortest_edge": 64,
"longest_edge": 1024,
},
},
}
_EMPTY_IMAGE_SCOPE_KWARGS: tuple[dict[str, object], ...] = (
{},
{"images_kwargs": None},
{"images_kwargs": {}},
)
@pytest.mark.parametrize("configured", _EMPTY_IMAGE_SCOPE_KWARGS)
@pytest.mark.parametrize("inference", _EMPTY_IMAGE_SCOPE_KWARGS)
def test_merge_mm_processor_kwargs_empty_scopes_are_absent(
configured: dict[str, object],
inference: dict[str, object],
):
config = MultiModalConfig(
mm_processor_kwargs=configured,
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs(inference) == {}
@pytest.mark.parametrize("empty_inference", _EMPTY_IMAGE_SCOPE_KWARGS)
def test_merge_mm_processor_kwargs_empty_inference_scope_preserves_config_merge(
empty_inference: dict[str, object],
):
config = MultiModalConfig(
mm_processor_kwargs={
"images_kwargs": {
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
},
},
mm_device_do_normalize=False,
)
inference = {
"size": {"shortest_edge": 128},
**empty_inference,
}
assert config.merge_mm_processor_kwargs(inference) == {
"size": {"shortest_edge": 128},
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
},
}
@pytest.mark.parametrize("empty_config", _EMPTY_IMAGE_SCOPE_KWARGS)
def test_merge_mm_processor_kwargs_empty_config_scope_allows_inference_merge(
empty_config: dict[str, object],
):
config = MultiModalConfig(
mm_processor_kwargs={
"size": {"longest_edge": 512},
**empty_config,
},
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs(
{
"size": {"shortest_edge": 128},
"images_kwargs": {"size": {"longest_edge": 1024}},
}
) == {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 1024,
},
},
}
@pytest.mark.parametrize("non_mapping_scope", [False, 0, "", [], 123])
def test_merge_mm_processor_kwargs_falsey_non_mapping_scopes_are_not_empty(
non_mapping_scope: object,
):
config = MultiModalConfig(
mm_processor_kwargs={"images_kwargs": non_mapping_scope},
mm_device_do_normalize=False,
)
for empty_inference in _EMPTY_IMAGE_SCOPE_KWARGS:
assert config.merge_mm_processor_kwargs(empty_inference) == {
"images_kwargs": non_mapping_scope
}
@pytest.mark.parametrize(
("configured_scope", "inference_scope", "expected_scope"),
[
(123, {"size": {"shortest_edge": 128}}, {"size": {"shortest_edge": 128}}),
({"size": {"shortest_edge": 64}}, 123, 123),
(123, 456, 456),
],
)
def test_merge_mm_processor_kwargs_non_mapping_scope_precedence(
configured_scope: object,
inference_scope: object,
expected_scope: object,
):
config = MultiModalConfig(
mm_processor_kwargs={"images_kwargs": configured_scope},
mm_device_do_normalize=False,
)
merged = config.merge_mm_processor_kwargs({"images_kwargs": inference_scope})
assert merged["images_kwargs"] == expected_scope
def test_merge_mm_processor_kwargs_none_leaf_is_explicit():
config = MultiModalConfig(mm_device_do_normalize=False)
assert config.merge_mm_processor_kwargs({"images_kwargs": {"do_resize": None}}) == {
"images_kwargs": {"do_resize": None}
}
def test_merge_mm_processor_kwargs_normalize_and_ownership():
configured: dict[str, object] = {
"images_kwargs": {
"size": {"shortest_edge": 64},
},
}
inference: dict[str, object] = {
"do_normalize": True,
"do_rescale": True,
"images_kwargs": {
"size": {"longest_edge": 1024},
},
}
configured_before = copy.deepcopy(configured)
inference_before = copy.deepcopy(inference)
config = MultiModalConfig(
mm_processor_kwargs=configured,
mm_device_do_normalize=True,
)
merged = config.merge_mm_processor_kwargs(inference)
assert merged["do_normalize"] is False
assert merged["do_rescale"] is False
assert configured == configured_before
assert inference == inference_before
assert config.mm_processor_kwargs == configured_before
images_kwargs = merged["images_kwargs"]
assert isinstance(images_kwargs, dict)
size = images_kwargs["size"]
assert isinstance(size, dict)
size["shortest_edge"] = 999
assert configured == configured_before
assert inference == inference_before
assert config.mm_processor_kwargs == configured_before
def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None:
ec_config = (
None
@@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
@pytest.mark.parametrize("kwargs_on_init", [False, True])
@pytest.mark.parametrize(
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
)
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
def test_video_cache_is_independent_of_image_kwargs(
model_id: str,
kwargs_on_init: bool,
image_kwargs: dict[str, object],
video_uuid: str | None,
):
"""Image overrides must not change video frames or depend on cache warmth."""
kwargs = {"images_kwargs": image_kwargs}
"""Request image overrides must not change video frames or make results
depend on cache warmth.
"""
request_kwargs = {"images_kwargs": image_kwargs}
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 1},
@@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs(
cache=cache,
)
baseline = process({}, cache=None)
if kwargs_on_init:
ctx = build_model_context(
model_id,
mm_processor_kwargs=kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
mm_processor_cache_gb=1,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
request_kwargs = {} if kwargs_on_init else kwargs
process(request_kwargs, cache=cache)
# Populate the video cache without image overrides, then change only
# images_kwargs on the next request. Since images_kwargs are excluded from
# the video hash, and must not affect the processed video, the cached result
# must match a fresh recomputation.
baseline = process({}, cache=cache)
cached = process(request_kwargs, cache=cache)
cache.clear_cache()
fresh = process(request_kwargs, cache=cache)
@@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image(
[
({}, (448, 448), 256),
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
(
{"images_kwargs": {"size": {"longest_edge": 1008}}},
(1540, 1540),
1296,
),
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
({"size": {"longest_edge": 1008}}, (29, 29), 4),
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
@@ -148,6 +153,40 @@ def test_processor_size_override(
assert prompt_update_tokens == hf_placeholder_tokens
def test_scoped_request_size_overrides_configured_flat_size():
ctx = build_model_context(
_MODEL_ID,
mm_processor_kwargs={"size": {"longest_edge": 1008}},
limit_mm_per_prompt={"image": 1},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = AutoProcessor.from_pretrained(
_MODEL_ID,
fix_mistral_regex=True,
)
hf_processor_mm_kwargs: dict[str, object] = {
"images_kwargs": {"size": {"longest_edge": 1288}}
}
image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
hf_processor, [image], merged_mm_kwargs
)
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
processor, [image], pixel_values, hf_processor_mm_kwargs
)
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
hf_processor, pixel_values[0]
)
assert expected_from_pixel_values == 1656
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
assert prompt_update_tokens == hf_placeholder_tokens
def test_lightonocr_keeps_vision_config_image_size():
ctx = build_model_context(
_LIGHTON_MODEL_ID,
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
@@ -8,16 +10,238 @@ import pytest
import torch
from PIL import Image
from vllm.config.multimodal import MultiModalConfig
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.multimodal.processing.context import InputProcessingContext
from vllm.platforms import current_platform
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize(
("mm_kwargs", "expected"),
[
(
{"size": {"shortest_edge": 64, "longest_edge": 1024}},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 64,
"max_pixels": 1024,
},
),
(
{"min_pixels": 64, "max_pixels": 1024},
{
"min_pixels": 64,
"max_pixels": 1024,
"size": {"shortest_edge": 64, "longest_edge": 1024},
},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
),
(
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {"max_pixels": 4096},
"videos_kwargs": {"size": {"longest_edge": 8192}},
},
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {
"max_pixels": 4096,
"size": {"longest_edge": 4096},
},
"videos_kwargs": {
"size": {"longest_edge": 8192},
"max_pixels": 8192,
},
},
),
(
{"min_pixels": 0},
{"min_pixels": 0, "size": {"shortest_edge": 0}},
),
(
{"size": None, "min_pixels": 64},
{"size": {"shortest_edge": 64}, "min_pixels": 64},
),
],
)
def test_complete_mm_processor_size_aliases(
mm_kwargs: dict[str, object],
expected: dict[str, object],
) -> None:
original = deepcopy(mm_kwargs)
actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
assert actual == expected
assert mm_kwargs == original
def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
mm_config = MultiModalConfig(
mm_processor_kwargs={
"images_kwargs": {"min_pixels": 64},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
info = Qwen2VLProcessingInfo(ctx)
monkeypatch.setattr(
info,
"get_supported_mm_processor_kwargs",
lambda: {"images_kwargs": {"size", "min_pixels"}},
)
assert info._merge_and_resolve_mm_processor_kwargs(
{"size": {"shortest_edge": 128}}
) == {
"images_kwargs": {
"size": {"shortest_edge": 128},
"min_pixels": 128,
},
}
@pytest.mark.parametrize(
("mm_kwargs", "default_size", "expected"),
[
(
{},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 32, "longest_edge": 1024},
),
(
{"size": {"shortest_edge": 64}},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 1024},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": 128,
"max_pixels": 4096,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 128, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64},
"max_pixels": 4096,
},
None,
{"shortest_edge": 64, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": None,
"max_pixels": 0,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 0},
),
],
)
def test_get_vision_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None,
expected: dict[str, object],
) -> None:
original_mm_kwargs = {
key: dict(value) if isinstance(value, dict) else value
for key, value in mm_kwargs.items()
}
original_default_size = dict(default_size) if default_size is not None else None
actual = Qwen2VLProcessingInfo._get_vision_size(
mm_kwargs,
default_size=default_size,
)
assert actual == expected
assert mm_kwargs == original_mm_kwargs
assert default_size == original_default_size
@pytest.mark.parametrize(
("modality", "scope"),
[
("image", "images_kwargs"),
("video", "videos_kwargs"),
],
)
def test_get_vision_info_reads_merged_modality_scope(
monkeypatch: pytest.MonkeyPatch,
modality: str,
scope: str,
) -> None:
vision_config = SimpleNamespace(
patch_size=14,
spatial_merge_size=2,
temporal_patch_size=2,
)
ctx = SimpleNamespace(
get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
)
info = Qwen2VLProcessingInfo(ctx)
merged = {
"images_kwargs": {"size": {"shortest_edge": 111}},
"videos_kwargs": {"size": {"shortest_edge": 222}},
}
monkeypatch.setattr(
info,
"_merge_and_resolve_mm_processor_kwargs",
lambda _: merged,
)
seen: list[dict[str, object]] = []
def get_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None = None,
) -> dict[str, object]:
seen.append(mm_kwargs)
return dict(default_size or {})
monkeypatch.setattr(info, "_get_vision_size", get_size)
image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
info._get_vision_info(
image_width=28,
image_height=28,
num_frames=2,
do_resize=False,
image_processor=image_processor,
mm_kwargs={},
modality=modality, # type: ignore[arg-type]
)
assert seen == [merged[scope]]
def test_jina_vl_processing_order() -> None:
"""Jina's document-first prompt keeps cached features and hashes aligned."""
ctx = build_model_context(
@@ -1,19 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for Qwen3-VL processor.
Covers the fix for num_frames-based timestamp calculation
(issue vllm-project/vllm#35909).
"""
"""Regression tests for Qwen3-VL multimodal processing."""
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
import torch
from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
from vllm.config import ModelConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.processing.context import BaseProcessingInfo
from ...registry import HF_EXAMPLE_MODELS
from ...utils import build_model_context
@@ -21,6 +23,32 @@ from ...utils import build_model_context
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
assert "size" in native_video_kwargs
assert "min_pixels" not in native_video_kwargs
assert "max_pixels" not in native_video_kwargs
info = Qwen3VLProcessingInfo(SimpleNamespace())
parent_supported = {
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels", "max_pixels"},
"videos_kwargs": set(native_video_kwargs),
}
monkeypatch.setattr(
BaseProcessingInfo,
"get_supported_mm_processor_kwargs",
lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
)
supported = info.supported_mm_processor_kwargs
assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
assert parent_supported["videos_kwargs"] == native_video_kwargs
def _build_video_mm_data(
num_frames: int,
width: int = 128,
@@ -45,6 +73,122 @@ def _build_video_mm_data(
return {"video": [(video, metadata)]}
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_image_size(model_id: str) -> None:
"""Partial image ``size`` must be completed and applied before the HF call.
Keep video compatibility and per-video kwargs in an image-only call as well:
Qwen3 must sanitize the video scope before HF validates it. Regression for
#56363.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = np.zeros((128, 128, 3), dtype=np.uint8)
min_pixels = 256 * 32 * 32
default_size = dict(processor.info.get_image_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [image]})
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"image_grid_thw"
]
stock_grid = process({})
complete_grid = process({"images_kwargs": {"size": complete_size}})
partial_grid = process(
{
"images_kwargs": {
"size": {"shortest_edge": min_pixels},
},
"videos_kwargs": {
"min_pixels": 4096,
"max_pixels": 469_762_048,
"fps": [2.0],
"num_frames": [8],
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_video_size(model_id: str) -> None:
"""Partial scoped video ``size`` must be completed and applied before HF."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
min_pixels = 512 * 32 * 32
default_size = dict(processor.info.get_video_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
complete_grid = process({"videos_kwargs": {"size": complete_size}})
partial_grid = process(
{
"videos_kwargs": {
"size": {"shortest_edge": min_pixels},
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
"""Flat min/max pixel aliases keep their historical Qwen3 video effect."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
pixel_budget = 512 * 32 * 32
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
scoped_grid = process(
{
"videos_kwargs": {
"size": {
"shortest_edge": pixel_budget,
"longest_edge": pixel_budget,
},
},
}
)
flat_grid = process(
{
"min_pixels": pixel_budget,
"max_pixels": pixel_budget,
}
)
assert torch.equal(flat_grid, scoped_grid)
assert not torch.equal(flat_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"num_frames",
@@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"hf_mm_kwargs",
[{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
("hf_mm_kwargs", "scalar_kwargs"),
[
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
],
)
def test_processor_multi_video_list_kwargs(
model_id: str,
hf_mm_kwargs: dict[str, Any],
scalar_kwargs: list[dict[str, Any]],
) -> None:
"""Regression test: a multi-video request with list-valued per-video
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
crash.
"""List-valued video kwargs must be scalarized per item without mutation.
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
video without slicing, so ``_get_video_second_idx`` received the list
where a scalar was expected and raised ``TypeError``.
Each video in the multi-video request must match processing the same input
independently with its corresponding scalar ``fps``/``num_frames`` value.
"""
ctx = build_model_context(
model_id,
@@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs(
)
mm_data = {
"video": [
_build_video_mm_data(num_frames=16)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
]
}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
single_mm_data = _build_video_mm_data(num_frames=32)
single = processor(
single_prompt,
mm_items=processor.info.parse_mm_data(single_mm_data),
hf_processor_mm_kwargs=hf_kwargs,
)
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
assert not torch.equal(expected_grids[0], expected_grids[1])
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
assert hf_mm_kwargs == hf_mm_kwargs_before
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 2, (
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
@@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
capped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
mm_processor_kwargs={
"videos_kwargs": {
"size": size,
"cap_pixels_per_frame": True,
},
},
limit_mm_per_prompt={"image": 0, "video": 1},
)
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
@@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
uncapped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"size": size},
mm_processor_kwargs={"videos_kwargs": {"size": size}},
limit_mm_per_prompt={"image": 0, "video": 1},
)
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
+295 -43
View File
@@ -3,18 +3,23 @@
import time
from contextlib import nullcontext
from copy import deepcopy
from types import SimpleNamespace
from typing import TypedDict
import numpy as np
import pytest
from vllm.config import ModelConfig, SchedulerConfig
from vllm.config.multimodal import MultiModalConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.parse import MultiModalDataParser
from vllm.multimodal.processing.context import (
BaseProcessingInfo,
InputProcessingContext,
overlay_modality_mm_kwargs,
_resolve_mm_processor_kwargs,
)
from vllm.multimodal.processing.inputs import ProcessorInputs
from vllm.multimodal.processing.processor import (
@@ -1207,58 +1212,305 @@ def test_apply_prompt_updates_falls_back_with_index_targets():
assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image():
"""HF-style videos_kwargs must overlay only when modality is video."""
video_size = {"longest_edge": 469762048, "shortest_edge": 4096}
kwargs = {"videos_kwargs": {"size": video_size}}
assert overlay_modality_mm_kwargs(kwargs, None) == kwargs
assert "size" not in overlay_modality_mm_kwargs(kwargs, "image")
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size
class _TextProcessorKwargs(TypedDict, total=False):
padding: bool
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_flat_size_stays_shared():
"""A flat size override keeps the current shared-namespace behavior."""
size = {"longest_edge": 469762048, "shortest_edge": 4096}
kwargs = {"size": size}
for modality in (None, "image", "video"):
assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size
class _AudioProcessorKwargs(TypedDict, total=False):
sampling_rate: int
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality():
"""A nested videos_kwargs size wins over a flat size for video reads."""
class _ProcessorKwargs(TypedDict, total=False):
text_kwargs: _TextProcessorKwargs
audio_kwargs: _AudioProcessorKwargs
class _ImageProcessorKwargs(TypedDict, total=False):
size: dict[str, int]
min_pixels: int
class _VideoProcessorKwargs(TypedDict, total=False):
size: dict[str, int]
fps: float
@pytest.mark.parametrize(
("supported_mm_limits", "processor", "expected"),
[
(
{"image": None, "video": None},
SimpleNamespace(
valid_processor_kwargs=_ProcessorKwargs,
image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs),
video_processor=SimpleNamespace(valid_kwargs=_VideoProcessorKwargs),
),
{
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels"},
"videos_kwargs": {"size", "fps"},
},
),
(
{"image": None},
SimpleNamespace(
valid_processor_kwargs=_ProcessorKwargs,
image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs),
),
{
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels"},
},
),
(
{"audio": None},
SimpleNamespace(valid_processor_kwargs=_ProcessorKwargs),
{
"text_kwargs": {"padding"},
"audio_kwargs": {"sampling_rate"},
},
),
],
)
def test_get_supported_mm_processor_kwargs_uses_supported_modalities(
monkeypatch: pytest.MonkeyPatch,
supported_mm_limits: dict[str, int | None],
processor: SimpleNamespace,
expected: dict[str, set[str]],
) -> None:
info = BaseProcessingInfo(SimpleNamespace())
info.__dict__["supported_mm_limits"] = supported_mm_limits
monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor)
assert info.get_supported_mm_processor_kwargs() == expected
def test_supported_mm_processor_kwargs_is_cached(
monkeypatch: pytest.MonkeyPatch,
) -> None:
info = BaseProcessingInfo(SimpleNamespace())
expected = {"images_kwargs": {"size"}}
calls = 0
def get_supported_mm_processor_kwargs() -> dict[str, set[str]]:
nonlocal calls
calls += 1
return expected
monkeypatch.setattr(
info,
"get_supported_mm_processor_kwargs",
get_supported_mm_processor_kwargs,
)
first = info.supported_mm_processor_kwargs
second = info.supported_mm_processor_kwargs
assert first is expected
assert second is first
assert calls == 1
def test_resolve_mm_processor_kwargs_routes_supported_flat_kwargs():
kwargs = {
"size": {"longest_edge": 1},
"videos_kwargs": {"size": {"longest_edge": 2}},
"images_kwargs": {"size": {"longest_edge": 3}},
"size": {"shortest_edge": 64},
"padding": True,
"sampling_rate": 16000,
"unknown": "keep",
"images_kwargs": {
"size": {"longest_edge": 1024},
"custom_image_kwarg": "keep",
},
}
kwargs_before = deepcopy(kwargs)
resolved = _resolve_mm_processor_kwargs(
kwargs,
supported_mm_processor_kwargs={
"text_kwargs": {"padding"},
"images_kwargs": {"size"},
"videos_kwargs": {"size"},
"audio_kwargs": {"sampling_rate"},
},
)
assert resolved == {
"unknown": "keep",
"text_kwargs": {"padding": True},
"images_kwargs": {
"size": {
"shortest_edge": 64,
"longest_edge": 1024,
},
"custom_image_kwarg": "keep",
},
"videos_kwargs": {
"size": {"shortest_edge": 64},
},
"audio_kwargs": {"sampling_rate": 16000},
}
assert kwargs == kwargs_before
video_kwargs = resolved["videos_kwargs"]
image_kwargs = resolved["images_kwargs"]
assert isinstance(video_kwargs, dict)
assert isinstance(image_kwargs, dict)
video_size = video_kwargs["size"]
image_size = image_kwargs["size"]
assert isinstance(video_size, dict)
assert isinstance(image_size, dict)
video_size["shortest_edge"] = 999
assert image_size["shortest_edge"] == 64
assert kwargs == kwargs_before
def test_resolve_mm_processor_kwargs_rejects_non_mapping_destination_scope():
with pytest.raises(TypeError, match="images_kwargs"):
_resolve_mm_processor_kwargs(
{
"size": {"shortest_edge": 64},
"images_kwargs": 123,
},
supported_mm_processor_kwargs={
"images_kwargs": {"size"},
},
)
def test_resolve_mm_processor_kwargs_without_schema_deduplicates_existing_scopes():
kwargs = {
"size": {"shortest_edge": 64},
"num_frames": 16,
"fps": 4,
"images_kwargs": {
"size": {"longest_edge": 1024},
},
"videos_kwargs": {
"num_frames": 8,
},
"audio_kwargs": 123,
"common_kwargs": {
"fps": 2,
},
}
kwargs_before = deepcopy(kwargs)
resolved = _resolve_mm_processor_kwargs(kwargs)
assert resolved == {
"fps": 4,
"images_kwargs": {
"size": {"longest_edge": 1024},
},
"videos_kwargs": {
"num_frames": 8,
},
"audio_kwargs": 123,
"common_kwargs": {
"fps": 2,
},
}
assert kwargs == kwargs_before
assert "text_kwargs" not in resolved
@pytest.mark.parametrize(
"inference_kwargs",
[
{"size": {"shortest_edge": 128}},
{"size": {"shortest_edge": 128}, "images_kwargs": None},
{"size": {"shortest_edge": 128}, "images_kwargs": {}},
],
)
def test_get_merged_mm_kwargs_treats_empty_scopes_as_absent_before_routing(
inference_kwargs: dict[str, object],
):
mm_config = MultiModalConfig(
mm_processor_kwargs={
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
"images_kwargs": {
"size": {"longest_edge": 1024},
},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
merged = ctx.get_merged_mm_kwargs(
inference_kwargs,
supported_mm_processor_kwargs={
"images_kwargs": {"size"},
"videos_kwargs": {"size"},
},
)
assert merged == {
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 1024,
},
},
"videos_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
},
}
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2}
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3}
assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1}
def test_get_merged_mm_kwargs_merges_before_routing():
mm_config = MultiModalConfig(
mm_processor_kwargs={
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
"images_kwargs": {
"size": {"longest_edge": 1024},
},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value():
kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}}
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1}
merged = ctx.get_merged_mm_kwargs(
{
"size": {
"shortest_edge": 128,
"longest_edge": 768,
},
"images_kwargs": {
"size": {"longest_edge": 2048},
},
},
supported_mm_processor_kwargs={
"images_kwargs": {"size"},
"videos_kwargs": {"size"},
},
)
@pytest.mark.skip_global_cleanup
def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
"""Configured videos_kwargs overlay only for video reads after merge."""
from vllm.config.multimodal import MultiModalConfig
size = {"longest_edge": 469762048, "shortest_edge": 4096}
mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}})
merged = mm_config.merge_mm_processor_kwargs({})
assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
assert "size" not in overlay_modality_mm_kwargs(merged, "image")
assert "size" not in overlay_modality_mm_kwargs(merged, None)
assert merged == {
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 2048,
},
},
"videos_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 768,
},
},
}
def test_processor_inputs_hashes_partial_uuids():