[Bugfix][Multimodal] Fix flat/scoped mm_processor_kwargs merge and resolution (#56372)

Signed-off-by: Daniel M. García-Ocaña Hernández <danielgarciaocana@gmail.com>
This commit is contained in:
Daniel M. García-Ocaña Hernández
2026-09-29 03:58:39 +00:00
committed by GitHub
parent 491f44adfa
commit 295a0b8d64
11 changed files with 1606 additions and 192 deletions
@@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
@pytest.mark.parametrize("kwargs_on_init", [False, True])
@pytest.mark.parametrize(
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
)
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
def test_video_cache_is_independent_of_image_kwargs(
model_id: str,
kwargs_on_init: bool,
image_kwargs: dict[str, object],
video_uuid: str | None,
):
"""Image overrides must not change video frames or depend on cache warmth."""
kwargs = {"images_kwargs": image_kwargs}
"""Request image overrides must not change video frames or make results
depend on cache warmth.
"""
request_kwargs = {"images_kwargs": image_kwargs}
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 1},
@@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs(
cache=cache,
)
baseline = process({}, cache=None)
if kwargs_on_init:
ctx = build_model_context(
model_id,
mm_processor_kwargs=kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
mm_processor_cache_gb=1,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
request_kwargs = {} if kwargs_on_init else kwargs
process(request_kwargs, cache=cache)
# Populate the video cache without image overrides, then change only
# images_kwargs on the next request. Since images_kwargs are excluded from
# the video hash, and must not affect the processed video, the cached result
# must match a fresh recomputation.
baseline = process({}, cache=cache)
cached = process(request_kwargs, cache=cache)
cache.clear_cache()
fresh = process(request_kwargs, cache=cache)
@@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image(
[
({}, (448, 448), 256),
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
(
{"images_kwargs": {"size": {"longest_edge": 1008}}},
(1540, 1540),
1296,
),
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
({"size": {"longest_edge": 1008}}, (29, 29), 4),
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
@@ -148,6 +153,40 @@ def test_processor_size_override(
assert prompt_update_tokens == hf_placeholder_tokens
def test_scoped_request_size_overrides_configured_flat_size():
ctx = build_model_context(
_MODEL_ID,
mm_processor_kwargs={"size": {"longest_edge": 1008}},
limit_mm_per_prompt={"image": 1},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = AutoProcessor.from_pretrained(
_MODEL_ID,
fix_mistral_regex=True,
)
hf_processor_mm_kwargs: dict[str, object] = {
"images_kwargs": {"size": {"longest_edge": 1288}}
}
image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
hf_processor, [image], merged_mm_kwargs
)
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
processor, [image], pixel_values, hf_processor_mm_kwargs
)
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
hf_processor, pixel_values[0]
)
assert expected_from_pixel_values == 1656
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
assert prompt_update_tokens == hf_placeholder_tokens
def test_lightonocr_keeps_vision_config_image_size():
ctx = build_model_context(
_LIGHTON_MODEL_ID,
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
@@ -8,16 +10,238 @@ import pytest
import torch
from PIL import Image
from vllm.config.multimodal import MultiModalConfig
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.multimodal.processing.context import InputProcessingContext
from vllm.platforms import current_platform
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize(
("mm_kwargs", "expected"),
[
(
{"size": {"shortest_edge": 64, "longest_edge": 1024}},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 64,
"max_pixels": 1024,
},
),
(
{"min_pixels": 64, "max_pixels": 1024},
{
"min_pixels": 64,
"max_pixels": 1024,
"size": {"shortest_edge": 64, "longest_edge": 1024},
},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
),
(
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {"max_pixels": 4096},
"videos_kwargs": {"size": {"longest_edge": 8192}},
},
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {
"max_pixels": 4096,
"size": {"longest_edge": 4096},
},
"videos_kwargs": {
"size": {"longest_edge": 8192},
"max_pixels": 8192,
},
},
),
(
{"min_pixels": 0},
{"min_pixels": 0, "size": {"shortest_edge": 0}},
),
(
{"size": None, "min_pixels": 64},
{"size": {"shortest_edge": 64}, "min_pixels": 64},
),
],
)
def test_complete_mm_processor_size_aliases(
mm_kwargs: dict[str, object],
expected: dict[str, object],
) -> None:
original = deepcopy(mm_kwargs)
actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
assert actual == expected
assert mm_kwargs == original
def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
mm_config = MultiModalConfig(
mm_processor_kwargs={
"images_kwargs": {"min_pixels": 64},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
info = Qwen2VLProcessingInfo(ctx)
monkeypatch.setattr(
info,
"get_supported_mm_processor_kwargs",
lambda: {"images_kwargs": {"size", "min_pixels"}},
)
assert info._merge_and_resolve_mm_processor_kwargs(
{"size": {"shortest_edge": 128}}
) == {
"images_kwargs": {
"size": {"shortest_edge": 128},
"min_pixels": 128,
},
}
@pytest.mark.parametrize(
("mm_kwargs", "default_size", "expected"),
[
(
{},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 32, "longest_edge": 1024},
),
(
{"size": {"shortest_edge": 64}},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 1024},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": 128,
"max_pixels": 4096,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 128, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64},
"max_pixels": 4096,
},
None,
{"shortest_edge": 64, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": None,
"max_pixels": 0,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 0},
),
],
)
def test_get_vision_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None,
expected: dict[str, object],
) -> None:
original_mm_kwargs = {
key: dict(value) if isinstance(value, dict) else value
for key, value in mm_kwargs.items()
}
original_default_size = dict(default_size) if default_size is not None else None
actual = Qwen2VLProcessingInfo._get_vision_size(
mm_kwargs,
default_size=default_size,
)
assert actual == expected
assert mm_kwargs == original_mm_kwargs
assert default_size == original_default_size
@pytest.mark.parametrize(
("modality", "scope"),
[
("image", "images_kwargs"),
("video", "videos_kwargs"),
],
)
def test_get_vision_info_reads_merged_modality_scope(
monkeypatch: pytest.MonkeyPatch,
modality: str,
scope: str,
) -> None:
vision_config = SimpleNamespace(
patch_size=14,
spatial_merge_size=2,
temporal_patch_size=2,
)
ctx = SimpleNamespace(
get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
)
info = Qwen2VLProcessingInfo(ctx)
merged = {
"images_kwargs": {"size": {"shortest_edge": 111}},
"videos_kwargs": {"size": {"shortest_edge": 222}},
}
monkeypatch.setattr(
info,
"_merge_and_resolve_mm_processor_kwargs",
lambda _: merged,
)
seen: list[dict[str, object]] = []
def get_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None = None,
) -> dict[str, object]:
seen.append(mm_kwargs)
return dict(default_size or {})
monkeypatch.setattr(info, "_get_vision_size", get_size)
image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
info._get_vision_info(
image_width=28,
image_height=28,
num_frames=2,
do_resize=False,
image_processor=image_processor,
mm_kwargs={},
modality=modality, # type: ignore[arg-type]
)
assert seen == [merged[scope]]
def test_jina_vl_processing_order() -> None:
"""Jina's document-first prompt keeps cached features and hashes aligned."""
ctx = build_model_context(
@@ -1,19 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for Qwen3-VL processor.
Covers the fix for num_frames-based timestamp calculation
(issue vllm-project/vllm#35909).
"""
"""Regression tests for Qwen3-VL multimodal processing."""
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
import torch
from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
from vllm.config import ModelConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.processing.context import BaseProcessingInfo
from ...registry import HF_EXAMPLE_MODELS
from ...utils import build_model_context
@@ -21,6 +23,32 @@ from ...utils import build_model_context
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
assert "size" in native_video_kwargs
assert "min_pixels" not in native_video_kwargs
assert "max_pixels" not in native_video_kwargs
info = Qwen3VLProcessingInfo(SimpleNamespace())
parent_supported = {
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels", "max_pixels"},
"videos_kwargs": set(native_video_kwargs),
}
monkeypatch.setattr(
BaseProcessingInfo,
"get_supported_mm_processor_kwargs",
lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
)
supported = info.supported_mm_processor_kwargs
assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
assert parent_supported["videos_kwargs"] == native_video_kwargs
def _build_video_mm_data(
num_frames: int,
width: int = 128,
@@ -45,6 +73,122 @@ def _build_video_mm_data(
return {"video": [(video, metadata)]}
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_image_size(model_id: str) -> None:
"""Partial image ``size`` must be completed and applied before the HF call.
Keep video compatibility and per-video kwargs in an image-only call as well:
Qwen3 must sanitize the video scope before HF validates it. Regression for
#56363.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = np.zeros((128, 128, 3), dtype=np.uint8)
min_pixels = 256 * 32 * 32
default_size = dict(processor.info.get_image_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [image]})
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"image_grid_thw"
]
stock_grid = process({})
complete_grid = process({"images_kwargs": {"size": complete_size}})
partial_grid = process(
{
"images_kwargs": {
"size": {"shortest_edge": min_pixels},
},
"videos_kwargs": {
"min_pixels": 4096,
"max_pixels": 469_762_048,
"fps": [2.0],
"num_frames": [8],
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_video_size(model_id: str) -> None:
"""Partial scoped video ``size`` must be completed and applied before HF."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
min_pixels = 512 * 32 * 32
default_size = dict(processor.info.get_video_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
complete_grid = process({"videos_kwargs": {"size": complete_size}})
partial_grid = process(
{
"videos_kwargs": {
"size": {"shortest_edge": min_pixels},
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
"""Flat min/max pixel aliases keep their historical Qwen3 video effect."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
pixel_budget = 512 * 32 * 32
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
scoped_grid = process(
{
"videos_kwargs": {
"size": {
"shortest_edge": pixel_budget,
"longest_edge": pixel_budget,
},
},
}
)
flat_grid = process(
{
"min_pixels": pixel_budget,
"max_pixels": pixel_budget,
}
)
assert torch.equal(flat_grid, scoped_grid)
assert not torch.equal(flat_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"num_frames",
@@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"hf_mm_kwargs",
[{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
("hf_mm_kwargs", "scalar_kwargs"),
[
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
],
)
def test_processor_multi_video_list_kwargs(
model_id: str,
hf_mm_kwargs: dict[str, Any],
scalar_kwargs: list[dict[str, Any]],
) -> None:
"""Regression test: a multi-video request with list-valued per-video
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
crash.
"""List-valued video kwargs must be scalarized per item without mutation.
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
video without slicing, so ``_get_video_second_idx`` received the list
where a scalar was expected and raised ``TypeError``.
Each video in the multi-video request must match processing the same input
independently with its corresponding scalar ``fps``/``num_frames`` value.
"""
ctx = build_model_context(
model_id,
@@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs(
)
mm_data = {
"video": [
_build_video_mm_data(num_frames=16)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
]
}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
single_mm_data = _build_video_mm_data(num_frames=32)
single = processor(
single_prompt,
mm_items=processor.info.parse_mm_data(single_mm_data),
hf_processor_mm_kwargs=hf_kwargs,
)
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
assert not torch.equal(expected_grids[0], expected_grids[1])
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
assert hf_mm_kwargs == hf_mm_kwargs_before
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 2, (
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
@@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
capped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
mm_processor_kwargs={
"videos_kwargs": {
"size": size,
"cap_pixels_per_frame": True,
},
},
limit_mm_per_prompt={"image": 0, "video": 1},
)
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
@@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
uncapped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"size": size},
mm_processor_kwargs={"videos_kwargs": {"size": size}},
limit_mm_per_prompt={"image": 0, "video": 1},
)
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)