[Bugfix][Multimodal] Honor modality-scoped mm_processor_kwargs (#53808)

Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com>
This commit is contained in:
Prudhvi Vuda
2026-08-31 06:37:12 +00:00
committed by GitHub
parent 4ae172231c
commit 7292ee2791
5 changed files with 172 additions and 9 deletions
@@ -142,6 +142,50 @@ def test_processor_multi_video(
)
# Qwen3-VL / Qwen3.8 "Long Video Understanding" pixel budget from the
# model card. Used to check --mm-processor-kwargs scoping (#52834).
_LONG_VIDEO_SIZE = {"longest_edge": 469762048, "shortest_edge": 4096}
def _probe_mm_token_budgets(
model_id: str, mm_processor_kwargs: dict[str, Any] | None
) -> tuple[int, int]:
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
)
info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info
video = info.get_max_video_tokens(
seq_len=500000, mm_counts={"video": 1, "image": 1}
)
return video, info.get_max_image_tokens()
@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
model_id: str,
) -> None:
"""``videos_kwargs.size`` must raise only the video token budget.
A flat ``size`` override still applies to both modalities (the previous
shared-namespace behavior). Regression for #52834.
"""
stock_video, stock_image = _probe_mm_token_budgets(model_id, None)
scoped_video, scoped_image = _probe_mm_token_budgets(
model_id, {"videos_kwargs": {"size": _LONG_VIDEO_SIZE}}
)
flat_video, flat_image = _probe_mm_token_budgets(
model_id, {"size": _LONG_VIDEO_SIZE}
)
assert scoped_video == flat_video
assert scoped_video > stock_video
assert scoped_image == stock_image
assert flat_image > stock_image
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"hf_mm_kwargs",
+58 -1
View File
@@ -10,7 +10,10 @@ import pytest
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.processing.context import InputProcessingContext
from vllm.multimodal.processing.context import (
InputProcessingContext,
overlay_modality_mm_kwargs,
)
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
PlaceholderFeaturesInfo,
@@ -1205,3 +1208,57 @@ def test_apply_prompt_updates_falls_back_with_index_targets():
assert new_token_ids == [200, 201, ord("d"), 9]
assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image():
"""HF-style videos_kwargs must overlay only when modality is video."""
video_size = {"longest_edge": 469762048, "shortest_edge": 4096}
kwargs = {"videos_kwargs": {"size": video_size}}
assert overlay_modality_mm_kwargs(kwargs, None) == kwargs
assert "size" not in overlay_modality_mm_kwargs(kwargs, "image")
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_flat_size_stays_shared():
"""A flat size override keeps the current shared-namespace behavior."""
size = {"longest_edge": 469762048, "shortest_edge": 4096}
kwargs = {"size": size}
for modality in (None, "image", "video"):
assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality():
"""A nested videos_kwargs size wins over a flat size for video reads."""
kwargs = {
"size": {"longest_edge": 1},
"videos_kwargs": {"size": {"longest_edge": 2}},
"images_kwargs": {"size": {"longest_edge": 3}},
}
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2}
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3}
assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1}
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value():
kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}}
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1}
@pytest.mark.skip_global_cleanup
def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
"""Configured videos_kwargs overlay only for video reads after merge."""
from vllm.config.multimodal import MultiModalConfig
size = {"longest_edge": 469762048, "shortest_edge": 4096}
mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}})
merged = mm_config.merge_mm_processor_kwargs({})
assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
assert "size" not in overlay_modality_mm_kwargs(merged, "image")
assert "size" not in overlay_modality_mm_kwargs(merged, None)