[Bugfix][Multimodal] Honor modality-scoped mm_processor_kwargs (#53808)
Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com>
This commit is contained in:
@@ -142,6 +142,50 @@ def test_processor_multi_video(
|
||||
)
|
||||
|
||||
|
||||
# Qwen3-VL / Qwen3.8 "Long Video Understanding" pixel budget from the
|
||||
# model card. Used to check --mm-processor-kwargs scoping (#52834).
|
||||
_LONG_VIDEO_SIZE = {"longest_edge": 469762048, "shortest_edge": 4096}
|
||||
|
||||
|
||||
def _probe_mm_token_budgets(
|
||||
model_id: str, mm_processor_kwargs: dict[str, Any] | None
|
||||
) -> tuple[int, int]:
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
)
|
||||
info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info
|
||||
video = info.get_max_video_tokens(
|
||||
seq_len=500000, mm_counts={"video": 1, "image": 1}
|
||||
)
|
||||
return video, info.get_max_image_tokens()
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
|
||||
model_id: str,
|
||||
) -> None:
|
||||
"""``videos_kwargs.size`` must raise only the video token budget.
|
||||
|
||||
A flat ``size`` override still applies to both modalities (the previous
|
||||
shared-namespace behavior). Regression for #52834.
|
||||
"""
|
||||
stock_video, stock_image = _probe_mm_token_budgets(model_id, None)
|
||||
scoped_video, scoped_image = _probe_mm_token_budgets(
|
||||
model_id, {"videos_kwargs": {"size": _LONG_VIDEO_SIZE}}
|
||||
)
|
||||
flat_video, flat_image = _probe_mm_token_budgets(
|
||||
model_id, {"size": _LONG_VIDEO_SIZE}
|
||||
)
|
||||
|
||||
assert scoped_video == flat_video
|
||||
assert scoped_video > stock_video
|
||||
assert scoped_image == stock_image
|
||||
assert flat_image > stock_image
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"hf_mm_kwargs",
|
||||
|
||||
@@ -10,7 +10,10 @@ import pytest
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.processing.context import InputProcessingContext
|
||||
from vllm.multimodal.processing.context import (
|
||||
InputProcessingContext,
|
||||
overlay_modality_mm_kwargs,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
PlaceholderFeaturesInfo,
|
||||
@@ -1205,3 +1208,57 @@ def test_apply_prompt_updates_falls_back_with_index_targets():
|
||||
|
||||
assert new_token_ids == [200, 201, ord("d"), 9]
|
||||
assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image():
|
||||
"""HF-style videos_kwargs must overlay only when modality is video."""
|
||||
video_size = {"longest_edge": 469762048, "shortest_edge": 4096}
|
||||
kwargs = {"videos_kwargs": {"size": video_size}}
|
||||
|
||||
assert overlay_modality_mm_kwargs(kwargs, None) == kwargs
|
||||
assert "size" not in overlay_modality_mm_kwargs(kwargs, "image")
|
||||
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_overlay_modality_mm_kwargs_flat_size_stays_shared():
|
||||
"""A flat size override keeps the current shared-namespace behavior."""
|
||||
size = {"longest_edge": 469762048, "shortest_edge": 4096}
|
||||
kwargs = {"size": size}
|
||||
|
||||
for modality in (None, "image", "video"):
|
||||
assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality():
|
||||
"""A nested videos_kwargs size wins over a flat size for video reads."""
|
||||
kwargs = {
|
||||
"size": {"longest_edge": 1},
|
||||
"videos_kwargs": {"size": {"longest_edge": 2}},
|
||||
"images_kwargs": {"size": {"longest_edge": 3}},
|
||||
}
|
||||
|
||||
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2}
|
||||
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3}
|
||||
assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1}
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value():
|
||||
kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}}
|
||||
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1}
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
|
||||
"""Configured videos_kwargs overlay only for video reads after merge."""
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
|
||||
size = {"longest_edge": 469762048, "shortest_edge": 4096}
|
||||
mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}})
|
||||
merged = mm_config.merge_mm_processor_kwargs({})
|
||||
assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
|
||||
assert "size" not in overlay_modality_mm_kwargs(merged, "image")
|
||||
assert "size" not in overlay_modality_mm_kwargs(merged, None)
|
||||
|
||||
@@ -881,6 +881,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
do_resize: bool = True,
|
||||
image_processor: Qwen2VLImageProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str | None = None,
|
||||
) -> tuple[ImageSize, int]:
|
||||
hf_config = self.get_hf_config()
|
||||
vision_config = hf_config.vision_config
|
||||
@@ -888,7 +889,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
temporal_patch_size = vision_config.temporal_patch_size
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
|
||||
size = image_processor.size
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
@@ -936,6 +937,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
num_frames=1,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs=mm_kwargs,
|
||||
modality="image",
|
||||
)
|
||||
return num_image_tokens
|
||||
|
||||
@@ -954,6 +956,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
num_frames=num_frames,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs=mm_kwargs,
|
||||
modality="video",
|
||||
)
|
||||
return num_video_tokens
|
||||
|
||||
@@ -981,7 +984,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
if max_pixels is None:
|
||||
image_processor = self.get_image_processor()
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image")
|
||||
size = image_processor.size
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
|
||||
@@ -941,6 +941,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
do_resize: bool = True,
|
||||
image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str | None = None,
|
||||
) -> tuple[ImageSize, int]:
|
||||
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
|
||||
|
||||
@@ -950,7 +951,9 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
merge_size = vision_config.spatial_merge_size
|
||||
temporal_patch_size = vision_config.temporal_patch_size
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
if modality is None:
|
||||
modality = "video" if is_video else "image"
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
|
||||
size = image_processor.size
|
||||
if override_size := mm_kwargs.get("size"):
|
||||
size = size | override_size
|
||||
@@ -1014,7 +1017,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
) -> int:
|
||||
video_processor = self.get_video_processor()
|
||||
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
|
||||
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video")
|
||||
video_size = mm_kwargs.get("size", video_processor.size)
|
||||
temporal_patch_size = mm_kwargs.get(
|
||||
"temporal_patch_size", video_processor.temporal_patch_size
|
||||
@@ -1150,7 +1153,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
|
||||
|
||||
video_processor = self.info.get_video_processor()
|
||||
|
||||
mm_kwargs = self.info.ctx.get_merged_mm_kwargs({})
|
||||
mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video")
|
||||
video_size = mm_kwargs.get("size", video_processor.size)
|
||||
temporal_patch_size = mm_kwargs.get(
|
||||
"temporal_patch_size", video_processor.temporal_patch_size
|
||||
@@ -1365,7 +1368,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# NOTE: a copy of is created to update do_sample_frames,
|
||||
# otherwise mm_hash for the object will be incorrect.
|
||||
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(
|
||||
hf_processor_mm_kwargs, modality="video"
|
||||
)
|
||||
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||
video_size = dict(self.info.get_video_processor().size)
|
||||
size_override = merged.get("size")
|
||||
|
||||
@@ -42,6 +42,46 @@ else:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# HuggingFace processors accept nested ``images_kwargs`` / ``videos_kwargs`` /
|
||||
# ``audio_kwargs`` in addition to a shared flat namespace. vLLM's token-budget
|
||||
# and dummy-input code only reads the flat keys, so scoped overrides have to
|
||||
# be overlaid per modality.
|
||||
_HF_MODALITY_PROCESSOR_KWARGS = {
|
||||
"image": "images_kwargs",
|
||||
"video": "videos_kwargs",
|
||||
"audio": "audio_kwargs",
|
||||
}
|
||||
|
||||
|
||||
def overlay_modality_mm_kwargs(
|
||||
kwargs: Mapping[str, object],
|
||||
modality: str | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Overlay HF-style nested processor kwargs onto the flat namespace.
|
||||
|
||||
Overlay only the requested modality so a video ``size`` bump does not leak
|
||||
into the image budget. Flat keys keep their current shared-namespace
|
||||
behavior when no scoped dict is present.
|
||||
|
||||
Args:
|
||||
kwargs: Merged multi-modal processor kwargs.
|
||||
modality: Target modality (``image``, ``video``, or ``audio``).
|
||||
When ``None``, ``kwargs`` is copied without overlay.
|
||||
|
||||
Returns:
|
||||
A new dict with the modality-scoped keys overlaid when present.
|
||||
"""
|
||||
merged = dict(kwargs)
|
||||
if modality is None:
|
||||
return merged
|
||||
scoped_key = _HF_MODALITY_PROCESSOR_KWARGS.get(modality)
|
||||
if scoped_key is None:
|
||||
return merged
|
||||
scoped = merged.get(scoped_key)
|
||||
if not isinstance(scoped, Mapping):
|
||||
return merged
|
||||
return merged | dict(scoped)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TimingContext:
|
||||
@@ -253,9 +293,23 @@ class InputProcessingContext:
|
||||
|
||||
return json_map_leaves(_postprocess_one, output)
|
||||
|
||||
def get_merged_mm_kwargs(self, kwargs: Mapping[str, object]):
|
||||
def get_merged_mm_kwargs(
|
||||
self,
|
||||
kwargs: Mapping[str, object],
|
||||
*,
|
||||
modality: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Merge configured and request ``mm_processor_kwargs``.
|
||||
|
||||
When ``modality`` is set, HF-style nested
|
||||
``images_kwargs`` / ``videos_kwargs`` / ``audio_kwargs`` are overlaid
|
||||
onto the flat namespace for vLLM-side reads (token budgets, dummy
|
||||
inputs). Processor construction and HF ``__call__`` should omit
|
||||
``modality`` so the nested dicts still reach the HF processor.
|
||||
"""
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
return mm_config.merge_mm_processor_kwargs(kwargs)
|
||||
merged = mm_config.merge_mm_processor_kwargs(kwargs)
|
||||
return overlay_modality_mm_kwargs(merged, modality)
|
||||
|
||||
def call_hf_processor(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user