diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index d17e19bd65..c58904a9e2 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -142,6 +142,50 @@ def test_processor_multi_video( ) +# Qwen3-VL / Qwen3.8 "Long Video Understanding" pixel budget from the +# model card. Used to check --mm-processor-kwargs scoping (#52834). +_LONG_VIDEO_SIZE = {"longest_edge": 469762048, "shortest_edge": 4096} + + +def _probe_mm_token_budgets( + model_id: str, mm_processor_kwargs: dict[str, Any] | None +) -> tuple[int, int]: + ctx = build_model_context( + model_id, + mm_processor_kwargs=mm_processor_kwargs, + limit_mm_per_prompt={"image": 1, "video": 1}, + ) + info = MULTIMODAL_REGISTRY.create_processor(ctx.model_config).info + video = info.get_max_video_tokens( + seq_len=500000, mm_counts={"video": 1, "image": 1} + ) + return video, info.get_max_image_tokens() + + +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize("model_id", [MODEL_ID]) +def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget( + model_id: str, +) -> None: + """``videos_kwargs.size`` must raise only the video token budget. + + A flat ``size`` override still applies to both modalities (the previous + shared-namespace behavior). Regression for #52834. + """ + stock_video, stock_image = _probe_mm_token_budgets(model_id, None) + scoped_video, scoped_image = _probe_mm_token_budgets( + model_id, {"videos_kwargs": {"size": _LONG_VIDEO_SIZE}} + ) + flat_video, flat_image = _probe_mm_token_budgets( + model_id, {"size": _LONG_VIDEO_SIZE} + ) + + assert scoped_video == flat_video + assert scoped_video > stock_video + assert scoped_image == stock_image + assert flat_image > stock_image + + @pytest.mark.parametrize("model_id", [MODEL_ID]) @pytest.mark.parametrize( "hf_mm_kwargs", diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index a3f4cf7e47..9d629fbf96 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -10,7 +10,10 @@ import pytest from vllm.config import ModelConfig from vllm.exceptions import VLLMValidationError from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.processing.context import InputProcessingContext +from vllm.multimodal.processing.context import ( + InputProcessingContext, + overlay_modality_mm_kwargs, +) from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, PlaceholderFeaturesInfo, @@ -1205,3 +1208,57 @@ def test_apply_prompt_updates_falls_back_with_index_targets(): assert new_token_ids == [200, 201, ord("d"), 9] assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]] + + +@pytest.mark.skip_global_cleanup +def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image(): + """HF-style videos_kwargs must overlay only when modality is video.""" + video_size = {"longest_edge": 469762048, "shortest_edge": 4096} + kwargs = {"videos_kwargs": {"size": video_size}} + + assert overlay_modality_mm_kwargs(kwargs, None) == kwargs + assert "size" not in overlay_modality_mm_kwargs(kwargs, "image") + assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size + + +@pytest.mark.skip_global_cleanup +def test_overlay_modality_mm_kwargs_flat_size_stays_shared(): + """A flat size override keeps the current shared-namespace behavior.""" + size = {"longest_edge": 469762048, "shortest_edge": 4096} + kwargs = {"size": size} + + for modality in (None, "image", "video"): + assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size + + +@pytest.mark.skip_global_cleanup +def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality(): + """A nested videos_kwargs size wins over a flat size for video reads.""" + kwargs = { + "size": {"longest_edge": 1}, + "videos_kwargs": {"size": {"longest_edge": 2}}, + "images_kwargs": {"size": {"longest_edge": 3}}, + } + + assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2} + assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3} + assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1} + + +@pytest.mark.skip_global_cleanup +def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value(): + kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}} + assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1} + + +@pytest.mark.skip_global_cleanup +def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping(): + """Configured videos_kwargs overlay only for video reads after merge.""" + from vllm.config.multimodal import MultiModalConfig + + size = {"longest_edge": 469762048, "shortest_edge": 4096} + mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}}) + merged = mm_config.merge_mm_processor_kwargs({}) + assert overlay_modality_mm_kwargs(merged, "video")["size"] == size + assert "size" not in overlay_modality_mm_kwargs(merged, "image") + assert "size" not in overlay_modality_mm_kwargs(merged, None) diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index e984e94a64..de75a59fee 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -881,6 +881,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): do_resize: bool = True, image_processor: Qwen2VLImageProcessor, mm_kwargs: Mapping[str, object], + modality: str | None = None, ) -> tuple[ImageSize, int]: hf_config = self.get_hf_config() vision_config = hf_config.vision_config @@ -888,7 +889,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): merge_size = vision_config.spatial_merge_size temporal_patch_size = vision_config.temporal_patch_size - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) + mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality) size = image_processor.size if override_size := mm_kwargs.get("size"): size = size | override_size @@ -936,6 +937,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): num_frames=1, image_processor=image_processor, mm_kwargs=mm_kwargs, + modality="image", ) return num_image_tokens @@ -954,6 +956,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): num_frames=num_frames, image_processor=image_processor, mm_kwargs=mm_kwargs, + modality="video", ) return num_video_tokens @@ -981,7 +984,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): if max_pixels is None: image_processor = self.get_image_processor() - mm_kwargs = self.ctx.get_merged_mm_kwargs({}) + mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image") size = image_processor.size if override_size := mm_kwargs.get("size"): size = size | override_size diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 3d2525035c..dd08f1cc04 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -941,6 +941,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): do_resize: bool = True, image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor, mm_kwargs: Mapping[str, object], + modality: str | None = None, ) -> tuple[ImageSize, int]: is_video = isinstance(image_processor, Qwen3VLVideoProcessor) @@ -950,7 +951,9 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): merge_size = vision_config.spatial_merge_size temporal_patch_size = vision_config.temporal_patch_size - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs) + if modality is None: + modality = "video" if is_video else "image" + mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality) size = image_processor.size if override_size := mm_kwargs.get("size"): size = size | override_size @@ -1014,7 +1017,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): ) -> int: video_processor = self.get_video_processor() - mm_kwargs = self.ctx.get_merged_mm_kwargs({}) + mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video") video_size = mm_kwargs.get("size", video_processor.size) temporal_patch_size = mm_kwargs.get( "temporal_patch_size", video_processor.temporal_patch_size @@ -1150,7 +1153,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): video_processor = self.info.get_video_processor() - mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}) + mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video") video_size = mm_kwargs.get("size", video_processor.size) temporal_patch_size = mm_kwargs.get( "temporal_patch_size", video_processor.temporal_patch_size @@ -1365,7 +1368,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # NOTE: a copy of is created to update do_sample_frames, # otherwise mm_hash for the object will be incorrect. video_mm_kwargs = dict(**hf_processor_mm_kwargs) - merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) + merged = self.info.ctx.get_merged_mm_kwargs( + hf_processor_mm_kwargs, modality="video" + ) if merged.keys() & {"size", "min_pixels", "max_pixels"}: video_size = dict(self.info.get_video_processor().size) size_override = merged.get("size") diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py index f13ca6ab88..07149dbc4a 100644 --- a/vllm/multimodal/processing/context.py +++ b/vllm/multimodal/processing/context.py @@ -42,6 +42,46 @@ else: logger = init_logger(__name__) +# HuggingFace processors accept nested ``images_kwargs`` / ``videos_kwargs`` / +# ``audio_kwargs`` in addition to a shared flat namespace. vLLM's token-budget +# and dummy-input code only reads the flat keys, so scoped overrides have to +# be overlaid per modality. +_HF_MODALITY_PROCESSOR_KWARGS = { + "image": "images_kwargs", + "video": "videos_kwargs", + "audio": "audio_kwargs", +} + + +def overlay_modality_mm_kwargs( + kwargs: Mapping[str, object], + modality: str | None, +) -> dict[str, Any]: + """Overlay HF-style nested processor kwargs onto the flat namespace. + + Overlay only the requested modality so a video ``size`` bump does not leak + into the image budget. Flat keys keep their current shared-namespace + behavior when no scoped dict is present. + + Args: + kwargs: Merged multi-modal processor kwargs. + modality: Target modality (``image``, ``video``, or ``audio``). + When ``None``, ``kwargs`` is copied without overlay. + + Returns: + A new dict with the modality-scoped keys overlaid when present. + """ + merged = dict(kwargs) + if modality is None: + return merged + scoped_key = _HF_MODALITY_PROCESSOR_KWARGS.get(modality) + if scoped_key is None: + return merged + scoped = merged.get(scoped_key) + if not isinstance(scoped, Mapping): + return merged + return merged | dict(scoped) + @dataclass class TimingContext: @@ -253,9 +293,23 @@ class InputProcessingContext: return json_map_leaves(_postprocess_one, output) - def get_merged_mm_kwargs(self, kwargs: Mapping[str, object]): + def get_merged_mm_kwargs( + self, + kwargs: Mapping[str, object], + *, + modality: str | None = None, + ) -> dict[str, Any]: + """Merge configured and request ``mm_processor_kwargs``. + + When ``modality`` is set, HF-style nested + ``images_kwargs`` / ``videos_kwargs`` / ``audio_kwargs`` are overlaid + onto the flat namespace for vLLM-side reads (token budgets, dummy + inputs). Processor construction and HF ``__call__`` should omit + ``modality`` so the nested dicts still reach the HF processor. + """ mm_config = self.model_config.get_multimodal_config() - return mm_config.merge_mm_processor_kwargs(kwargs) + merged = mm_config.merge_mm_processor_kwargs(kwargs) + return overlay_modality_mm_kwargs(merged, modality) def call_hf_processor( self,