diff --git a/tests/config/test_multimodal_config.py b/tests/config/test_multimodal_config.py index b36f5776da..bd9343bd15 100644 --- a/tests/config/test_multimodal_config.py +++ b/tests/config/test_multimodal_config.py @@ -188,6 +188,250 @@ def test_mm_processor_device_type_normalizes(device: object, expected: str | Non assert config.get_mm_processor_device_type() == expected +def test_merge_mm_processor_kwargs_scoped_precedence(): + configured = { + "size": { + "shortest_edge": 64, + "longest_edge": 512, + }, + "do_resize": True, + "padding": False, + "images_kwargs": { + "size": {"longest_edge": 1024}, + "do_resize": False, + }, + "videos_kwargs": { + "size": {"longest_edge": 2048}, + }, + } + inference = { + "size": {"shortest_edge": 128}, + "do_resize": True, + "images_kwargs": { + "size": {"shortest_edge": 256}, + }, + } + + config = MultiModalConfig( + mm_processor_kwargs=configured, + mm_device_do_normalize=False, + ) + + assert config.merge_mm_processor_kwargs(inference) == { + "size": { + "shortest_edge": 128, + "longest_edge": 512, + }, + "do_resize": True, + "padding": False, + "images_kwargs": { + "size": { + "shortest_edge": 256, + "longest_edge": 1024, + }, + "do_resize": True, + }, + "videos_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 2048, + }, + }, + } + + +def test_merge_mm_processor_kwargs_mapping_vs_replacement(): + config = MultiModalConfig( + mm_processor_kwargs={ + "size": {"shortest_edge": 64}, + "images_kwargs": {"size": {"longest_edge": 1024}}, + }, + mm_device_do_normalize=False, + ) + + assert config.merge_mm_processor_kwargs({"size": 7}) == { + "size": 7, + "images_kwargs": {"size": 7}, + } + assert config.merge_mm_processor_kwargs({"size": {}}) == { + "size": {"shortest_edge": 64}, + "images_kwargs": { + "size": { + "shortest_edge": 64, + "longest_edge": 1024, + }, + }, + } + + +_EMPTY_IMAGE_SCOPE_KWARGS: tuple[dict[str, object], ...] = ( + {}, + {"images_kwargs": None}, + {"images_kwargs": {}}, +) + + +@pytest.mark.parametrize("configured", _EMPTY_IMAGE_SCOPE_KWARGS) +@pytest.mark.parametrize("inference", _EMPTY_IMAGE_SCOPE_KWARGS) +def test_merge_mm_processor_kwargs_empty_scopes_are_absent( + configured: dict[str, object], + inference: dict[str, object], +): + config = MultiModalConfig( + mm_processor_kwargs=configured, + mm_device_do_normalize=False, + ) + + assert config.merge_mm_processor_kwargs(inference) == {} + + +@pytest.mark.parametrize("empty_inference", _EMPTY_IMAGE_SCOPE_KWARGS) +def test_merge_mm_processor_kwargs_empty_inference_scope_preserves_config_merge( + empty_inference: dict[str, object], +): + config = MultiModalConfig( + mm_processor_kwargs={ + "images_kwargs": { + "size": { + "shortest_edge": 64, + "longest_edge": 512, + }, + }, + }, + mm_device_do_normalize=False, + ) + inference = { + "size": {"shortest_edge": 128}, + **empty_inference, + } + + assert config.merge_mm_processor_kwargs(inference) == { + "size": {"shortest_edge": 128}, + "images_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 512, + }, + }, + } + + +@pytest.mark.parametrize("empty_config", _EMPTY_IMAGE_SCOPE_KWARGS) +def test_merge_mm_processor_kwargs_empty_config_scope_allows_inference_merge( + empty_config: dict[str, object], +): + config = MultiModalConfig( + mm_processor_kwargs={ + "size": {"longest_edge": 512}, + **empty_config, + }, + mm_device_do_normalize=False, + ) + + assert config.merge_mm_processor_kwargs( + { + "size": {"shortest_edge": 128}, + "images_kwargs": {"size": {"longest_edge": 1024}}, + } + ) == { + "size": { + "shortest_edge": 128, + "longest_edge": 512, + }, + "images_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 1024, + }, + }, + } + + +@pytest.mark.parametrize("non_mapping_scope", [False, 0, "", [], 123]) +def test_merge_mm_processor_kwargs_falsey_non_mapping_scopes_are_not_empty( + non_mapping_scope: object, +): + config = MultiModalConfig( + mm_processor_kwargs={"images_kwargs": non_mapping_scope}, + mm_device_do_normalize=False, + ) + + for empty_inference in _EMPTY_IMAGE_SCOPE_KWARGS: + assert config.merge_mm_processor_kwargs(empty_inference) == { + "images_kwargs": non_mapping_scope + } + + +@pytest.mark.parametrize( + ("configured_scope", "inference_scope", "expected_scope"), + [ + (123, {"size": {"shortest_edge": 128}}, {"size": {"shortest_edge": 128}}), + ({"size": {"shortest_edge": 64}}, 123, 123), + (123, 456, 456), + ], +) +def test_merge_mm_processor_kwargs_non_mapping_scope_precedence( + configured_scope: object, + inference_scope: object, + expected_scope: object, +): + config = MultiModalConfig( + mm_processor_kwargs={"images_kwargs": configured_scope}, + mm_device_do_normalize=False, + ) + + merged = config.merge_mm_processor_kwargs({"images_kwargs": inference_scope}) + + assert merged["images_kwargs"] == expected_scope + + +def test_merge_mm_processor_kwargs_none_leaf_is_explicit(): + config = MultiModalConfig(mm_device_do_normalize=False) + + assert config.merge_mm_processor_kwargs({"images_kwargs": {"do_resize": None}}) == { + "images_kwargs": {"do_resize": None} + } + + +def test_merge_mm_processor_kwargs_normalize_and_ownership(): + configured: dict[str, object] = { + "images_kwargs": { + "size": {"shortest_edge": 64}, + }, + } + inference: dict[str, object] = { + "do_normalize": True, + "do_rescale": True, + "images_kwargs": { + "size": {"longest_edge": 1024}, + }, + } + configured_before = copy.deepcopy(configured) + inference_before = copy.deepcopy(inference) + + config = MultiModalConfig( + mm_processor_kwargs=configured, + mm_device_do_normalize=True, + ) + merged = config.merge_mm_processor_kwargs(inference) + + assert merged["do_normalize"] is False + assert merged["do_rescale"] is False + assert configured == configured_before + assert inference == inference_before + assert config.mm_processor_kwargs == configured_before + + images_kwargs = merged["images_kwargs"] + assert isinstance(images_kwargs, dict) + size = images_kwargs["size"] + assert isinstance(size, dict) + size["shortest_edge"] = 999 + + assert configured == configured_before + assert inference == inference_before + assert config.mm_processor_kwargs == configured_before + + def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None: ec_config = ( None diff --git a/tests/models/multimodal/processing/test_gemma4.py b/tests/models/multimodal/processing/test_gemma4.py index 6430f40d8b..47dd0e55b6 100644 --- a/tests/models/multimodal/processing/test_gemma4.py +++ b/tests/models/multimodal/processing/test_gemma4.py @@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str): @pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID]) -@pytest.mark.parametrize("kwargs_on_init", [False, True]) @pytest.mark.parametrize( "image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}] ) @pytest.mark.parametrize("video_uuid", [None, "same-video"]) def test_video_cache_is_independent_of_image_kwargs( model_id: str, - kwargs_on_init: bool, image_kwargs: dict[str, object], video_uuid: str | None, ): - """Image overrides must not change video frames or depend on cache warmth.""" - kwargs = {"images_kwargs": image_kwargs} + """Request image overrides must not change video frames or make results + depend on cache warmth. + """ + request_kwargs = {"images_kwargs": image_kwargs} ctx = build_model_context( model_id, limit_mm_per_prompt={"image": 1, "video": 1}, @@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs( cache=cache, ) - baseline = process({}, cache=None) - if kwargs_on_init: - ctx = build_model_context( - model_id, - mm_processor_kwargs=kwargs, - limit_mm_per_prompt={"image": 1, "video": 1}, - mm_processor_cache_gb=1, - ) - cache = MultiModalProcessorOnlyCache(ctx.model_config) - processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) - request_kwargs = {} if kwargs_on_init else kwargs - process(request_kwargs, cache=cache) + # Populate the video cache without image overrides, then change only + # images_kwargs on the next request. Since images_kwargs are excluded from + # the video hash, and must not affect the processed video, the cached result + # must match a fresh recomputation. + baseline = process({}, cache=cache) cached = process(request_kwargs, cache=cache) cache.clear_cache() fresh = process(request_kwargs, cache=cache) diff --git a/tests/models/multimodal/processing/test_mistral3.py b/tests/models/multimodal/processing/test_mistral3.py index 4d900e627f..3445dd86fe 100644 --- a/tests/models/multimodal/processing/test_mistral3.py +++ b/tests/models/multimodal/processing/test_mistral3.py @@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image( [ ({}, (448, 448), 256), ({"size": {"longest_edge": 1008}}, (1540, 1540), 1296), + ( + {"images_kwargs": {"size": {"longest_edge": 1008}}}, + (1540, 1540), + 1296, + ), ({"size": {"longest_edge": 1288}}, (1536, 1187), 1656), ({"size": {"longest_edge": 1008}}, (29, 29), 4), ({"size": {"longest_edge": 1000}}, (1540, 1700), 1188), @@ -148,6 +153,40 @@ def test_processor_size_override( assert prompt_update_tokens == hf_placeholder_tokens +def test_scoped_request_size_overrides_configured_flat_size(): + ctx = build_model_context( + _MODEL_ID, + mm_processor_kwargs={"size": {"longest_edge": 1008}}, + limit_mm_per_prompt={"image": 1}, + model_config_kwargs=_MODEL_CONFIG_KWARGS, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + hf_processor = AutoProcessor.from_pretrained( + _MODEL_ID, + fix_mistral_regex=True, + ) + + hf_processor_mm_kwargs: dict[str, object] = { + "images_kwargs": {"size": {"longest_edge": 1288}} + } + image = Image.new("RGB", (1536, 1187), color=(127, 127, 127)) + + merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs) + pixel_values, hf_placeholder_tokens = _process_images_with_hf( + hf_processor, [image], merged_mm_kwargs + ) + prompt_update_tokens = _placeholder_tokens_from_prompt_updates( + processor, [image], pixel_values, hf_processor_mm_kwargs + ) + + expected_from_pixel_values = _expected_placeholder_tokens_per_image( + hf_processor, pixel_values[0] + ) + assert expected_from_pixel_values == 1656 + assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656 + assert prompt_update_tokens == hf_placeholder_tokens + + def test_lightonocr_keeps_vision_config_image_size(): ctx = build_model_context( _LIGHTON_MODEL_ID, diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index 5ea538555e..b957ebf2c8 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from copy import deepcopy +from types import SimpleNamespace from typing import Any import numpy as np @@ -8,16 +10,238 @@ import pytest import torch from PIL import Image +from vllm.config.multimodal import MultiModalConfig from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm +from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.cache import MultiModalProcessorOnlyCache from vllm.multimodal.inputs import batched_tensors_equal +from vllm.multimodal.processing.context import InputProcessingContext from vllm.platforms import current_platform from ....conftest import ImageTestAssets from ...utils import build_model_context +@pytest.mark.parametrize( + ("mm_kwargs", "expected"), + [ + ( + {"size": {"shortest_edge": 64, "longest_edge": 1024}}, + { + "size": {"shortest_edge": 64, "longest_edge": 1024}, + "min_pixels": 64, + "max_pixels": 1024, + }, + ), + ( + {"min_pixels": 64, "max_pixels": 1024}, + { + "min_pixels": 64, + "max_pixels": 1024, + "size": {"shortest_edge": 64, "longest_edge": 1024}, + }, + ), + ( + { + "size": {"shortest_edge": 64, "longest_edge": 1024}, + "min_pixels": 128, + "max_pixels": 2048, + }, + { + "size": {"shortest_edge": 64, "longest_edge": 1024}, + "min_pixels": 128, + "max_pixels": 2048, + }, + ), + ( + { + "min_pixels": None, + "size": {"shortest_edge": 0}, + "images_kwargs": {"max_pixels": 4096}, + "videos_kwargs": {"size": {"longest_edge": 8192}}, + }, + { + "min_pixels": None, + "size": {"shortest_edge": 0}, + "images_kwargs": { + "max_pixels": 4096, + "size": {"longest_edge": 4096}, + }, + "videos_kwargs": { + "size": {"longest_edge": 8192}, + "max_pixels": 8192, + }, + }, + ), + ( + {"min_pixels": 0}, + {"min_pixels": 0, "size": {"shortest_edge": 0}}, + ), + ( + {"size": None, "min_pixels": 64}, + {"size": {"shortest_edge": 64}, "min_pixels": 64}, + ), + ], +) +def test_complete_mm_processor_size_aliases( + mm_kwargs: dict[str, object], + expected: dict[str, object], +) -> None: + original = deepcopy(mm_kwargs) + + actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs) + + assert actual == expected + assert mm_kwargs == original + + +def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + mm_config = MultiModalConfig( + mm_processor_kwargs={ + "images_kwargs": {"min_pixels": 64}, + }, + mm_device_do_normalize=False, + ) + model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config) + ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type] + info = Qwen2VLProcessingInfo(ctx) + monkeypatch.setattr( + info, + "get_supported_mm_processor_kwargs", + lambda: {"images_kwargs": {"size", "min_pixels"}}, + ) + + assert info._merge_and_resolve_mm_processor_kwargs( + {"size": {"shortest_edge": 128}} + ) == { + "images_kwargs": { + "size": {"shortest_edge": 128}, + "min_pixels": 128, + }, + } + + +@pytest.mark.parametrize( + ("mm_kwargs", "default_size", "expected"), + [ + ( + {}, + {"shortest_edge": 32, "longest_edge": 1024}, + {"shortest_edge": 32, "longest_edge": 1024}, + ), + ( + {"size": {"shortest_edge": 64}}, + {"shortest_edge": 32, "longest_edge": 1024}, + {"shortest_edge": 64, "longest_edge": 1024}, + ), + ( + { + "size": {"shortest_edge": 64, "longest_edge": 2048}, + "min_pixels": 128, + "max_pixels": 4096, + }, + {"shortest_edge": 32, "longest_edge": 1024}, + {"shortest_edge": 128, "longest_edge": 4096}, + ), + ( + { + "size": {"shortest_edge": 64}, + "max_pixels": 4096, + }, + None, + {"shortest_edge": 64, "longest_edge": 4096}, + ), + ( + { + "size": {"shortest_edge": 64, "longest_edge": 2048}, + "min_pixels": None, + "max_pixels": 0, + }, + {"shortest_edge": 32, "longest_edge": 1024}, + {"shortest_edge": 64, "longest_edge": 0}, + ), + ], +) +def test_get_vision_size( + mm_kwargs: dict[str, object], + default_size: dict[str, object] | None, + expected: dict[str, object], +) -> None: + original_mm_kwargs = { + key: dict(value) if isinstance(value, dict) else value + for key, value in mm_kwargs.items() + } + original_default_size = dict(default_size) if default_size is not None else None + + actual = Qwen2VLProcessingInfo._get_vision_size( + mm_kwargs, + default_size=default_size, + ) + + assert actual == expected + assert mm_kwargs == original_mm_kwargs + assert default_size == original_default_size + + +@pytest.mark.parametrize( + ("modality", "scope"), + [ + ("image", "images_kwargs"), + ("video", "videos_kwargs"), + ], +) +def test_get_vision_info_reads_merged_modality_scope( + monkeypatch: pytest.MonkeyPatch, + modality: str, + scope: str, +) -> None: + vision_config = SimpleNamespace( + patch_size=14, + spatial_merge_size=2, + temporal_patch_size=2, + ) + ctx = SimpleNamespace( + get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config) + ) + info = Qwen2VLProcessingInfo(ctx) + merged = { + "images_kwargs": {"size": {"shortest_edge": 111}}, + "videos_kwargs": {"size": {"shortest_edge": 222}}, + } + monkeypatch.setattr( + info, + "_merge_and_resolve_mm_processor_kwargs", + lambda _: merged, + ) + + seen: list[dict[str, object]] = [] + + def get_size( + mm_kwargs: dict[str, object], + default_size: dict[str, object] | None = None, + ) -> dict[str, object]: + seen.append(mm_kwargs) + return dict(default_size or {}) + + monkeypatch.setattr(info, "_get_vision_size", get_size) + image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024}) + + info._get_vision_info( + image_width=28, + image_height=28, + num_frames=2, + do_resize=False, + image_processor=image_processor, + mm_kwargs={}, + modality=modality, # type: ignore[arg-type] + ) + + assert seen == [merged[scope]] + + def test_jina_vl_processing_order() -> None: """Jina's document-first prompt keeps cached features and hashes aligned.""" ctx = build_model_context( diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index 04f87049c9..2428109815 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -1,19 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Regression tests for Qwen3-VL processor. - -Covers the fix for num_frames-based timestamp calculation -(issue vllm-project/vllm#35909). -""" +"""Regression tests for Qwen3-VL multimodal processing.""" +from copy import deepcopy +from types import SimpleNamespace from typing import Any import numpy as np import pytest +import torch +from transformers.models.qwen3_vl import Qwen3VLVideoProcessor from vllm.config import ModelConfig from vllm.config.multimodal import MultiModalDummyOptions +from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.processing.context import BaseProcessingInfo from ...registry import HF_EXAMPLE_MODELS from ...utils import build_model_context @@ -21,6 +23,32 @@ from ...utils import build_model_context MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct" +def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__) + assert "size" in native_video_kwargs + assert "min_pixels" not in native_video_kwargs + assert "max_pixels" not in native_video_kwargs + + info = Qwen3VLProcessingInfo(SimpleNamespace()) + parent_supported = { + "text_kwargs": {"padding"}, + "images_kwargs": {"size", "min_pixels", "max_pixels"}, + "videos_kwargs": set(native_video_kwargs), + } + monkeypatch.setattr( + BaseProcessingInfo, + "get_supported_mm_processor_kwargs", + lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()}, + ) + + supported = info.supported_mm_processor_kwargs + + assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"] + assert parent_supported["videos_kwargs"] == native_video_kwargs + + def _build_video_mm_data( num_frames: int, width: int = 128, @@ -45,6 +73,122 @@ def _build_video_mm_data( return {"video": [(video, metadata)]} +@pytest.mark.parametrize("model_id", [MODEL_ID]) +def test_processor_partial_scoped_image_size(model_id: str) -> None: + """Partial image ``size`` must be completed and applied before the HF call. + + Keep video compatibility and per-video kwargs in an image-only call as well: + Qwen3 must sanitize the video scope before HF validates it. Regression for + #56363. + """ + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 1, "video": 0}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + image = np.zeros((128, 128, 3), dtype=np.uint8) + min_pixels = 256 * 32 * 32 + default_size = dict(processor.info.get_image_processor().size) + complete_size = default_size | {"shortest_edge": min_pixels} + + def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: + mm_items = processor.info.parse_mm_data({"image": [image]}) + return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ + "image_grid_thw" + ] + + stock_grid = process({}) + complete_grid = process({"images_kwargs": {"size": complete_size}}) + partial_grid = process( + { + "images_kwargs": { + "size": {"shortest_edge": min_pixels}, + }, + "videos_kwargs": { + "min_pixels": 4096, + "max_pixels": 469_762_048, + "fps": [2.0], + "num_frames": [8], + }, + } + ) + + assert torch.equal(partial_grid, complete_grid) + assert not torch.equal(partial_grid, stock_grid) + + +@pytest.mark.parametrize("model_id", [MODEL_ID]) +def test_processor_partial_scoped_video_size(model_id: str) -> None: + """Partial scoped video ``size`` must be completed and applied before HF.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 0, "video": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + min_pixels = 512 * 32 * 32 + default_size = dict(processor.info.get_video_processor().size) + complete_size = default_size | {"shortest_edge": min_pixels} + + def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: + mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8)) + return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ + "video_grid_thw" + ] + + stock_grid = process({}) + complete_grid = process({"videos_kwargs": {"size": complete_size}}) + partial_grid = process( + { + "videos_kwargs": { + "size": {"shortest_edge": min_pixels}, + }, + } + ) + + assert torch.equal(partial_grid, complete_grid) + assert not torch.equal(partial_grid, stock_grid) + + +@pytest.mark.parametrize("model_id", [MODEL_ID]) +def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None: + """Flat min/max pixel aliases keep their historical Qwen3 video effect.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 0, "video": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + pixel_budget = 512 * 32 * 32 + + def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor: + mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8)) + return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[ + "video_grid_thw" + ] + + stock_grid = process({}) + scoped_grid = process( + { + "videos_kwargs": { + "size": { + "shortest_edge": pixel_budget, + "longest_edge": pixel_budget, + }, + }, + } + ) + flat_grid = process( + { + "min_pixels": pixel_budget, + "max_pixels": pixel_budget, + } + ) + + assert torch.equal(flat_grid, scoped_grid) + assert not torch.equal(flat_grid, stock_grid) + + @pytest.mark.parametrize("model_id", [MODEL_ID]) @pytest.mark.parametrize( "num_frames", @@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget( @pytest.mark.parametrize("model_id", [MODEL_ID]) @pytest.mark.parametrize( - "hf_mm_kwargs", - [{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}], + ("hf_mm_kwargs", "scalar_kwargs"), + [ + ({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]), + ({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]), + ], ) def test_processor_multi_video_list_kwargs( model_id: str, hf_mm_kwargs: dict[str, Any], + scalar_kwargs: list[dict[str, Any]], ) -> None: - """Regression test: a multi-video request with list-valued per-video - ``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not - crash. + """List-valued video kwargs must be scalarized per item without mutation. - Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every - video without slicing, so ``_get_video_second_idx`` received the list - where a scalar was expected and raised ``TypeError``. + Each video in the multi-video request must match processing the same input + independently with its corresponding scalar ``fps``/``num_frames`` value. """ ctx = build_model_context( model_id, @@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs( ) mm_data = { "video": [ - _build_video_mm_data(num_frames=16)["video"][0], + _build_video_mm_data(num_frames=32)["video"][0], _build_video_mm_data(num_frames=32)["video"][0], ] } + hf_mm_kwargs_before = deepcopy(hf_mm_kwargs) processed = processor( prompt, mm_items=processor.info.parse_mm_data(mm_data), hf_processor_mm_kwargs=hf_mm_kwargs, ) + def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor: + single_prompt = "<|vision_start|><|video_pad|><|vision_end|>" + single_mm_data = _build_video_mm_data(num_frames=32) + single = processor( + single_prompt, + mm_items=processor.info.parse_mm_data(single_mm_data), + hf_processor_mm_kwargs=hf_kwargs, + ) + return single["mm_kwargs"].get_data()["video_grid_thw"][0] + + expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs] + multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"] + + assert not torch.equal(expected_grids[0], expected_grids[1]) + assert torch.equal(multi_grids[0], expected_grids[0]) + assert torch.equal(multi_grids[1], expected_grids[1]) + assert hf_mm_kwargs == hf_mm_kwargs_before + video_phs = processed["mm_placeholders"].get("video", []) assert len(video_phs) == 2, ( f"Expected exactly 2 video placeholders, got {len(video_phs)}" @@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non capped_ctx = build_model_context( model_id, - mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True}, + mm_processor_kwargs={ + "videos_kwargs": { + "size": size, + "cap_pixels_per_frame": True, + }, + }, limit_mm_per_prompt={"image": 0, "video": 1}, ) capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config) @@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non uncapped_ctx = build_model_context( model_id, - mm_processor_kwargs={"size": size}, + mm_processor_kwargs={"videos_kwargs": {"size": size}}, limit_mm_per_prompt={"image": 0, "video": 1}, ) uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config) diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 0bd19a0ea0..485ea31659 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -3,18 +3,23 @@ import time from contextlib import nullcontext +from copy import deepcopy +from types import SimpleNamespace +from typing import TypedDict import numpy as np import pytest from vllm.config import ModelConfig, SchedulerConfig +from vllm.config.multimodal import MultiModalConfig from vllm.exceptions import VLLMValidationError from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.hasher import MultiModalHasher from vllm.multimodal.parse import MultiModalDataParser from vllm.multimodal.processing.context import ( + BaseProcessingInfo, InputProcessingContext, - overlay_modality_mm_kwargs, + _resolve_mm_processor_kwargs, ) from vllm.multimodal.processing.inputs import ProcessorInputs from vllm.multimodal.processing.processor import ( @@ -1207,58 +1212,305 @@ def test_apply_prompt_updates_falls_back_with_index_targets(): assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]] -@pytest.mark.skip_global_cleanup -def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image(): - """HF-style videos_kwargs must overlay only when modality is video.""" - video_size = {"longest_edge": 469762048, "shortest_edge": 4096} - kwargs = {"videos_kwargs": {"size": video_size}} - - assert overlay_modality_mm_kwargs(kwargs, None) == kwargs - assert "size" not in overlay_modality_mm_kwargs(kwargs, "image") - assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size +class _TextProcessorKwargs(TypedDict, total=False): + padding: bool -@pytest.mark.skip_global_cleanup -def test_overlay_modality_mm_kwargs_flat_size_stays_shared(): - """A flat size override keeps the current shared-namespace behavior.""" - size = {"longest_edge": 469762048, "shortest_edge": 4096} - kwargs = {"size": size} - - for modality in (None, "image", "video"): - assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size +class _AudioProcessorKwargs(TypedDict, total=False): + sampling_rate: int -@pytest.mark.skip_global_cleanup -def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality(): - """A nested videos_kwargs size wins over a flat size for video reads.""" +class _ProcessorKwargs(TypedDict, total=False): + text_kwargs: _TextProcessorKwargs + audio_kwargs: _AudioProcessorKwargs + + +class _ImageProcessorKwargs(TypedDict, total=False): + size: dict[str, int] + min_pixels: int + + +class _VideoProcessorKwargs(TypedDict, total=False): + size: dict[str, int] + fps: float + + +@pytest.mark.parametrize( + ("supported_mm_limits", "processor", "expected"), + [ + ( + {"image": None, "video": None}, + SimpleNamespace( + valid_processor_kwargs=_ProcessorKwargs, + image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs), + video_processor=SimpleNamespace(valid_kwargs=_VideoProcessorKwargs), + ), + { + "text_kwargs": {"padding"}, + "images_kwargs": {"size", "min_pixels"}, + "videos_kwargs": {"size", "fps"}, + }, + ), + ( + {"image": None}, + SimpleNamespace( + valid_processor_kwargs=_ProcessorKwargs, + image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs), + ), + { + "text_kwargs": {"padding"}, + "images_kwargs": {"size", "min_pixels"}, + }, + ), + ( + {"audio": None}, + SimpleNamespace(valid_processor_kwargs=_ProcessorKwargs), + { + "text_kwargs": {"padding"}, + "audio_kwargs": {"sampling_rate"}, + }, + ), + ], +) +def test_get_supported_mm_processor_kwargs_uses_supported_modalities( + monkeypatch: pytest.MonkeyPatch, + supported_mm_limits: dict[str, int | None], + processor: SimpleNamespace, + expected: dict[str, set[str]], +) -> None: + info = BaseProcessingInfo(SimpleNamespace()) + info.__dict__["supported_mm_limits"] = supported_mm_limits + monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor) + + assert info.get_supported_mm_processor_kwargs() == expected + + +def test_supported_mm_processor_kwargs_is_cached( + monkeypatch: pytest.MonkeyPatch, +) -> None: + info = BaseProcessingInfo(SimpleNamespace()) + expected = {"images_kwargs": {"size"}} + calls = 0 + + def get_supported_mm_processor_kwargs() -> dict[str, set[str]]: + nonlocal calls + calls += 1 + return expected + + monkeypatch.setattr( + info, + "get_supported_mm_processor_kwargs", + get_supported_mm_processor_kwargs, + ) + + first = info.supported_mm_processor_kwargs + second = info.supported_mm_processor_kwargs + + assert first is expected + assert second is first + assert calls == 1 + + +def test_resolve_mm_processor_kwargs_routes_supported_flat_kwargs(): kwargs = { - "size": {"longest_edge": 1}, - "videos_kwargs": {"size": {"longest_edge": 2}}, - "images_kwargs": {"size": {"longest_edge": 3}}, + "size": {"shortest_edge": 64}, + "padding": True, + "sampling_rate": 16000, + "unknown": "keep", + "images_kwargs": { + "size": {"longest_edge": 1024}, + "custom_image_kwarg": "keep", + }, + } + kwargs_before = deepcopy(kwargs) + + resolved = _resolve_mm_processor_kwargs( + kwargs, + supported_mm_processor_kwargs={ + "text_kwargs": {"padding"}, + "images_kwargs": {"size"}, + "videos_kwargs": {"size"}, + "audio_kwargs": {"sampling_rate"}, + }, + ) + + assert resolved == { + "unknown": "keep", + "text_kwargs": {"padding": True}, + "images_kwargs": { + "size": { + "shortest_edge": 64, + "longest_edge": 1024, + }, + "custom_image_kwarg": "keep", + }, + "videos_kwargs": { + "size": {"shortest_edge": 64}, + }, + "audio_kwargs": {"sampling_rate": 16000}, + } + assert kwargs == kwargs_before + + video_kwargs = resolved["videos_kwargs"] + image_kwargs = resolved["images_kwargs"] + assert isinstance(video_kwargs, dict) + assert isinstance(image_kwargs, dict) + video_size = video_kwargs["size"] + image_size = image_kwargs["size"] + assert isinstance(video_size, dict) + assert isinstance(image_size, dict) + video_size["shortest_edge"] = 999 + + assert image_size["shortest_edge"] == 64 + assert kwargs == kwargs_before + + +def test_resolve_mm_processor_kwargs_rejects_non_mapping_destination_scope(): + with pytest.raises(TypeError, match="images_kwargs"): + _resolve_mm_processor_kwargs( + { + "size": {"shortest_edge": 64}, + "images_kwargs": 123, + }, + supported_mm_processor_kwargs={ + "images_kwargs": {"size"}, + }, + ) + + +def test_resolve_mm_processor_kwargs_without_schema_deduplicates_existing_scopes(): + kwargs = { + "size": {"shortest_edge": 64}, + "num_frames": 16, + "fps": 4, + "images_kwargs": { + "size": {"longest_edge": 1024}, + }, + "videos_kwargs": { + "num_frames": 8, + }, + "audio_kwargs": 123, + "common_kwargs": { + "fps": 2, + }, + } + kwargs_before = deepcopy(kwargs) + + resolved = _resolve_mm_processor_kwargs(kwargs) + + assert resolved == { + "fps": 4, + "images_kwargs": { + "size": {"longest_edge": 1024}, + }, + "videos_kwargs": { + "num_frames": 8, + }, + "audio_kwargs": 123, + "common_kwargs": { + "fps": 2, + }, + } + assert kwargs == kwargs_before + assert "text_kwargs" not in resolved + + +@pytest.mark.parametrize( + "inference_kwargs", + [ + {"size": {"shortest_edge": 128}}, + {"size": {"shortest_edge": 128}, "images_kwargs": None}, + {"size": {"shortest_edge": 128}, "images_kwargs": {}}, + ], +) +def test_get_merged_mm_kwargs_treats_empty_scopes_as_absent_before_routing( + inference_kwargs: dict[str, object], +): + mm_config = MultiModalConfig( + mm_processor_kwargs={ + "size": { + "shortest_edge": 64, + "longest_edge": 512, + }, + "images_kwargs": { + "size": {"longest_edge": 1024}, + }, + }, + mm_device_do_normalize=False, + ) + model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config) + ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type] + + merged = ctx.get_merged_mm_kwargs( + inference_kwargs, + supported_mm_processor_kwargs={ + "images_kwargs": {"size"}, + "videos_kwargs": {"size"}, + }, + ) + + assert merged == { + "images_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 1024, + }, + }, + "videos_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 512, + }, + }, } - assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2} - assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3} - assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1} +def test_get_merged_mm_kwargs_merges_before_routing(): + mm_config = MultiModalConfig( + mm_processor_kwargs={ + "size": { + "shortest_edge": 64, + "longest_edge": 512, + }, + "images_kwargs": { + "size": {"longest_edge": 1024}, + }, + }, + mm_device_do_normalize=False, + ) + model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config) + ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type] -@pytest.mark.skip_global_cleanup -def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value(): - kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}} - assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1} + merged = ctx.get_merged_mm_kwargs( + { + "size": { + "shortest_edge": 128, + "longest_edge": 768, + }, + "images_kwargs": { + "size": {"longest_edge": 2048}, + }, + }, + supported_mm_processor_kwargs={ + "images_kwargs": {"size"}, + "videos_kwargs": {"size"}, + }, + ) - -@pytest.mark.skip_global_cleanup -def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping(): - """Configured videos_kwargs overlay only for video reads after merge.""" - from vllm.config.multimodal import MultiModalConfig - - size = {"longest_edge": 469762048, "shortest_edge": 4096} - mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}}) - merged = mm_config.merge_mm_processor_kwargs({}) - assert overlay_modality_mm_kwargs(merged, "video")["size"] == size - assert "size" not in overlay_modality_mm_kwargs(merged, "image") - assert "size" not in overlay_modality_mm_kwargs(merged, None) + assert merged == { + "images_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 2048, + }, + }, + "videos_kwargs": { + "size": { + "shortest_edge": 128, + "longest_edge": 768, + }, + }, + } def test_processor_inputs_hashes_partial_uuids(): diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 5ce309456b..24c534b364 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -27,6 +27,186 @@ logger = init_logger(__name__) _T = TypeVar("_T") +# HuggingFace processors accept a shared flat namespace in addition to the +# nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` / +# ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when +# resolving ``mm_processor_kwargs``. +_HF_PROCESSOR_KWARG_SCOPES = ( + "text_kwargs", + "images_kwargs", + "videos_kwargs", + "audio_kwargs", +) + + +def _recursively_merge_mm_processor_kwargs( + defaults: Mapping[str, object], + overrides: Mapping[str, object], +) -> dict[str, object]: + """Merge two mappings recursively, with ``overrides`` taking precedence. + + Nested values are merged only when both values are mappings; otherwise the + override replaces the existing value. + """ + merged = dict(defaults) + + for key, value in overrides.items(): + default_value = merged.get(key) + if isinstance(default_value, Mapping) and isinstance(value, Mapping): + merged[key] = _recursively_merge_mm_processor_kwargs(default_value, value) + else: + merged[key] = value + + return merged + + +def _merge_mm_processor_scope( + config_flat: Mapping[str, object], + config_scoped: Mapping[str, object], + inference_flat: Mapping[str, object], + inference_scoped: Mapping[str, object], +) -> dict[str, object]: + """Merge kwargs for one existing HuggingFace processor kwarg scope. + + Flat kwargs contribute only to keys present in either the configured or + inference-time scope. For those keys, precedence is: + configured flat < configured scoped < inference flat < inference scoped. + + Nested mappings are merged recursively; otherwise the higher-priority value + replaces the lower-priority value. + """ + # Flat kwargs contribute only to keys present in this scope in either input. + scoped_keys = config_scoped.keys() | inference_scoped.keys() + config_flat_for_scope = { + key: value for key, value in config_flat.items() if key in scoped_keys + } + inference_flat_for_scope = { + key: value for key, value in inference_flat.items() if key in scoped_keys + } + + # Merge from lowest to highest priority: + # configured flat -> configured scoped -> inference flat -> inference scoped. + merged = _recursively_merge_mm_processor_kwargs( + config_flat_for_scope, config_scoped + ) + merged = _recursively_merge_mm_processor_kwargs(merged, inference_flat_for_scope) + merged = _recursively_merge_mm_processor_kwargs(merged, inference_scoped) + return merged + + +def _merge_mm_processor_kwargs( + config_kwargs: Mapping[str, object], + inference_kwargs: Mapping[str, object], +) -> dict[str, object]: + """Merge configured and inference-time multi-modal processor kwargs. + + The merge is performed in two parts: + + 1. The shared flat kwargs are merged recursively, with inference-time values + taking precedence. + 2. Each existing HuggingFace processor kwarg scope is merged separately. + For keys present in either the configured or inference-time scope, + precedence is: + configured flat < configured scoped < inference flat < inference scoped. + + For example: + + configured = { + "size": {"shortest_edge": 256}, + "videos_kwargs": { + "size": {"longest_edge": 1024}, + "num_frames": 16, + }, + } + inference = { + "size": {"shortest_edge": 512}, + "videos_kwargs": {"fps": 4}, + } + + produces: + + { + "size": {"shortest_edge": 512}, + "videos_kwargs": { + "size": { + "shortest_edge": 512, + "longest_edge": 1024, + }, + "num_frames": 16, + "fps": 4, + }, + } + + Scoped values do not modify the shared flat kwargs or sibling scopes. + Missing scopes, None-valued scopes, and empty scope mappings are treated as + absent and do not create a scope. + """ + # Merge the shared flat kwargs first; nested scopes are handled separately + # below. + config_flat = { + key: value + for key, value in config_kwargs.items() + if key not in _HF_PROCESSOR_KWARG_SCOPES + } + inference_flat = { + key: value + for key, value in inference_kwargs.items() + if key not in _HF_PROCESSOR_KWARG_SCOPES + } + merged = _recursively_merge_mm_processor_kwargs(config_flat, inference_flat) + + # Merge each nested processor kwarg scope present in either input. + for scoped_key in _HF_PROCESSOR_KWARG_SCOPES: + config_scope_value = config_kwargs.get(scoped_key) + inference_scope_value = inference_kwargs.get(scoped_key) + + # Treat missing scopes, None-valued scopes, and empty scope mappings as + # absent. + if isinstance(config_scope_value, Mapping) and not config_scope_value: + config_scope_value = None + if isinstance(inference_scope_value, Mapping) and not inference_scope_value: + inference_scope_value = None + + # Skip adding the scope to the merged result when it is absent from both + # sources. + if config_scope_value is None and inference_scope_value is None: + continue + + # Handle the inference scope first because it has higher precedence; only + # mapping-valued scopes participate in the recursive scoped merge. + inference_scope_kwargs: Mapping[str, object] + if inference_scope_value is None: + inference_scope_kwargs = {} + elif not isinstance(inference_scope_value, Mapping): + # Non-mapping scopes cannot participate in the recursive scoped merge; + # the higher-priority inference value replaces the configured scope. + merged[scoped_key] = inference_scope_value + continue + else: + inference_scope_kwargs = inference_scope_value + + # Handle the configured scope next because it has lower precedence; only + # mapping-valued scopes participate in the recursive scoped merge. + if isinstance(config_scope_value, Mapping): + config_scope_kwargs = config_scope_value + elif inference_scope_value is None: + # Non-mapping scopes cannot participate in the recursive scoped merge; + # preserve the configured value when inference contributes nothing. + merged[scoped_key] = config_scope_value + continue + else: + config_scope_kwargs = {} + + # Merge this scope together with the matching flat kwargs. + merged[scoped_key] = _merge_mm_processor_scope( + config_flat, + config_scope_kwargs, + inference_flat, + inference_scope_kwargs, + ) + + return merged + @dataclass class BaseDummyOptions: @@ -533,8 +713,14 @@ class MultiModalConfig: ) -> dict[str, object]: """Get the keyword arguments to pass to the multi-modal processor according to the extra arguments passed during inference. + + Nested mappings are merged recursively, with inference-time values taking + precedence over configured values. """ - merged = dict(self.mm_processor_kwargs or {}) | dict(inference_kwargs) + merged = _merge_mm_processor_kwargs( + self.mm_processor_kwargs or {}, + inference_kwargs, + ) if self.mm_device_do_normalize: # The model expects raw pixels and normalises on device; # overriding these per request would silently double-normalise. diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index a70d8e1de5..593ba90ec0 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -3,7 +3,7 @@ import math from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Literal +from typing import Annotated, Any, Literal, cast import torch import torch.nn as nn @@ -212,8 +212,14 @@ class Mistral3ProcessingInfo(BaseProcessingInfo): ) -> PixtralHFEncoderInfo: processor = self.get_hf_processor() size = processor.image_processor.size - merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {}) - if override_size := merged_kwargs.get("size"): + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs( + mm_processor_kwargs or {} + ) + image_mm_kwargs = cast( + Mapping[str, Any], + merged_mm_kwargs.get("images_kwargs", {}), + ) + if override_size := image_mm_kwargs.get("size"): size = size | override_size image_size = size["longest_edge"] diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 5a7ccacc48..7f0ed0e6a7 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -28,7 +28,7 @@ import math from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence from functools import partial -from typing import Annotated, Any, Literal, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias, cast import numpy as np import torch @@ -865,6 +865,118 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": None, "video": None} + @staticmethod + def _complete_mm_processor_size_aliases( + mm_kwargs: Mapping[str, object], + ) -> dict[str, object]: + """Complete missing Qwen-VL size aliases. + + Qwen-VL accepts ``size.shortest_edge`` / ``min_pixels`` and + ``size.longest_edge`` / ``max_pixels`` as equivalent controls. When + only one representation is provided with a non-``None`` value, add the + missing equivalent representation. If both are already present, leave + them unchanged even when their values conflict. + + Apply this independently to the flat kwargs and to each existing + ``images_kwargs`` / ``videos_kwargs`` mapping. Values are not propagated + between these mappings. + """ + + def add_missing_size_aliases( + kwargs: Mapping[str, object], + ) -> dict[str, object]: + """Add missing size aliases within a kwargs mapping.""" + out_kwargs = dict(kwargs) + # Copy `size` separately because `out_kwargs` is a shallow copy and + # size edges may be added below. + raw_size = cast( + Mapping[str, object] | None, + out_kwargs.get("size"), + ) + size_kwargs = {} if raw_size is None else dict(raw_size) + + for edge_key, pixel_key in ( + ("shortest_edge", "min_pixels"), + ("longest_edge", "max_pixels"), + ): + # Add the pixel alias from `size` when only the size edge is + # present. + if pixel_key not in out_kwargs: + if edge_key in size_kwargs and size_kwargs[edge_key] is not None: + out_kwargs[pixel_key] = size_kwargs[edge_key] + continue + + pixel_value = out_kwargs[pixel_key] + # HF treats `min_pixels` / `max_pixels` set to None as no size + # override, so do not propagate them to the equivalent size edge. + if pixel_value is None: + continue + + # Add the size edge when only the pixel alias is present. + if edge_key not in size_kwargs: + size_kwargs[edge_key] = pixel_value + out_kwargs["size"] = size_kwargs + + return out_kwargs + + prepared_mm_kwargs = add_missing_size_aliases(mm_kwargs) + # Complete aliases independently inside existing image/video kwargs; + # flat values are not copied into the nested mappings. + for nested_kwargs_key in ("images_kwargs", "videos_kwargs"): + nested_kwargs = prepared_mm_kwargs.get(nested_kwargs_key) + if isinstance(nested_kwargs, Mapping): + prepared_mm_kwargs[nested_kwargs_key] = add_missing_size_aliases( + nested_kwargs + ) + + return prepared_mm_kwargs + + def _merge_and_resolve_mm_processor_kwargs( + self, + mm_kwargs: Mapping[str, object], + ) -> dict[str, Any]: + """Merge configured and request Qwen-VL ``mm_processor_kwargs``. + + Request-side ``size.shortest_edge`` / ``min_pixels`` and + ``size.longest_edge`` / ``max_pixels`` aliases are completed before the + merge so request precedence is preserved across equivalent + representations. + """ + # Complete request-side Qwen-VL size aliases before merging with + # configured kwargs. + prepared_mm_kwargs = self._complete_mm_processor_size_aliases(mm_kwargs) + + return super()._merge_and_resolve_mm_processor_kwargs(prepared_mm_kwargs) + + @staticmethod + def _get_vision_size( + mm_kwargs: Mapping[str, object], + default_size: Mapping[str, int] | None = None, + ) -> dict[str, int]: + """Resolve Qwen-VL vision size kwargs with optional processor defaults. + + ``size`` overrides any processor defaults, then ``min_pixels`` and + ``max_pixels`` override the corresponding size edges. + """ + size = dict(default_size) if default_size is not None else {} + + override_size = cast( + Mapping[str, int] | None, + mm_kwargs.get("size"), + ) + if override_size is not None: + size.update(override_size) + + min_pixels = cast(int | None, mm_kwargs.get("min_pixels")) + if min_pixels is not None: + size["shortest_edge"] = min_pixels + + max_pixels = cast(int | None, mm_kwargs.get("max_pixels")) + if max_pixels is not None: + size["longest_edge"] = max_pixels + + return size + def get_mm_max_tokens_per_item( self, seq_len: int, @@ -883,7 +995,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): do_resize: bool = True, image_processor: Qwen2VLImageProcessor, mm_kwargs: Mapping[str, object], - modality: str | None = None, + modality: Literal["image", "video"] | None = None, ) -> tuple[ImageSize, int]: hf_config = self.get_hf_config() vision_config = hf_config.vision_config @@ -891,14 +1003,16 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): merge_size = vision_config.spatial_merge_size temporal_patch_size = vision_config.temporal_patch_size - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality) - size = image_processor.size - if override_size := mm_kwargs.get("size"): - size = size | override_size - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None: - size = size | {"shortest_edge": override_min_pixels} - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: - size = size | {"longest_edge": override_max_pixels} + if modality is None: + modality = "image" + + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs) + scope = "images_kwargs" if modality == "image" else "videos_kwargs" + scoped_mm_kwargs = merged_mm_kwargs.get(scope, {}) + size = self._get_vision_size( + scoped_mm_kwargs, + default_size=image_processor.size, + ) if do_resize: resized_height, resized_width = smart_resize( @@ -986,14 +1100,12 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): if max_pixels is None: image_processor = self.get_image_processor() - mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image") - size = image_processor.size - if override_size := mm_kwargs.get("size"): - size = size | override_size - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None: - size = size | {"shortest_edge": override_min_pixels} - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: - size = size | {"longest_edge": override_max_pixels} + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({}) + image_mm_kwargs = merged_mm_kwargs.get("images_kwargs", {}) + size = self._get_vision_size( + image_mm_kwargs, + default_size=image_processor.size, + ) max_pixels = size["longest_edge"] diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index d534d408ca..6648949c73 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -27,7 +27,7 @@ from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence from functools import lru_cache, partial from itertools import islice -from typing import Any +from typing import Any, Literal import numpy as np import torch @@ -944,6 +944,16 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, ) + def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]: + supported = super().get_supported_mm_processor_kwargs() + + # vLLM has historically accepted `min_pixels` / `max_pixels` for + # Qwen3-VL video. Include them in the routing schema even though + # Qwen3VLVideoProcessor only accepts the equivalent `size`; they are + # folded into `size` before calling HF. + supported["videos_kwargs"].update({"min_pixels", "max_pixels"}) + return supported + def _get_vision_info( self, *, @@ -953,7 +963,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): do_resize: bool = True, image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor, mm_kwargs: Mapping[str, object], - modality: str | None = None, + modality: Literal["image", "video"] | None = None, ) -> tuple[ImageSize, int]: is_video = isinstance(image_processor, Qwen3VLVideoProcessor) @@ -965,14 +975,14 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): if modality is None: modality = "video" if is_video else "image" - mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality) - size = image_processor.size - if override_size := mm_kwargs.get("size"): - size = size | override_size - if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None: - size = size | {"shortest_edge": override_min_pixels} - if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: - size = size | {"longest_edge": override_max_pixels} + + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs) + scope = "videos_kwargs" if modality == "video" else "images_kwargs" + scoped_mm_kwargs = merged_mm_kwargs.get(scope, {}) + size = self._get_vision_size( + scoped_mm_kwargs, + default_size=image_processor.size, + ) if do_resize: if is_video: @@ -1030,9 +1040,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): ) -> int: video_processor = self.get_video_processor() - mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video") - video_size = mm_kwargs.get("size", video_processor.size) - temporal_patch_size = mm_kwargs.get( + merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({}) + video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {}) + video_size = self._get_vision_size( + video_mm_kwargs, + default_size=video_processor.size, + ) + temporal_patch_size = video_mm_kwargs.get( "temporal_patch_size", video_processor.temporal_patch_size ) @@ -1162,9 +1176,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): video_processor = self.info.get_video_processor() - mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video") - video_size = mm_kwargs.get("size", video_processor.size) - temporal_patch_size = mm_kwargs.get( + merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs({}) + video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {}) + video_size = self.info._get_vision_size( + video_mm_kwargs, + default_size=video_processor.size, + ) + temporal_patch_size = video_mm_kwargs.get( "temporal_patch_size", video_processor.temporal_patch_size ) @@ -1180,13 +1198,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): # over enough frames that the cap is not binding for the dummy. # This takes precedence over a shrinking num_frames override, which # would otherwise reintroduce the under-profiling. - if mm_kwargs.get("cap_pixels_per_frame"): - max_video_tokens = mm_kwargs.get( + if video_mm_kwargs.get("cap_pixels_per_frame"): + max_video_tokens = video_mm_kwargs.get( "max_video_tokens", getattr(video_processor, "max_video_tokens", 768), ) - patch_size = mm_kwargs.get("patch_size", video_processor.patch_size) - merge_size = mm_kwargs.get("merge_size", video_processor.merge_size) + patch_size = video_mm_kwargs.get("patch_size", video_processor.patch_size) + merge_size = video_mm_kwargs.get("merge_size", video_processor.merge_size) per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2 target_num_frames = max( target_num_frames, @@ -1348,6 +1366,47 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) if not hf_data: return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data) + # Merge and resolve the multimodal processor kwargs once so every HF + # call below starts from the same engine/request state and scoped form. + merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs(hf_kwargs) + + if "images_kwargs" in merged_mm_kwargs: + image_scoped_kwargs = dict(merged_mm_kwargs["images_kwargs"]) + + # Qwen2VLImageProcessor requires both `size` edges. If a partial size + # override is passed, fill missing edges from the processor defaults + # before any HF call. + if image_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}: + default_image_size = self.info.get_image_processor().size + image_scoped_kwargs["size"] = self.info._get_vision_size( + image_scoped_kwargs, + default_size=default_image_size, + ) + + merged_mm_kwargs["images_kwargs"] = image_scoped_kwargs + + if "videos_kwargs" in merged_mm_kwargs: + video_scoped_kwargs = dict(merged_mm_kwargs["videos_kwargs"]) + + # Qwen3VLVideoProcessor requires both `size` edges. If a partial size + # override is passed, fill missing edges from the processor defaults + # before any HF call. + if video_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}: + default_video_size = self.info.get_video_processor().size + video_scoped_kwargs["size"] = self.info._get_vision_size( + video_scoped_kwargs, + default_size=default_video_size, + ) + + # Qwen3VLVideoProcessor does not accept vLLM's min/max pixel + # compatibility kwargs. They have already been folded into `size`, + # so remove them before any HF call, including calls that contain + # only text/images. + video_scoped_kwargs.pop("min_pixels", None) + video_scoped_kwargs.pop("max_pixels", None) + + merged_mm_kwargs["videos_kwargs"] = video_scoped_kwargs + # Separate video processing from image processing. Because the videos # are processed into several image patches video_input_ids_lst: list[list[int]] = [] @@ -1373,32 +1432,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # used to calculate the timestamps. Make sure that # do_sample_frames in mm_kwargs is false for presampled videos. - # NOTE: a copy of is created to update do_sample_frames, - # otherwise mm_hash for the object will be incorrect. - video_mm_kwargs: dict[str, Any] = dict(hf_kwargs) - merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video") - if merged.keys() & {"size", "min_pixels", "max_pixels"}: - video_size = dict(self.info.get_video_processor().size) - size_override = merged.get("size") - if size_override is not None: - video_size = video_size | size_override - min_pixels = merged.get("min_pixels") - if min_pixels is not None: - video_size["shortest_edge"] = min_pixels - max_pixels = merged.get("max_pixels") - if max_pixels is not None: - video_size["longest_edge"] = max_pixels - video_mm_kwargs["size"] = video_size - sampled_fps = video_mm_kwargs.get("fps") + # The root copy is shallow, so give each item its own video + # scope before modifying per-video values below. + video_mm_kwargs = dict(merged_mm_kwargs) + video_scoped_kwargs = dict(video_mm_kwargs.get("videos_kwargs", {})) + video_mm_kwargs["videos_kwargs"] = video_scoped_kwargs + + sampled_fps = video_scoped_kwargs.get("fps") if is_list_of(sampled_fps, float): - video_mm_kwargs["fps"] = sampled_fps[item_idx] - sampled_num_frames = video_mm_kwargs.get("num_frames") + video_scoped_kwargs["fps"] = sampled_fps[item_idx] + sampled_num_frames = video_scoped_kwargs.get("num_frames") if is_list_of(sampled_num_frames, int): - video_mm_kwargs["num_frames"] = sampled_num_frames[item_idx] - if "do_sample_frames" not in video_mm_kwargs: + video_scoped_kwargs["num_frames"] = sampled_num_frames[item_idx] + if "do_sample_frames" not in video_scoped_kwargs: # qwen_vl_utils already has "do_sample_frames" in # mm_kwargs, don't overwrite it. - video_mm_kwargs["do_sample_frames"] = metadata.get( + video_scoped_kwargs["do_sample_frames"] = metadata.get( "do_sample_frames", False ) @@ -1409,9 +1458,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # Compute timestamps here where we have access to metadata timestamps = self.info._get_video_second_idx( metadata=metadata, - do_sample_frames=video_mm_kwargs["do_sample_frames"], - sampled_fps=video_mm_kwargs.get("fps"), - sampled_num_frames=video_mm_kwargs.get("num_frames"), + do_sample_frames=video_scoped_kwargs["do_sample_frames"], + sampled_fps=video_scoped_kwargs.get("fps"), + sampled_num_frames=video_scoped_kwargs.get("num_frames"), ) timestamps_per_video.append(timestamps) @@ -1423,8 +1472,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # to prevent HF's BaseVideoProcessor.preprocess() from # filling in the class default (fps=2) via setdefault(), # which would conflict with num_frames (mutually exclusive). - if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs: - video_mm_kwargs["fps"] = None + if ( + "num_frames" in video_scoped_kwargs + and "fps" not in video_scoped_kwargs + ): + video_scoped_kwargs["fps"] = None video_outputs = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**video_mm_kwargs), @@ -1485,11 +1537,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) else: video_outputs = dict() - # fps/num_frames are video-only kwargs already consumed by the loop; - # exclude them so the text/image processor call below never gets a list. - non_video_mm_kwargs = { - k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames") - } + # The root copy is shallow, so copy the video scope before removing + # per-video kwargs to avoid mutating `merged_mm_kwargs`. + non_video_mm_kwargs = dict(merged_mm_kwargs) + if "videos_kwargs" in non_video_mm_kwargs: + non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"]) + non_video_scoped_kwargs.pop("fps", None) + non_video_scoped_kwargs.pop("num_frames", None) + non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs processed_data = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**non_video_mm_kwargs), hf_data, diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py index db1cffefd6..46adcd7093 100644 --- a/vllm/multimodal/processing/context.py +++ b/vllm/multimodal/processing/context.py @@ -5,7 +5,7 @@ from collections.abc import Callable, Mapping from contextlib import contextmanager from dataclasses import dataclass, field from functools import cached_property -from typing import TYPE_CHECKING, Any, overload +from typing import TYPE_CHECKING, Any, get_type_hints, overload import torch from typing_extensions import TypeVar @@ -42,46 +42,129 @@ else: logger = init_logger(__name__) -# HuggingFace processors accept nested ``images_kwargs`` / ``videos_kwargs`` / -# ``audio_kwargs`` in addition to a shared flat namespace. vLLM's token-budget -# and dummy-input code only reads the flat keys, so scoped overrides have to -# be overlaid per modality. -_HF_MODALITY_PROCESSOR_KWARGS = { - "image": "images_kwargs", - "video": "videos_kwargs", - "audio": "audio_kwargs", -} +# HuggingFace processors accept a shared flat namespace in addition to the +# nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` / +# ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when +# resolving ``mm_processor_kwargs``. +_HF_PROCESSOR_KWARG_SCOPES = ( + "text_kwargs", + "images_kwargs", + "videos_kwargs", + "audio_kwargs", +) -def overlay_modality_mm_kwargs( - kwargs: Mapping[str, object], - modality: str | None, -) -> dict[str, Any]: - """Overlay HF-style nested processor kwargs onto the flat namespace. - - Overlay only the requested modality so a video ``size`` bump does not leak - into the image budget. Flat keys keep their current shared-namespace - behavior when no scoped dict is present. - - Args: - kwargs: Merged multi-modal processor kwargs. - modality: Target modality (``image``, ``video``, or ``audio``). - When ``None``, ``kwargs`` is copied without overlay. - - Returns: - A new dict with the modality-scoped keys overlaid when present. +def _merge_scoped_mm_processor_value( + flat_value: object, + scoped_value: object, +) -> object: + """Merge a flat value with an existing, more specific scoped value. + Mappings are merged recursively; otherwise the scoped value replaces the + flat value. """ - merged = dict(kwargs) - if modality is None: + if isinstance(flat_value, Mapping) and isinstance(scoped_value, Mapping): + # Merge recursively so scoped leaves override matching flat leaves + # without dropping unrelated flat entries. + merged = dict(flat_value) + for key, value in scoped_value.items(): + if key in merged: + merged[key] = _merge_scoped_mm_processor_value(merged[key], value) + else: + merged[key] = value return merged - scoped_key = _HF_MODALITY_PROCESSOR_KWARGS.get(modality) - if scoped_key is None: - return merged - scoped = merged.get(scoped_key) - if not isinstance(scoped, Mapping): - return merged - return merged | dict(scoped) + return scoped_value + + +def _resolve_mm_processor_kwargs( + kwargs: Mapping[str, object], + supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None, +) -> dict[str, object]: + """Resolve flat processor kwargs into HuggingFace processor kwarg scopes. + + With ``supported_mm_processor_kwargs``, each scope lists the flat keys it + supports. Matching flat keys are routed to every scope that supports them, + existing scoped values take precedence, and keys not supported by any scope + remain flat. + + Without ``supported_mm_processor_kwargs``, flat keys already represented in + at least one existing mapping-valued scope are removed from the shared flat + namespace; all other flat keys are left unchanged because there is no + information to determine which scopes should receive them. + """ + resolved = dict(kwargs) + + # Without ``supported_mm_processor_kwargs``, remove flat entries already + # represented in at least one existing mapping-valued scope. There is no + # information to route the remaining flat entries. + if supported_mm_processor_kwargs is None: + # Collect all keys represented in existing mapping-valued scopes. + # Non-mapping scope values do not contain scoped kwargs and are left + # untouched. + represented_scoped_keys: set[str] = set() + for scoped_key in _HF_PROCESSOR_KWARG_SCOPES: + scoped_kwargs = resolved.get(scoped_key) + if isinstance(scoped_kwargs, Mapping): + represented_scoped_keys.update(scoped_kwargs) + + # Remove flat entries for keys already represented in a scope. + for key in represented_scoped_keys: + if key not in _HF_PROCESSOR_KWARG_SCOPES: + resolved.pop(key, None) + return resolved + + # With ``supported_mm_processor_kwargs``, collect the flat kwargs before + # routing each one to the processor kwarg scopes that support it. + flat_kwargs = { + key: value + for key, value in resolved.items() + if key not in _HF_PROCESSOR_KWARG_SCOPES + } + + for key, flat_value in flat_kwargs.items(): + # Find every processor kwarg scope that supports this flat key. If none + # supports it, keep the key in the shared flat namespace. + supported_scopes = [ + scoped_key + for scoped_key in _HF_PROCESSOR_KWARG_SCOPES + if key in supported_mm_processor_kwargs.get(scoped_key, set()) + ] + if not supported_scopes: + continue + + # Add the flat value to each supported destination scope. + for scoped_key in supported_scopes: + # Start from a copy of the existing scoped kwargs, or an empty scope + # if absent. + if scoped_key in resolved: + scoped_kwargs = resolved[scoped_key] + # A scope must be a mapping before keys can be added to it. Do + # not replace an explicit non-mapping value. + if not isinstance(scoped_kwargs, Mapping): + raise TypeError(f"`{scoped_key}` must be a mapping") + scoped_kwargs = dict(scoped_kwargs) + else: + scoped_kwargs = {} + + # Add the flat value to this scope. Existing scoped values take + # precedence. + if key in scoped_kwargs: + scoped_kwargs[key] = _merge_scoped_mm_processor_value( + flat_value, scoped_kwargs[key] + ) + else: + # Assign the flat value to this scope. For mappings, assign a + # copy so the same mapping object is not shared between the flat + # value and multiple scopes. + scoped_kwargs[key] = ( + dict(flat_value) if isinstance(flat_value, Mapping) else flat_value + ) + resolved[scoped_key] = scoped_kwargs + + # After routing to every destination, remove the flat copy. + resolved.pop(key) + + return resolved @dataclass @@ -294,19 +377,27 @@ class InputProcessingContext: self, kwargs: Mapping[str, object], *, - modality: str | None = None, + supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None, ) -> dict[str, Any]: """Merge configured and request ``mm_processor_kwargs``. - When ``modality`` is set, HF-style nested - ``images_kwargs`` / ``videos_kwargs`` / ``audio_kwargs`` are overlaid - onto the flat namespace for vLLM-side reads (token budgets, dummy - inputs). Processor construction and HF ``__call__`` should omit - ``modality`` so the nested dicts still reach the HF processor. + When ``supported_mm_processor_kwargs`` is provided, flat keys are matched + against the keys supported by each HuggingFace processor kwarg scope. + Matching flat keys are routed to every scope that supports them and + removed from the shared flat namespace; keys not supported by any scope + remain flat. + + Without ``supported_mm_processor_kwargs``, flat keys already represented + in at least one existing mapping-valued scope are removed from the shared + flat namespace; all other flat keys are left unchanged because there is + no information to determine which scopes should receive them. """ mm_config = self.model_config.get_multimodal_config() merged = mm_config.merge_mm_processor_kwargs(kwargs) - return overlay_modality_mm_kwargs(merged, modality) + return _resolve_mm_processor_kwargs( + merged, + supported_mm_processor_kwargs=supported_mm_processor_kwargs, + ) def call_hf_processor( self, @@ -380,6 +471,49 @@ class BaseProcessingInfo: """ return self.ctx.get_hf_processor(**kwargs) + def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]: + """Return supported kwarg names for each HF processor kwargs scope.""" + processor = self.get_hf_processor() + processor_kwargs = get_type_hints(processor.valid_processor_kwargs) + + supported = { + "text_kwargs": set(processor_kwargs["text_kwargs"].__annotations__), + } + if "image" in self.supported_mm_limits: + supported["images_kwargs"] = set( + processor.image_processor.valid_kwargs.__annotations__ + ) + if "video" in self.supported_mm_limits: + supported["videos_kwargs"] = set( + processor.video_processor.valid_kwargs.__annotations__ + ) + if "audio" in self.supported_mm_limits: + supported["audio_kwargs"] = set( + processor_kwargs["audio_kwargs"].__annotations__ + ) + + return supported + + @cached_property + def supported_mm_processor_kwargs(self) -> dict[str, set[str]]: + """Supported kwarg names for each HF processor kwargs scope.""" + return self.get_supported_mm_processor_kwargs() + + def _merge_and_resolve_mm_processor_kwargs( + self, + mm_kwargs: Mapping[str, object], + ) -> dict[str, Any]: + """Merge configured and request ``mm_processor_kwargs``. + + Flat kwargs are routed into the HuggingFace processor kwarg scopes that + support them after the configured/request merge. When a routed flat value + conflicts with an existing scoped value, the scoped value takes precedence. + """ + return self.ctx.get_merged_mm_kwargs( + mm_kwargs, + supported_mm_processor_kwargs=self.supported_mm_processor_kwargs, + ) + def get_default_tok_params(self) -> TokenizeParams: """Construct the default parameters for tokenization.""" from vllm.renderers import TokenizeParams