[Bugfix][Multimodal] Fix flat/scoped mm_processor_kwargs merge and resolution (#56372)

Signed-off-by: Daniel M. García-Ocaña Hernández <danielgarciaocana@gmail.com>
This commit is contained in:
Daniel M. García-Ocaña Hernández
2026-09-29 03:58:39 +00:00
committed by GitHub
parent 491f44adfa
commit 295a0b8d64
11 changed files with 1606 additions and 192 deletions
+244
View File
@@ -188,6 +188,250 @@ def test_mm_processor_device_type_normalizes(device: object, expected: str | Non
assert config.get_mm_processor_device_type() == expected
def test_merge_mm_processor_kwargs_scoped_precedence():
configured = {
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
"do_resize": True,
"padding": False,
"images_kwargs": {
"size": {"longest_edge": 1024},
"do_resize": False,
},
"videos_kwargs": {
"size": {"longest_edge": 2048},
},
}
inference = {
"size": {"shortest_edge": 128},
"do_resize": True,
"images_kwargs": {
"size": {"shortest_edge": 256},
},
}
config = MultiModalConfig(
mm_processor_kwargs=configured,
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs(inference) == {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
"do_resize": True,
"padding": False,
"images_kwargs": {
"size": {
"shortest_edge": 256,
"longest_edge": 1024,
},
"do_resize": True,
},
"videos_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 2048,
},
},
}
def test_merge_mm_processor_kwargs_mapping_vs_replacement():
config = MultiModalConfig(
mm_processor_kwargs={
"size": {"shortest_edge": 64},
"images_kwargs": {"size": {"longest_edge": 1024}},
},
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs({"size": 7}) == {
"size": 7,
"images_kwargs": {"size": 7},
}
assert config.merge_mm_processor_kwargs({"size": {}}) == {
"size": {"shortest_edge": 64},
"images_kwargs": {
"size": {
"shortest_edge": 64,
"longest_edge": 1024,
},
},
}
_EMPTY_IMAGE_SCOPE_KWARGS: tuple[dict[str, object], ...] = (
{},
{"images_kwargs": None},
{"images_kwargs": {}},
)
@pytest.mark.parametrize("configured", _EMPTY_IMAGE_SCOPE_KWARGS)
@pytest.mark.parametrize("inference", _EMPTY_IMAGE_SCOPE_KWARGS)
def test_merge_mm_processor_kwargs_empty_scopes_are_absent(
configured: dict[str, object],
inference: dict[str, object],
):
config = MultiModalConfig(
mm_processor_kwargs=configured,
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs(inference) == {}
@pytest.mark.parametrize("empty_inference", _EMPTY_IMAGE_SCOPE_KWARGS)
def test_merge_mm_processor_kwargs_empty_inference_scope_preserves_config_merge(
empty_inference: dict[str, object],
):
config = MultiModalConfig(
mm_processor_kwargs={
"images_kwargs": {
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
},
},
mm_device_do_normalize=False,
)
inference = {
"size": {"shortest_edge": 128},
**empty_inference,
}
assert config.merge_mm_processor_kwargs(inference) == {
"size": {"shortest_edge": 128},
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
},
}
@pytest.mark.parametrize("empty_config", _EMPTY_IMAGE_SCOPE_KWARGS)
def test_merge_mm_processor_kwargs_empty_config_scope_allows_inference_merge(
empty_config: dict[str, object],
):
config = MultiModalConfig(
mm_processor_kwargs={
"size": {"longest_edge": 512},
**empty_config,
},
mm_device_do_normalize=False,
)
assert config.merge_mm_processor_kwargs(
{
"size": {"shortest_edge": 128},
"images_kwargs": {"size": {"longest_edge": 1024}},
}
) == {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 1024,
},
},
}
@pytest.mark.parametrize("non_mapping_scope", [False, 0, "", [], 123])
def test_merge_mm_processor_kwargs_falsey_non_mapping_scopes_are_not_empty(
non_mapping_scope: object,
):
config = MultiModalConfig(
mm_processor_kwargs={"images_kwargs": non_mapping_scope},
mm_device_do_normalize=False,
)
for empty_inference in _EMPTY_IMAGE_SCOPE_KWARGS:
assert config.merge_mm_processor_kwargs(empty_inference) == {
"images_kwargs": non_mapping_scope
}
@pytest.mark.parametrize(
("configured_scope", "inference_scope", "expected_scope"),
[
(123, {"size": {"shortest_edge": 128}}, {"size": {"shortest_edge": 128}}),
({"size": {"shortest_edge": 64}}, 123, 123),
(123, 456, 456),
],
)
def test_merge_mm_processor_kwargs_non_mapping_scope_precedence(
configured_scope: object,
inference_scope: object,
expected_scope: object,
):
config = MultiModalConfig(
mm_processor_kwargs={"images_kwargs": configured_scope},
mm_device_do_normalize=False,
)
merged = config.merge_mm_processor_kwargs({"images_kwargs": inference_scope})
assert merged["images_kwargs"] == expected_scope
def test_merge_mm_processor_kwargs_none_leaf_is_explicit():
config = MultiModalConfig(mm_device_do_normalize=False)
assert config.merge_mm_processor_kwargs({"images_kwargs": {"do_resize": None}}) == {
"images_kwargs": {"do_resize": None}
}
def test_merge_mm_processor_kwargs_normalize_and_ownership():
configured: dict[str, object] = {
"images_kwargs": {
"size": {"shortest_edge": 64},
},
}
inference: dict[str, object] = {
"do_normalize": True,
"do_rescale": True,
"images_kwargs": {
"size": {"longest_edge": 1024},
},
}
configured_before = copy.deepcopy(configured)
inference_before = copy.deepcopy(inference)
config = MultiModalConfig(
mm_processor_kwargs=configured,
mm_device_do_normalize=True,
)
merged = config.merge_mm_processor_kwargs(inference)
assert merged["do_normalize"] is False
assert merged["do_rescale"] is False
assert configured == configured_before
assert inference == inference_before
assert config.mm_processor_kwargs == configured_before
images_kwargs = merged["images_kwargs"]
assert isinstance(images_kwargs, dict)
size = images_kwargs["size"]
assert isinstance(size, dict)
size["shortest_edge"] = 999
assert configured == configured_before
assert inference == inference_before
assert config.mm_processor_kwargs == configured_before
def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None:
ec_config = (
None
@@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
@pytest.mark.parametrize("kwargs_on_init", [False, True])
@pytest.mark.parametrize(
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
)
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
def test_video_cache_is_independent_of_image_kwargs(
model_id: str,
kwargs_on_init: bool,
image_kwargs: dict[str, object],
video_uuid: str | None,
):
"""Image overrides must not change video frames or depend on cache warmth."""
kwargs = {"images_kwargs": image_kwargs}
"""Request image overrides must not change video frames or make results
depend on cache warmth.
"""
request_kwargs = {"images_kwargs": image_kwargs}
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 1},
@@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs(
cache=cache,
)
baseline = process({}, cache=None)
if kwargs_on_init:
ctx = build_model_context(
model_id,
mm_processor_kwargs=kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
mm_processor_cache_gb=1,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
request_kwargs = {} if kwargs_on_init else kwargs
process(request_kwargs, cache=cache)
# Populate the video cache without image overrides, then change only
# images_kwargs on the next request. Since images_kwargs are excluded from
# the video hash, and must not affect the processed video, the cached result
# must match a fresh recomputation.
baseline = process({}, cache=cache)
cached = process(request_kwargs, cache=cache)
cache.clear_cache()
fresh = process(request_kwargs, cache=cache)
@@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image(
[
({}, (448, 448), 256),
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
(
{"images_kwargs": {"size": {"longest_edge": 1008}}},
(1540, 1540),
1296,
),
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
({"size": {"longest_edge": 1008}}, (29, 29), 4),
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
@@ -148,6 +153,40 @@ def test_processor_size_override(
assert prompt_update_tokens == hf_placeholder_tokens
def test_scoped_request_size_overrides_configured_flat_size():
ctx = build_model_context(
_MODEL_ID,
mm_processor_kwargs={"size": {"longest_edge": 1008}},
limit_mm_per_prompt={"image": 1},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = AutoProcessor.from_pretrained(
_MODEL_ID,
fix_mistral_regex=True,
)
hf_processor_mm_kwargs: dict[str, object] = {
"images_kwargs": {"size": {"longest_edge": 1288}}
}
image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
hf_processor, [image], merged_mm_kwargs
)
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
processor, [image], pixel_values, hf_processor_mm_kwargs
)
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
hf_processor, pixel_values[0]
)
assert expected_from_pixel_values == 1656
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
assert prompt_update_tokens == hf_placeholder_tokens
def test_lightonocr_keeps_vision_config_image_size():
ctx = build_model_context(
_LIGHTON_MODEL_ID,
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
@@ -8,16 +10,238 @@ import pytest
import torch
from PIL import Image
from vllm.config.multimodal import MultiModalConfig
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import batched_tensors_equal
from vllm.multimodal.processing.context import InputProcessingContext
from vllm.platforms import current_platform
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@pytest.mark.parametrize(
("mm_kwargs", "expected"),
[
(
{"size": {"shortest_edge": 64, "longest_edge": 1024}},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 64,
"max_pixels": 1024,
},
),
(
{"min_pixels": 64, "max_pixels": 1024},
{
"min_pixels": 64,
"max_pixels": 1024,
"size": {"shortest_edge": 64, "longest_edge": 1024},
},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
{
"size": {"shortest_edge": 64, "longest_edge": 1024},
"min_pixels": 128,
"max_pixels": 2048,
},
),
(
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {"max_pixels": 4096},
"videos_kwargs": {"size": {"longest_edge": 8192}},
},
{
"min_pixels": None,
"size": {"shortest_edge": 0},
"images_kwargs": {
"max_pixels": 4096,
"size": {"longest_edge": 4096},
},
"videos_kwargs": {
"size": {"longest_edge": 8192},
"max_pixels": 8192,
},
},
),
(
{"min_pixels": 0},
{"min_pixels": 0, "size": {"shortest_edge": 0}},
),
(
{"size": None, "min_pixels": 64},
{"size": {"shortest_edge": 64}, "min_pixels": 64},
),
],
)
def test_complete_mm_processor_size_aliases(
mm_kwargs: dict[str, object],
expected: dict[str, object],
) -> None:
original = deepcopy(mm_kwargs)
actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
assert actual == expected
assert mm_kwargs == original
def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
mm_config = MultiModalConfig(
mm_processor_kwargs={
"images_kwargs": {"min_pixels": 64},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
info = Qwen2VLProcessingInfo(ctx)
monkeypatch.setattr(
info,
"get_supported_mm_processor_kwargs",
lambda: {"images_kwargs": {"size", "min_pixels"}},
)
assert info._merge_and_resolve_mm_processor_kwargs(
{"size": {"shortest_edge": 128}}
) == {
"images_kwargs": {
"size": {"shortest_edge": 128},
"min_pixels": 128,
},
}
@pytest.mark.parametrize(
("mm_kwargs", "default_size", "expected"),
[
(
{},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 32, "longest_edge": 1024},
),
(
{"size": {"shortest_edge": 64}},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 1024},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": 128,
"max_pixels": 4096,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 128, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64},
"max_pixels": 4096,
},
None,
{"shortest_edge": 64, "longest_edge": 4096},
),
(
{
"size": {"shortest_edge": 64, "longest_edge": 2048},
"min_pixels": None,
"max_pixels": 0,
},
{"shortest_edge": 32, "longest_edge": 1024},
{"shortest_edge": 64, "longest_edge": 0},
),
],
)
def test_get_vision_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None,
expected: dict[str, object],
) -> None:
original_mm_kwargs = {
key: dict(value) if isinstance(value, dict) else value
for key, value in mm_kwargs.items()
}
original_default_size = dict(default_size) if default_size is not None else None
actual = Qwen2VLProcessingInfo._get_vision_size(
mm_kwargs,
default_size=default_size,
)
assert actual == expected
assert mm_kwargs == original_mm_kwargs
assert default_size == original_default_size
@pytest.mark.parametrize(
("modality", "scope"),
[
("image", "images_kwargs"),
("video", "videos_kwargs"),
],
)
def test_get_vision_info_reads_merged_modality_scope(
monkeypatch: pytest.MonkeyPatch,
modality: str,
scope: str,
) -> None:
vision_config = SimpleNamespace(
patch_size=14,
spatial_merge_size=2,
temporal_patch_size=2,
)
ctx = SimpleNamespace(
get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
)
info = Qwen2VLProcessingInfo(ctx)
merged = {
"images_kwargs": {"size": {"shortest_edge": 111}},
"videos_kwargs": {"size": {"shortest_edge": 222}},
}
monkeypatch.setattr(
info,
"_merge_and_resolve_mm_processor_kwargs",
lambda _: merged,
)
seen: list[dict[str, object]] = []
def get_size(
mm_kwargs: dict[str, object],
default_size: dict[str, object] | None = None,
) -> dict[str, object]:
seen.append(mm_kwargs)
return dict(default_size or {})
monkeypatch.setattr(info, "_get_vision_size", get_size)
image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
info._get_vision_info(
image_width=28,
image_height=28,
num_frames=2,
do_resize=False,
image_processor=image_processor,
mm_kwargs={},
modality=modality, # type: ignore[arg-type]
)
assert seen == [merged[scope]]
def test_jina_vl_processing_order() -> None:
"""Jina's document-first prompt keeps cached features and hashes aligned."""
ctx = build_model_context(
@@ -1,19 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for Qwen3-VL processor.
Covers the fix for num_frames-based timestamp calculation
(issue vllm-project/vllm#35909).
"""
"""Regression tests for Qwen3-VL multimodal processing."""
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import numpy as np
import pytest
import torch
from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
from vllm.config import ModelConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.processing.context import BaseProcessingInfo
from ...registry import HF_EXAMPLE_MODELS
from ...utils import build_model_context
@@ -21,6 +23,32 @@ from ...utils import build_model_context
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
assert "size" in native_video_kwargs
assert "min_pixels" not in native_video_kwargs
assert "max_pixels" not in native_video_kwargs
info = Qwen3VLProcessingInfo(SimpleNamespace())
parent_supported = {
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels", "max_pixels"},
"videos_kwargs": set(native_video_kwargs),
}
monkeypatch.setattr(
BaseProcessingInfo,
"get_supported_mm_processor_kwargs",
lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
)
supported = info.supported_mm_processor_kwargs
assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
assert parent_supported["videos_kwargs"] == native_video_kwargs
def _build_video_mm_data(
num_frames: int,
width: int = 128,
@@ -45,6 +73,122 @@ def _build_video_mm_data(
return {"video": [(video, metadata)]}
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_image_size(model_id: str) -> None:
"""Partial image ``size`` must be completed and applied before the HF call.
Keep video compatibility and per-video kwargs in an image-only call as well:
Qwen3 must sanitize the video scope before HF validates it. Regression for
#56363.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 0},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = np.zeros((128, 128, 3), dtype=np.uint8)
min_pixels = 256 * 32 * 32
default_size = dict(processor.info.get_image_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [image]})
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"image_grid_thw"
]
stock_grid = process({})
complete_grid = process({"images_kwargs": {"size": complete_size}})
partial_grid = process(
{
"images_kwargs": {
"size": {"shortest_edge": min_pixels},
},
"videos_kwargs": {
"min_pixels": 4096,
"max_pixels": 469_762_048,
"fps": [2.0],
"num_frames": [8],
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_partial_scoped_video_size(model_id: str) -> None:
"""Partial scoped video ``size`` must be completed and applied before HF."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
min_pixels = 512 * 32 * 32
default_size = dict(processor.info.get_video_processor().size)
complete_size = default_size | {"shortest_edge": min_pixels}
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
complete_grid = process({"videos_kwargs": {"size": complete_size}})
partial_grid = process(
{
"videos_kwargs": {
"size": {"shortest_edge": min_pixels},
},
}
)
assert torch.equal(partial_grid, complete_grid)
assert not torch.equal(partial_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
"""Flat min/max pixel aliases keep their historical Qwen3 video effect."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
pixel_budget = 512 * 32 * 32
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
"video_grid_thw"
]
stock_grid = process({})
scoped_grid = process(
{
"videos_kwargs": {
"size": {
"shortest_edge": pixel_budget,
"longest_edge": pixel_budget,
},
},
}
)
flat_grid = process(
{
"min_pixels": pixel_budget,
"max_pixels": pixel_budget,
}
)
assert torch.equal(flat_grid, scoped_grid)
assert not torch.equal(flat_grid, stock_grid)
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"num_frames",
@@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize(
"hf_mm_kwargs",
[{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
("hf_mm_kwargs", "scalar_kwargs"),
[
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
],
)
def test_processor_multi_video_list_kwargs(
model_id: str,
hf_mm_kwargs: dict[str, Any],
scalar_kwargs: list[dict[str, Any]],
) -> None:
"""Regression test: a multi-video request with list-valued per-video
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
crash.
"""List-valued video kwargs must be scalarized per item without mutation.
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
video without slicing, so ``_get_video_second_idx`` received the list
where a scalar was expected and raised ``TypeError``.
Each video in the multi-video request must match processing the same input
independently with its corresponding scalar ``fps``/``num_frames`` value.
"""
ctx = build_model_context(
model_id,
@@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs(
)
mm_data = {
"video": [
_build_video_mm_data(num_frames=16)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
]
}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
single_mm_data = _build_video_mm_data(num_frames=32)
single = processor(
single_prompt,
mm_items=processor.info.parse_mm_data(single_mm_data),
hf_processor_mm_kwargs=hf_kwargs,
)
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
assert not torch.equal(expected_grids[0], expected_grids[1])
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
assert hf_mm_kwargs == hf_mm_kwargs_before
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 2, (
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
@@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
capped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
mm_processor_kwargs={
"videos_kwargs": {
"size": size,
"cap_pixels_per_frame": True,
},
},
limit_mm_per_prompt={"image": 0, "video": 1},
)
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
@@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
uncapped_ctx = build_model_context(
model_id,
mm_processor_kwargs={"size": size},
mm_processor_kwargs={"videos_kwargs": {"size": size}},
limit_mm_per_prompt={"image": 0, "video": 1},
)
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
+295 -43
View File
@@ -3,18 +3,23 @@
import time
from contextlib import nullcontext
from copy import deepcopy
from types import SimpleNamespace
from typing import TypedDict
import numpy as np
import pytest
from vllm.config import ModelConfig, SchedulerConfig
from vllm.config.multimodal import MultiModalConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.parse import MultiModalDataParser
from vllm.multimodal.processing.context import (
BaseProcessingInfo,
InputProcessingContext,
overlay_modality_mm_kwargs,
_resolve_mm_processor_kwargs,
)
from vllm.multimodal.processing.inputs import ProcessorInputs
from vllm.multimodal.processing.processor import (
@@ -1207,58 +1212,305 @@ def test_apply_prompt_updates_falls_back_with_index_targets():
assert [p.tokens for p in placeholders["image"]] == [[200, 201], [9]]
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_scoped_video_does_not_leak_to_image():
"""HF-style videos_kwargs must overlay only when modality is video."""
video_size = {"longest_edge": 469762048, "shortest_edge": 4096}
kwargs = {"videos_kwargs": {"size": video_size}}
assert overlay_modality_mm_kwargs(kwargs, None) == kwargs
assert "size" not in overlay_modality_mm_kwargs(kwargs, "image")
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == video_size
class _TextProcessorKwargs(TypedDict, total=False):
padding: bool
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_flat_size_stays_shared():
"""A flat size override keeps the current shared-namespace behavior."""
size = {"longest_edge": 469762048, "shortest_edge": 4096}
kwargs = {"size": size}
for modality in (None, "image", "video"):
assert overlay_modality_mm_kwargs(kwargs, modality)["size"] == size
class _AudioProcessorKwargs(TypedDict, total=False):
sampling_rate: int
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_scoped_wins_over_flat_for_modality():
"""A nested videos_kwargs size wins over a flat size for video reads."""
class _ProcessorKwargs(TypedDict, total=False):
text_kwargs: _TextProcessorKwargs
audio_kwargs: _AudioProcessorKwargs
class _ImageProcessorKwargs(TypedDict, total=False):
size: dict[str, int]
min_pixels: int
class _VideoProcessorKwargs(TypedDict, total=False):
size: dict[str, int]
fps: float
@pytest.mark.parametrize(
("supported_mm_limits", "processor", "expected"),
[
(
{"image": None, "video": None},
SimpleNamespace(
valid_processor_kwargs=_ProcessorKwargs,
image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs),
video_processor=SimpleNamespace(valid_kwargs=_VideoProcessorKwargs),
),
{
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels"},
"videos_kwargs": {"size", "fps"},
},
),
(
{"image": None},
SimpleNamespace(
valid_processor_kwargs=_ProcessorKwargs,
image_processor=SimpleNamespace(valid_kwargs=_ImageProcessorKwargs),
),
{
"text_kwargs": {"padding"},
"images_kwargs": {"size", "min_pixels"},
},
),
(
{"audio": None},
SimpleNamespace(valid_processor_kwargs=_ProcessorKwargs),
{
"text_kwargs": {"padding"},
"audio_kwargs": {"sampling_rate"},
},
),
],
)
def test_get_supported_mm_processor_kwargs_uses_supported_modalities(
monkeypatch: pytest.MonkeyPatch,
supported_mm_limits: dict[str, int | None],
processor: SimpleNamespace,
expected: dict[str, set[str]],
) -> None:
info = BaseProcessingInfo(SimpleNamespace())
info.__dict__["supported_mm_limits"] = supported_mm_limits
monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor)
assert info.get_supported_mm_processor_kwargs() == expected
def test_supported_mm_processor_kwargs_is_cached(
monkeypatch: pytest.MonkeyPatch,
) -> None:
info = BaseProcessingInfo(SimpleNamespace())
expected = {"images_kwargs": {"size"}}
calls = 0
def get_supported_mm_processor_kwargs() -> dict[str, set[str]]:
nonlocal calls
calls += 1
return expected
monkeypatch.setattr(
info,
"get_supported_mm_processor_kwargs",
get_supported_mm_processor_kwargs,
)
first = info.supported_mm_processor_kwargs
second = info.supported_mm_processor_kwargs
assert first is expected
assert second is first
assert calls == 1
def test_resolve_mm_processor_kwargs_routes_supported_flat_kwargs():
kwargs = {
"size": {"longest_edge": 1},
"videos_kwargs": {"size": {"longest_edge": 2}},
"images_kwargs": {"size": {"longest_edge": 3}},
"size": {"shortest_edge": 64},
"padding": True,
"sampling_rate": 16000,
"unknown": "keep",
"images_kwargs": {
"size": {"longest_edge": 1024},
"custom_image_kwarg": "keep",
},
}
kwargs_before = deepcopy(kwargs)
resolved = _resolve_mm_processor_kwargs(
kwargs,
supported_mm_processor_kwargs={
"text_kwargs": {"padding"},
"images_kwargs": {"size"},
"videos_kwargs": {"size"},
"audio_kwargs": {"sampling_rate"},
},
)
assert resolved == {
"unknown": "keep",
"text_kwargs": {"padding": True},
"images_kwargs": {
"size": {
"shortest_edge": 64,
"longest_edge": 1024,
},
"custom_image_kwarg": "keep",
},
"videos_kwargs": {
"size": {"shortest_edge": 64},
},
"audio_kwargs": {"sampling_rate": 16000},
}
assert kwargs == kwargs_before
video_kwargs = resolved["videos_kwargs"]
image_kwargs = resolved["images_kwargs"]
assert isinstance(video_kwargs, dict)
assert isinstance(image_kwargs, dict)
video_size = video_kwargs["size"]
image_size = image_kwargs["size"]
assert isinstance(video_size, dict)
assert isinstance(image_size, dict)
video_size["shortest_edge"] = 999
assert image_size["shortest_edge"] == 64
assert kwargs == kwargs_before
def test_resolve_mm_processor_kwargs_rejects_non_mapping_destination_scope():
with pytest.raises(TypeError, match="images_kwargs"):
_resolve_mm_processor_kwargs(
{
"size": {"shortest_edge": 64},
"images_kwargs": 123,
},
supported_mm_processor_kwargs={
"images_kwargs": {"size"},
},
)
def test_resolve_mm_processor_kwargs_without_schema_deduplicates_existing_scopes():
kwargs = {
"size": {"shortest_edge": 64},
"num_frames": 16,
"fps": 4,
"images_kwargs": {
"size": {"longest_edge": 1024},
},
"videos_kwargs": {
"num_frames": 8,
},
"audio_kwargs": 123,
"common_kwargs": {
"fps": 2,
},
}
kwargs_before = deepcopy(kwargs)
resolved = _resolve_mm_processor_kwargs(kwargs)
assert resolved == {
"fps": 4,
"images_kwargs": {
"size": {"longest_edge": 1024},
},
"videos_kwargs": {
"num_frames": 8,
},
"audio_kwargs": 123,
"common_kwargs": {
"fps": 2,
},
}
assert kwargs == kwargs_before
assert "text_kwargs" not in resolved
@pytest.mark.parametrize(
"inference_kwargs",
[
{"size": {"shortest_edge": 128}},
{"size": {"shortest_edge": 128}, "images_kwargs": None},
{"size": {"shortest_edge": 128}, "images_kwargs": {}},
],
)
def test_get_merged_mm_kwargs_treats_empty_scopes_as_absent_before_routing(
inference_kwargs: dict[str, object],
):
mm_config = MultiModalConfig(
mm_processor_kwargs={
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
"images_kwargs": {
"size": {"longest_edge": 1024},
},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
merged = ctx.get_merged_mm_kwargs(
inference_kwargs,
supported_mm_processor_kwargs={
"images_kwargs": {"size"},
"videos_kwargs": {"size"},
},
)
assert merged == {
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 1024,
},
},
"videos_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 512,
},
},
}
assert overlay_modality_mm_kwargs(kwargs, "video")["size"] == {"longest_edge": 2}
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 3}
assert overlay_modality_mm_kwargs(kwargs, None)["size"] == {"longest_edge": 1}
def test_get_merged_mm_kwargs_merges_before_routing():
mm_config = MultiModalConfig(
mm_processor_kwargs={
"size": {
"shortest_edge": 64,
"longest_edge": 512,
},
"images_kwargs": {
"size": {"longest_edge": 1024},
},
},
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
@pytest.mark.skip_global_cleanup
def test_overlay_modality_mm_kwargs_ignores_non_mapping_scoped_value():
kwargs = {"images_kwargs": "not-a-dict", "size": {"longest_edge": 1}}
assert overlay_modality_mm_kwargs(kwargs, "image")["size"] == {"longest_edge": 1}
merged = ctx.get_merged_mm_kwargs(
{
"size": {
"shortest_edge": 128,
"longest_edge": 768,
},
"images_kwargs": {
"size": {"longest_edge": 2048},
},
},
supported_mm_processor_kwargs={
"images_kwargs": {"size"},
"videos_kwargs": {"size"},
},
)
@pytest.mark.skip_global_cleanup
def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
"""Configured videos_kwargs overlay only for video reads after merge."""
from vllm.config.multimodal import MultiModalConfig
size = {"longest_edge": 469762048, "shortest_edge": 4096}
mm_config = MultiModalConfig(mm_processor_kwargs={"videos_kwargs": {"size": size}})
merged = mm_config.merge_mm_processor_kwargs({})
assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
assert "size" not in overlay_modality_mm_kwargs(merged, "image")
assert "size" not in overlay_modality_mm_kwargs(merged, None)
assert merged == {
"images_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 2048,
},
},
"videos_kwargs": {
"size": {
"shortest_edge": 128,
"longest_edge": 768,
},
},
}
def test_processor_inputs_hashes_partial_uuids():
+187 -1
View File
@@ -27,6 +27,186 @@ logger = init_logger(__name__)
_T = TypeVar("_T")
# HuggingFace processors accept a shared flat namespace in addition to the
# nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` /
# ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when
# resolving ``mm_processor_kwargs``.
_HF_PROCESSOR_KWARG_SCOPES = (
"text_kwargs",
"images_kwargs",
"videos_kwargs",
"audio_kwargs",
)
def _recursively_merge_mm_processor_kwargs(
defaults: Mapping[str, object],
overrides: Mapping[str, object],
) -> dict[str, object]:
"""Merge two mappings recursively, with ``overrides`` taking precedence.
Nested values are merged only when both values are mappings; otherwise the
override replaces the existing value.
"""
merged = dict(defaults)
for key, value in overrides.items():
default_value = merged.get(key)
if isinstance(default_value, Mapping) and isinstance(value, Mapping):
merged[key] = _recursively_merge_mm_processor_kwargs(default_value, value)
else:
merged[key] = value
return merged
def _merge_mm_processor_scope(
config_flat: Mapping[str, object],
config_scoped: Mapping[str, object],
inference_flat: Mapping[str, object],
inference_scoped: Mapping[str, object],
) -> dict[str, object]:
"""Merge kwargs for one existing HuggingFace processor kwarg scope.
Flat kwargs contribute only to keys present in either the configured or
inference-time scope. For those keys, precedence is:
configured flat < configured scoped < inference flat < inference scoped.
Nested mappings are merged recursively; otherwise the higher-priority value
replaces the lower-priority value.
"""
# Flat kwargs contribute only to keys present in this scope in either input.
scoped_keys = config_scoped.keys() | inference_scoped.keys()
config_flat_for_scope = {
key: value for key, value in config_flat.items() if key in scoped_keys
}
inference_flat_for_scope = {
key: value for key, value in inference_flat.items() if key in scoped_keys
}
# Merge from lowest to highest priority:
# configured flat -> configured scoped -> inference flat -> inference scoped.
merged = _recursively_merge_mm_processor_kwargs(
config_flat_for_scope, config_scoped
)
merged = _recursively_merge_mm_processor_kwargs(merged, inference_flat_for_scope)
merged = _recursively_merge_mm_processor_kwargs(merged, inference_scoped)
return merged
def _merge_mm_processor_kwargs(
config_kwargs: Mapping[str, object],
inference_kwargs: Mapping[str, object],
) -> dict[str, object]:
"""Merge configured and inference-time multi-modal processor kwargs.
The merge is performed in two parts:
1. The shared flat kwargs are merged recursively, with inference-time values
taking precedence.
2. Each existing HuggingFace processor kwarg scope is merged separately.
For keys present in either the configured or inference-time scope,
precedence is:
configured flat < configured scoped < inference flat < inference scoped.
For example:
configured = {
"size": {"shortest_edge": 256},
"videos_kwargs": {
"size": {"longest_edge": 1024},
"num_frames": 16,
},
}
inference = {
"size": {"shortest_edge": 512},
"videos_kwargs": {"fps": 4},
}
produces:
{
"size": {"shortest_edge": 512},
"videos_kwargs": {
"size": {
"shortest_edge": 512,
"longest_edge": 1024,
},
"num_frames": 16,
"fps": 4,
},
}
Scoped values do not modify the shared flat kwargs or sibling scopes.
Missing scopes, None-valued scopes, and empty scope mappings are treated as
absent and do not create a scope.
"""
# Merge the shared flat kwargs first; nested scopes are handled separately
# below.
config_flat = {
key: value
for key, value in config_kwargs.items()
if key not in _HF_PROCESSOR_KWARG_SCOPES
}
inference_flat = {
key: value
for key, value in inference_kwargs.items()
if key not in _HF_PROCESSOR_KWARG_SCOPES
}
merged = _recursively_merge_mm_processor_kwargs(config_flat, inference_flat)
# Merge each nested processor kwarg scope present in either input.
for scoped_key in _HF_PROCESSOR_KWARG_SCOPES:
config_scope_value = config_kwargs.get(scoped_key)
inference_scope_value = inference_kwargs.get(scoped_key)
# Treat missing scopes, None-valued scopes, and empty scope mappings as
# absent.
if isinstance(config_scope_value, Mapping) and not config_scope_value:
config_scope_value = None
if isinstance(inference_scope_value, Mapping) and not inference_scope_value:
inference_scope_value = None
# Skip adding the scope to the merged result when it is absent from both
# sources.
if config_scope_value is None and inference_scope_value is None:
continue
# Handle the inference scope first because it has higher precedence; only
# mapping-valued scopes participate in the recursive scoped merge.
inference_scope_kwargs: Mapping[str, object]
if inference_scope_value is None:
inference_scope_kwargs = {}
elif not isinstance(inference_scope_value, Mapping):
# Non-mapping scopes cannot participate in the recursive scoped merge;
# the higher-priority inference value replaces the configured scope.
merged[scoped_key] = inference_scope_value
continue
else:
inference_scope_kwargs = inference_scope_value
# Handle the configured scope next because it has lower precedence; only
# mapping-valued scopes participate in the recursive scoped merge.
if isinstance(config_scope_value, Mapping):
config_scope_kwargs = config_scope_value
elif inference_scope_value is None:
# Non-mapping scopes cannot participate in the recursive scoped merge;
# preserve the configured value when inference contributes nothing.
merged[scoped_key] = config_scope_value
continue
else:
config_scope_kwargs = {}
# Merge this scope together with the matching flat kwargs.
merged[scoped_key] = _merge_mm_processor_scope(
config_flat,
config_scope_kwargs,
inference_flat,
inference_scope_kwargs,
)
return merged
@dataclass
class BaseDummyOptions:
@@ -533,8 +713,14 @@ class MultiModalConfig:
) -> dict[str, object]:
"""Get the keyword arguments to pass to the multi-modal processor
according to the extra arguments passed during inference.
Nested mappings are merged recursively, with inference-time values taking
precedence over configured values.
"""
merged = dict(self.mm_processor_kwargs or {}) | dict(inference_kwargs)
merged = _merge_mm_processor_kwargs(
self.mm_processor_kwargs or {},
inference_kwargs,
)
if self.mm_device_do_normalize:
# The model expects raw pixels and normalises on device;
# overriding these per request would silently double-normalise.
+9 -3
View File
@@ -3,7 +3,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal
from typing import Annotated, Any, Literal, cast
import torch
import torch.nn as nn
@@ -212,8 +212,14 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
) -> PixtralHFEncoderInfo:
processor = self.get_hf_processor()
size = processor.image_processor.size
merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})
if override_size := merged_kwargs.get("size"):
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(
mm_processor_kwargs or {}
)
image_mm_kwargs = cast(
Mapping[str, Any],
merged_mm_kwargs.get("images_kwargs", {}),
)
if override_size := image_mm_kwargs.get("size"):
size = size | override_size
image_size = size["longest_edge"]
+130 -18
View File
@@ -28,7 +28,7 @@
import math
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
from functools import partial
from typing import Annotated, Any, Literal, TypeAlias
from typing import Annotated, Any, Literal, TypeAlias, cast
import numpy as np
import torch
@@ -865,6 +865,118 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None}
@staticmethod
def _complete_mm_processor_size_aliases(
mm_kwargs: Mapping[str, object],
) -> dict[str, object]:
"""Complete missing Qwen-VL size aliases.
Qwen-VL accepts ``size.shortest_edge`` / ``min_pixels`` and
``size.longest_edge`` / ``max_pixels`` as equivalent controls. When
only one representation is provided with a non-``None`` value, add the
missing equivalent representation. If both are already present, leave
them unchanged even when their values conflict.
Apply this independently to the flat kwargs and to each existing
``images_kwargs`` / ``videos_kwargs`` mapping. Values are not propagated
between these mappings.
"""
def add_missing_size_aliases(
kwargs: Mapping[str, object],
) -> dict[str, object]:
"""Add missing size aliases within a kwargs mapping."""
out_kwargs = dict(kwargs)
# Copy `size` separately because `out_kwargs` is a shallow copy and
# size edges may be added below.
raw_size = cast(
Mapping[str, object] | None,
out_kwargs.get("size"),
)
size_kwargs = {} if raw_size is None else dict(raw_size)
for edge_key, pixel_key in (
("shortest_edge", "min_pixels"),
("longest_edge", "max_pixels"),
):
# Add the pixel alias from `size` when only the size edge is
# present.
if pixel_key not in out_kwargs:
if edge_key in size_kwargs and size_kwargs[edge_key] is not None:
out_kwargs[pixel_key] = size_kwargs[edge_key]
continue
pixel_value = out_kwargs[pixel_key]
# HF treats `min_pixels` / `max_pixels` set to None as no size
# override, so do not propagate them to the equivalent size edge.
if pixel_value is None:
continue
# Add the size edge when only the pixel alias is present.
if edge_key not in size_kwargs:
size_kwargs[edge_key] = pixel_value
out_kwargs["size"] = size_kwargs
return out_kwargs
prepared_mm_kwargs = add_missing_size_aliases(mm_kwargs)
# Complete aliases independently inside existing image/video kwargs;
# flat values are not copied into the nested mappings.
for nested_kwargs_key in ("images_kwargs", "videos_kwargs"):
nested_kwargs = prepared_mm_kwargs.get(nested_kwargs_key)
if isinstance(nested_kwargs, Mapping):
prepared_mm_kwargs[nested_kwargs_key] = add_missing_size_aliases(
nested_kwargs
)
return prepared_mm_kwargs
def _merge_and_resolve_mm_processor_kwargs(
self,
mm_kwargs: Mapping[str, object],
) -> dict[str, Any]:
"""Merge configured and request Qwen-VL ``mm_processor_kwargs``.
Request-side ``size.shortest_edge`` / ``min_pixels`` and
``size.longest_edge`` / ``max_pixels`` aliases are completed before the
merge so request precedence is preserved across equivalent
representations.
"""
# Complete request-side Qwen-VL size aliases before merging with
# configured kwargs.
prepared_mm_kwargs = self._complete_mm_processor_size_aliases(mm_kwargs)
return super()._merge_and_resolve_mm_processor_kwargs(prepared_mm_kwargs)
@staticmethod
def _get_vision_size(
mm_kwargs: Mapping[str, object],
default_size: Mapping[str, int] | None = None,
) -> dict[str, int]:
"""Resolve Qwen-VL vision size kwargs with optional processor defaults.
``size`` overrides any processor defaults, then ``min_pixels`` and
``max_pixels`` override the corresponding size edges.
"""
size = dict(default_size) if default_size is not None else {}
override_size = cast(
Mapping[str, int] | None,
mm_kwargs.get("size"),
)
if override_size is not None:
size.update(override_size)
min_pixels = cast(int | None, mm_kwargs.get("min_pixels"))
if min_pixels is not None:
size["shortest_edge"] = min_pixels
max_pixels = cast(int | None, mm_kwargs.get("max_pixels"))
if max_pixels is not None:
size["longest_edge"] = max_pixels
return size
def get_mm_max_tokens_per_item(
self,
seq_len: int,
@@ -883,7 +995,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
do_resize: bool = True,
image_processor: Qwen2VLImageProcessor,
mm_kwargs: Mapping[str, object],
modality: str | None = None,
modality: Literal["image", "video"] | None = None,
) -> tuple[ImageSize, int]:
hf_config = self.get_hf_config()
vision_config = hf_config.vision_config
@@ -891,14 +1003,16 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
merge_size = vision_config.spatial_merge_size
temporal_patch_size = vision_config.temporal_patch_size
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
size = size | {"shortest_edge": override_min_pixels}
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
size = size | {"longest_edge": override_max_pixels}
if modality is None:
modality = "image"
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
scope = "images_kwargs" if modality == "image" else "videos_kwargs"
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
size = self._get_vision_size(
scoped_mm_kwargs,
default_size=image_processor.size,
)
if do_resize:
resized_height, resized_width = smart_resize(
@@ -986,14 +1100,12 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
if max_pixels is None:
image_processor = self.get_image_processor()
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image")
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
size = size | {"shortest_edge": override_min_pixels}
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
size = size | {"longest_edge": override_max_pixels}
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
image_mm_kwargs = merged_mm_kwargs.get("images_kwargs", {})
size = self._get_vision_size(
image_mm_kwargs,
default_size=image_processor.size,
)
max_pixels = size["longest_edge"]
+107 -52
View File
@@ -27,7 +27,7 @@
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
from functools import lru_cache, partial
from itertools import islice
from typing import Any
from typing import Any, Literal
import numpy as np
import torch
@@ -944,6 +944,16 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
supported = super().get_supported_mm_processor_kwargs()
# vLLM has historically accepted `min_pixels` / `max_pixels` for
# Qwen3-VL video. Include them in the routing schema even though
# Qwen3VLVideoProcessor only accepts the equivalent `size`; they are
# folded into `size` before calling HF.
supported["videos_kwargs"].update({"min_pixels", "max_pixels"})
return supported
def _get_vision_info(
self,
*,
@@ -953,7 +963,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
do_resize: bool = True,
image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
mm_kwargs: Mapping[str, object],
modality: str | None = None,
modality: Literal["image", "video"] | None = None,
) -> tuple[ImageSize, int]:
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
@@ -965,14 +975,14 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
if modality is None:
modality = "video" if is_video else "image"
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
size = size | {"shortest_edge": override_min_pixels}
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
size = size | {"longest_edge": override_max_pixels}
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
scope = "videos_kwargs" if modality == "video" else "images_kwargs"
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
size = self._get_vision_size(
scoped_mm_kwargs,
default_size=image_processor.size,
)
if do_resize:
if is_video:
@@ -1030,9 +1040,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
) -> int:
video_processor = self.get_video_processor()
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video")
video_size = mm_kwargs.get("size", video_processor.size)
temporal_patch_size = mm_kwargs.get(
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
video_size = self._get_vision_size(
video_mm_kwargs,
default_size=video_processor.size,
)
temporal_patch_size = video_mm_kwargs.get(
"temporal_patch_size", video_processor.temporal_patch_size
)
@@ -1162,9 +1176,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
video_processor = self.info.get_video_processor()
mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video")
video_size = mm_kwargs.get("size", video_processor.size)
temporal_patch_size = mm_kwargs.get(
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs({})
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
video_size = self.info._get_vision_size(
video_mm_kwargs,
default_size=video_processor.size,
)
temporal_patch_size = video_mm_kwargs.get(
"temporal_patch_size", video_processor.temporal_patch_size
)
@@ -1180,13 +1198,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
# over enough frames that the cap is not binding for the dummy.
# This takes precedence over a shrinking num_frames override, which
# would otherwise reintroduce the under-profiling.
if mm_kwargs.get("cap_pixels_per_frame"):
max_video_tokens = mm_kwargs.get(
if video_mm_kwargs.get("cap_pixels_per_frame"):
max_video_tokens = video_mm_kwargs.get(
"max_video_tokens",
getattr(video_processor, "max_video_tokens", 768),
)
patch_size = mm_kwargs.get("patch_size", video_processor.patch_size)
merge_size = mm_kwargs.get("merge_size", video_processor.merge_size)
patch_size = video_mm_kwargs.get("patch_size", video_processor.patch_size)
merge_size = video_mm_kwargs.get("merge_size", video_processor.merge_size)
per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2
target_num_frames = max(
target_num_frames,
@@ -1348,6 +1366,47 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
# Merge and resolve the multimodal processor kwargs once so every HF
# call below starts from the same engine/request state and scoped form.
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs(hf_kwargs)
if "images_kwargs" in merged_mm_kwargs:
image_scoped_kwargs = dict(merged_mm_kwargs["images_kwargs"])
# Qwen2VLImageProcessor requires both `size` edges. If a partial size
# override is passed, fill missing edges from the processor defaults
# before any HF call.
if image_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
default_image_size = self.info.get_image_processor().size
image_scoped_kwargs["size"] = self.info._get_vision_size(
image_scoped_kwargs,
default_size=default_image_size,
)
merged_mm_kwargs["images_kwargs"] = image_scoped_kwargs
if "videos_kwargs" in merged_mm_kwargs:
video_scoped_kwargs = dict(merged_mm_kwargs["videos_kwargs"])
# Qwen3VLVideoProcessor requires both `size` edges. If a partial size
# override is passed, fill missing edges from the processor defaults
# before any HF call.
if video_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
default_video_size = self.info.get_video_processor().size
video_scoped_kwargs["size"] = self.info._get_vision_size(
video_scoped_kwargs,
default_size=default_video_size,
)
# Qwen3VLVideoProcessor does not accept vLLM's min/max pixel
# compatibility kwargs. They have already been folded into `size`,
# so remove them before any HF call, including calls that contain
# only text/images.
video_scoped_kwargs.pop("min_pixels", None)
video_scoped_kwargs.pop("max_pixels", None)
merged_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
# Separate video processing from image processing. Because the videos
# are processed into several image patches
video_input_ids_lst: list[list[int]] = []
@@ -1373,32 +1432,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# used to calculate the timestamps. Make sure that
# do_sample_frames in mm_kwargs is false for presampled videos.
# NOTE: a copy of is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect.
video_mm_kwargs: dict[str, Any] = dict(hf_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
video_size = dict(self.info.get_video_processor().size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
video_mm_kwargs["size"] = video_size
sampled_fps = video_mm_kwargs.get("fps")
# The root copy is shallow, so give each item its own video
# scope before modifying per-video values below.
video_mm_kwargs = dict(merged_mm_kwargs)
video_scoped_kwargs = dict(video_mm_kwargs.get("videos_kwargs", {}))
video_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
sampled_fps = video_scoped_kwargs.get("fps")
if is_list_of(sampled_fps, float):
video_mm_kwargs["fps"] = sampled_fps[item_idx]
sampled_num_frames = video_mm_kwargs.get("num_frames")
video_scoped_kwargs["fps"] = sampled_fps[item_idx]
sampled_num_frames = video_scoped_kwargs.get("num_frames")
if is_list_of(sampled_num_frames, int):
video_mm_kwargs["num_frames"] = sampled_num_frames[item_idx]
if "do_sample_frames" not in video_mm_kwargs:
video_scoped_kwargs["num_frames"] = sampled_num_frames[item_idx]
if "do_sample_frames" not in video_scoped_kwargs:
# qwen_vl_utils already has "do_sample_frames" in
# mm_kwargs, don't overwrite it.
video_mm_kwargs["do_sample_frames"] = metadata.get(
video_scoped_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", False
)
@@ -1409,9 +1458,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# Compute timestamps here where we have access to metadata
timestamps = self.info._get_video_second_idx(
metadata=metadata,
do_sample_frames=video_mm_kwargs["do_sample_frames"],
sampled_fps=video_mm_kwargs.get("fps"),
sampled_num_frames=video_mm_kwargs.get("num_frames"),
do_sample_frames=video_scoped_kwargs["do_sample_frames"],
sampled_fps=video_scoped_kwargs.get("fps"),
sampled_num_frames=video_scoped_kwargs.get("num_frames"),
)
timestamps_per_video.append(timestamps)
@@ -1423,8 +1472,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# to prevent HF's BaseVideoProcessor.preprocess() from
# filling in the class default (fps=2) via setdefault(),
# which would conflict with num_frames (mutually exclusive).
if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
video_mm_kwargs["fps"] = None
if (
"num_frames" in video_scoped_kwargs
and "fps" not in video_scoped_kwargs
):
video_scoped_kwargs["fps"] = None
video_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**video_mm_kwargs),
@@ -1485,11 +1537,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
else:
video_outputs = dict()
# fps/num_frames are video-only kwargs already consumed by the loop;
# exclude them so the text/image processor call below never gets a list.
non_video_mm_kwargs = {
k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames")
}
# The root copy is shallow, so copy the video scope before removing
# per-video kwargs to avoid mutating `merged_mm_kwargs`.
non_video_mm_kwargs = dict(merged_mm_kwargs)
if "videos_kwargs" in non_video_mm_kwargs:
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
non_video_scoped_kwargs.pop("fps", None)
non_video_scoped_kwargs.pop("num_frames", None)
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**non_video_mm_kwargs),
hf_data,
+177 -43
View File
@@ -5,7 +5,7 @@ from collections.abc import Callable, Mapping
from contextlib import contextmanager
from dataclasses import dataclass, field
from functools import cached_property
from typing import TYPE_CHECKING, Any, overload
from typing import TYPE_CHECKING, Any, get_type_hints, overload
import torch
from typing_extensions import TypeVar
@@ -42,46 +42,129 @@ else:
logger = init_logger(__name__)
# HuggingFace processors accept nested ``images_kwargs`` / ``videos_kwargs`` /
# ``audio_kwargs`` in addition to a shared flat namespace. vLLM's token-budget
# and dummy-input code only reads the flat keys, so scoped overrides have to
# be overlaid per modality.
_HF_MODALITY_PROCESSOR_KWARGS = {
"image": "images_kwargs",
"video": "videos_kwargs",
"audio": "audio_kwargs",
}
# HuggingFace processors accept a shared flat namespace in addition to the
# nested processor kwarg scopes ``text_kwargs`` / ``images_kwargs`` /
# ``videos_kwargs`` / ``audio_kwargs``. vLLM recognizes these scopes when
# resolving ``mm_processor_kwargs``.
_HF_PROCESSOR_KWARG_SCOPES = (
"text_kwargs",
"images_kwargs",
"videos_kwargs",
"audio_kwargs",
)
def overlay_modality_mm_kwargs(
kwargs: Mapping[str, object],
modality: str | None,
) -> dict[str, Any]:
"""Overlay HF-style nested processor kwargs onto the flat namespace.
Overlay only the requested modality so a video ``size`` bump does not leak
into the image budget. Flat keys keep their current shared-namespace
behavior when no scoped dict is present.
Args:
kwargs: Merged multi-modal processor kwargs.
modality: Target modality (``image``, ``video``, or ``audio``).
When ``None``, ``kwargs`` is copied without overlay.
Returns:
A new dict with the modality-scoped keys overlaid when present.
def _merge_scoped_mm_processor_value(
flat_value: object,
scoped_value: object,
) -> object:
"""Merge a flat value with an existing, more specific scoped value.
Mappings are merged recursively; otherwise the scoped value replaces the
flat value.
"""
merged = dict(kwargs)
if modality is None:
if isinstance(flat_value, Mapping) and isinstance(scoped_value, Mapping):
# Merge recursively so scoped leaves override matching flat leaves
# without dropping unrelated flat entries.
merged = dict(flat_value)
for key, value in scoped_value.items():
if key in merged:
merged[key] = _merge_scoped_mm_processor_value(merged[key], value)
else:
merged[key] = value
return merged
scoped_key = _HF_MODALITY_PROCESSOR_KWARGS.get(modality)
if scoped_key is None:
return merged
scoped = merged.get(scoped_key)
if not isinstance(scoped, Mapping):
return merged
return merged | dict(scoped)
return scoped_value
def _resolve_mm_processor_kwargs(
kwargs: Mapping[str, object],
supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None,
) -> dict[str, object]:
"""Resolve flat processor kwargs into HuggingFace processor kwarg scopes.
With ``supported_mm_processor_kwargs``, each scope lists the flat keys it
supports. Matching flat keys are routed to every scope that supports them,
existing scoped values take precedence, and keys not supported by any scope
remain flat.
Without ``supported_mm_processor_kwargs``, flat keys already represented in
at least one existing mapping-valued scope are removed from the shared flat
namespace; all other flat keys are left unchanged because there is no
information to determine which scopes should receive them.
"""
resolved = dict(kwargs)
# Without ``supported_mm_processor_kwargs``, remove flat entries already
# represented in at least one existing mapping-valued scope. There is no
# information to route the remaining flat entries.
if supported_mm_processor_kwargs is None:
# Collect all keys represented in existing mapping-valued scopes.
# Non-mapping scope values do not contain scoped kwargs and are left
# untouched.
represented_scoped_keys: set[str] = set()
for scoped_key in _HF_PROCESSOR_KWARG_SCOPES:
scoped_kwargs = resolved.get(scoped_key)
if isinstance(scoped_kwargs, Mapping):
represented_scoped_keys.update(scoped_kwargs)
# Remove flat entries for keys already represented in a scope.
for key in represented_scoped_keys:
if key not in _HF_PROCESSOR_KWARG_SCOPES:
resolved.pop(key, None)
return resolved
# With ``supported_mm_processor_kwargs``, collect the flat kwargs before
# routing each one to the processor kwarg scopes that support it.
flat_kwargs = {
key: value
for key, value in resolved.items()
if key not in _HF_PROCESSOR_KWARG_SCOPES
}
for key, flat_value in flat_kwargs.items():
# Find every processor kwarg scope that supports this flat key. If none
# supports it, keep the key in the shared flat namespace.
supported_scopes = [
scoped_key
for scoped_key in _HF_PROCESSOR_KWARG_SCOPES
if key in supported_mm_processor_kwargs.get(scoped_key, set())
]
if not supported_scopes:
continue
# Add the flat value to each supported destination scope.
for scoped_key in supported_scopes:
# Start from a copy of the existing scoped kwargs, or an empty scope
# if absent.
if scoped_key in resolved:
scoped_kwargs = resolved[scoped_key]
# A scope must be a mapping before keys can be added to it. Do
# not replace an explicit non-mapping value.
if not isinstance(scoped_kwargs, Mapping):
raise TypeError(f"`{scoped_key}` must be a mapping")
scoped_kwargs = dict(scoped_kwargs)
else:
scoped_kwargs = {}
# Add the flat value to this scope. Existing scoped values take
# precedence.
if key in scoped_kwargs:
scoped_kwargs[key] = _merge_scoped_mm_processor_value(
flat_value, scoped_kwargs[key]
)
else:
# Assign the flat value to this scope. For mappings, assign a
# copy so the same mapping object is not shared between the flat
# value and multiple scopes.
scoped_kwargs[key] = (
dict(flat_value) if isinstance(flat_value, Mapping) else flat_value
)
resolved[scoped_key] = scoped_kwargs
# After routing to every destination, remove the flat copy.
resolved.pop(key)
return resolved
@dataclass
@@ -294,19 +377,27 @@ class InputProcessingContext:
self,
kwargs: Mapping[str, object],
*,
modality: str | None = None,
supported_mm_processor_kwargs: Mapping[str, set[str]] | None = None,
) -> dict[str, Any]:
"""Merge configured and request ``mm_processor_kwargs``.
When ``modality`` is set, HF-style nested
``images_kwargs`` / ``videos_kwargs`` / ``audio_kwargs`` are overlaid
onto the flat namespace for vLLM-side reads (token budgets, dummy
inputs). Processor construction and HF ``__call__`` should omit
``modality`` so the nested dicts still reach the HF processor.
When ``supported_mm_processor_kwargs`` is provided, flat keys are matched
against the keys supported by each HuggingFace processor kwarg scope.
Matching flat keys are routed to every scope that supports them and
removed from the shared flat namespace; keys not supported by any scope
remain flat.
Without ``supported_mm_processor_kwargs``, flat keys already represented
in at least one existing mapping-valued scope are removed from the shared
flat namespace; all other flat keys are left unchanged because there is
no information to determine which scopes should receive them.
"""
mm_config = self.model_config.get_multimodal_config()
merged = mm_config.merge_mm_processor_kwargs(kwargs)
return overlay_modality_mm_kwargs(merged, modality)
return _resolve_mm_processor_kwargs(
merged,
supported_mm_processor_kwargs=supported_mm_processor_kwargs,
)
def call_hf_processor(
self,
@@ -380,6 +471,49 @@ class BaseProcessingInfo:
"""
return self.ctx.get_hf_processor(**kwargs)
def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
"""Return supported kwarg names for each HF processor kwargs scope."""
processor = self.get_hf_processor()
processor_kwargs = get_type_hints(processor.valid_processor_kwargs)
supported = {
"text_kwargs": set(processor_kwargs["text_kwargs"].__annotations__),
}
if "image" in self.supported_mm_limits:
supported["images_kwargs"] = set(
processor.image_processor.valid_kwargs.__annotations__
)
if "video" in self.supported_mm_limits:
supported["videos_kwargs"] = set(
processor.video_processor.valid_kwargs.__annotations__
)
if "audio" in self.supported_mm_limits:
supported["audio_kwargs"] = set(
processor_kwargs["audio_kwargs"].__annotations__
)
return supported
@cached_property
def supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
"""Supported kwarg names for each HF processor kwargs scope."""
return self.get_supported_mm_processor_kwargs()
def _merge_and_resolve_mm_processor_kwargs(
self,
mm_kwargs: Mapping[str, object],
) -> dict[str, Any]:
"""Merge configured and request ``mm_processor_kwargs``.
Flat kwargs are routed into the HuggingFace processor kwarg scopes that
support them after the configured/request merge. When a routed flat value
conflicts with an existing scoped value, the scoped value takes precedence.
"""
return self.ctx.get_merged_mm_kwargs(
mm_kwargs,
supported_mm_processor_kwargs=self.supported_mm_processor_kwargs,
)
def get_default_tok_params(self) -> TokenizeParams:
"""Construct the default parameters for tokenization."""
from vllm.renderers import TokenizeParams