[Bugfix][Multimodal] Fix flat/scoped mm_processor_kwargs merge and resolution (#56372)
Signed-off-by: Daniel M. García-Ocaña Hernández <danielgarciaocana@gmail.com>
This commit is contained in:
@@ -202,19 +202,19 @@ def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
@pytest.mark.parametrize("kwargs_on_init", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
|
||||
)
|
||||
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
|
||||
def test_video_cache_is_independent_of_image_kwargs(
|
||||
model_id: str,
|
||||
kwargs_on_init: bool,
|
||||
image_kwargs: dict[str, object],
|
||||
video_uuid: str | None,
|
||||
):
|
||||
"""Image overrides must not change video frames or depend on cache warmth."""
|
||||
kwargs = {"images_kwargs": image_kwargs}
|
||||
"""Request image overrides must not change video frames or make results
|
||||
depend on cache warmth.
|
||||
"""
|
||||
request_kwargs = {"images_kwargs": image_kwargs}
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
@@ -239,18 +239,11 @@ def test_video_cache_is_independent_of_image_kwargs(
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
baseline = process({}, cache=None)
|
||||
if kwargs_on_init:
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=kwargs,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
request_kwargs = {} if kwargs_on_init else kwargs
|
||||
process(request_kwargs, cache=cache)
|
||||
# Populate the video cache without image overrides, then change only
|
||||
# images_kwargs on the next request. Since images_kwargs are excluded from
|
||||
# the video hash, and must not affect the processed video, the cached result
|
||||
# must match a fresh recomputation.
|
||||
baseline = process({}, cache=cache)
|
||||
cached = process(request_kwargs, cache=cache)
|
||||
cache.clear_cache()
|
||||
fresh = process(request_kwargs, cache=cache)
|
||||
|
||||
@@ -100,6 +100,11 @@ def _expected_placeholder_tokens_per_image(
|
||||
[
|
||||
({}, (448, 448), 256),
|
||||
({"size": {"longest_edge": 1008}}, (1540, 1540), 1296),
|
||||
(
|
||||
{"images_kwargs": {"size": {"longest_edge": 1008}}},
|
||||
(1540, 1540),
|
||||
1296,
|
||||
),
|
||||
({"size": {"longest_edge": 1288}}, (1536, 1187), 1656),
|
||||
({"size": {"longest_edge": 1008}}, (29, 29), 4),
|
||||
({"size": {"longest_edge": 1000}}, (1540, 1700), 1188),
|
||||
@@ -148,6 +153,40 @@ def test_processor_size_override(
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
def test_scoped_request_size_overrides_configured_flat_size():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
mm_processor_kwargs={"size": {"longest_edge": 1008}},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
model_config_kwargs=_MODEL_CONFIG_KWARGS,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = AutoProcessor.from_pretrained(
|
||||
_MODEL_ID,
|
||||
fix_mistral_regex=True,
|
||||
)
|
||||
|
||||
hf_processor_mm_kwargs: dict[str, object] = {
|
||||
"images_kwargs": {"size": {"longest_edge": 1288}}
|
||||
}
|
||||
image = Image.new("RGB", (1536, 1187), color=(127, 127, 127))
|
||||
|
||||
merged_mm_kwargs = processor.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
|
||||
pixel_values, hf_placeholder_tokens = _process_images_with_hf(
|
||||
hf_processor, [image], merged_mm_kwargs
|
||||
)
|
||||
prompt_update_tokens = _placeholder_tokens_from_prompt_updates(
|
||||
processor, [image], pixel_values, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
expected_from_pixel_values = _expected_placeholder_tokens_per_image(
|
||||
hf_processor, pixel_values[0]
|
||||
)
|
||||
assert expected_from_pixel_values == 1656
|
||||
assert hf_placeholder_tokens.count(hf_processor.image_token_id) == 1656
|
||||
assert prompt_update_tokens == hf_placeholder_tokens
|
||||
|
||||
|
||||
def test_lightonocr_keeps_vision_config_image_size():
|
||||
ctx = build_model_context(
|
||||
_LIGHTON_MODEL_ID,
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
@@ -8,16 +10,238 @@ import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.models.qwen2_vl import Qwen2VLProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import batched_tensors_equal
|
||||
from vllm.multimodal.processing.context import InputProcessingContext
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_kwargs", "expected"),
|
||||
[
|
||||
(
|
||||
{"size": {"shortest_edge": 64, "longest_edge": 1024}},
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 64,
|
||||
"max_pixels": 1024,
|
||||
},
|
||||
),
|
||||
(
|
||||
{"min_pixels": 64, "max_pixels": 1024},
|
||||
{
|
||||
"min_pixels": 64,
|
||||
"max_pixels": 1024,
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 2048,
|
||||
},
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 1024},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 2048,
|
||||
},
|
||||
),
|
||||
(
|
||||
{
|
||||
"min_pixels": None,
|
||||
"size": {"shortest_edge": 0},
|
||||
"images_kwargs": {"max_pixels": 4096},
|
||||
"videos_kwargs": {"size": {"longest_edge": 8192}},
|
||||
},
|
||||
{
|
||||
"min_pixels": None,
|
||||
"size": {"shortest_edge": 0},
|
||||
"images_kwargs": {
|
||||
"max_pixels": 4096,
|
||||
"size": {"longest_edge": 4096},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"size": {"longest_edge": 8192},
|
||||
"max_pixels": 8192,
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
{"min_pixels": 0},
|
||||
{"min_pixels": 0, "size": {"shortest_edge": 0}},
|
||||
),
|
||||
(
|
||||
{"size": None, "min_pixels": 64},
|
||||
{"size": {"shortest_edge": 64}, "min_pixels": 64},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_complete_mm_processor_size_aliases(
|
||||
mm_kwargs: dict[str, object],
|
||||
expected: dict[str, object],
|
||||
) -> None:
|
||||
original = deepcopy(mm_kwargs)
|
||||
|
||||
actual = Qwen2VLProcessingInfo._complete_mm_processor_size_aliases(mm_kwargs)
|
||||
|
||||
assert actual == expected
|
||||
assert mm_kwargs == original
|
||||
|
||||
|
||||
def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedence(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={
|
||||
"images_kwargs": {"min_pixels": 64},
|
||||
},
|
||||
mm_device_do_normalize=False,
|
||||
)
|
||||
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
|
||||
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
|
||||
info = Qwen2VLProcessingInfo(ctx)
|
||||
monkeypatch.setattr(
|
||||
info,
|
||||
"get_supported_mm_processor_kwargs",
|
||||
lambda: {"images_kwargs": {"size", "min_pixels"}},
|
||||
)
|
||||
|
||||
assert info._merge_and_resolve_mm_processor_kwargs(
|
||||
{"size": {"shortest_edge": 128}}
|
||||
) == {
|
||||
"images_kwargs": {
|
||||
"size": {"shortest_edge": 128},
|
||||
"min_pixels": 128,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_kwargs", "default_size", "expected"),
|
||||
[
|
||||
(
|
||||
{},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
),
|
||||
(
|
||||
{"size": {"shortest_edge": 64}},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 64, "longest_edge": 1024},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 2048},
|
||||
"min_pixels": 128,
|
||||
"max_pixels": 4096,
|
||||
},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 128, "longest_edge": 4096},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64},
|
||||
"max_pixels": 4096,
|
||||
},
|
||||
None,
|
||||
{"shortest_edge": 64, "longest_edge": 4096},
|
||||
),
|
||||
(
|
||||
{
|
||||
"size": {"shortest_edge": 64, "longest_edge": 2048},
|
||||
"min_pixels": None,
|
||||
"max_pixels": 0,
|
||||
},
|
||||
{"shortest_edge": 32, "longest_edge": 1024},
|
||||
{"shortest_edge": 64, "longest_edge": 0},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_vision_size(
|
||||
mm_kwargs: dict[str, object],
|
||||
default_size: dict[str, object] | None,
|
||||
expected: dict[str, object],
|
||||
) -> None:
|
||||
original_mm_kwargs = {
|
||||
key: dict(value) if isinstance(value, dict) else value
|
||||
for key, value in mm_kwargs.items()
|
||||
}
|
||||
original_default_size = dict(default_size) if default_size is not None else None
|
||||
|
||||
actual = Qwen2VLProcessingInfo._get_vision_size(
|
||||
mm_kwargs,
|
||||
default_size=default_size,
|
||||
)
|
||||
|
||||
assert actual == expected
|
||||
assert mm_kwargs == original_mm_kwargs
|
||||
assert default_size == original_default_size
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("modality", "scope"),
|
||||
[
|
||||
("image", "images_kwargs"),
|
||||
("video", "videos_kwargs"),
|
||||
],
|
||||
)
|
||||
def test_get_vision_info_reads_merged_modality_scope(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
modality: str,
|
||||
scope: str,
|
||||
) -> None:
|
||||
vision_config = SimpleNamespace(
|
||||
patch_size=14,
|
||||
spatial_merge_size=2,
|
||||
temporal_patch_size=2,
|
||||
)
|
||||
ctx = SimpleNamespace(
|
||||
get_hf_config=lambda *_: SimpleNamespace(vision_config=vision_config)
|
||||
)
|
||||
info = Qwen2VLProcessingInfo(ctx)
|
||||
merged = {
|
||||
"images_kwargs": {"size": {"shortest_edge": 111}},
|
||||
"videos_kwargs": {"size": {"shortest_edge": 222}},
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
info,
|
||||
"_merge_and_resolve_mm_processor_kwargs",
|
||||
lambda _: merged,
|
||||
)
|
||||
|
||||
seen: list[dict[str, object]] = []
|
||||
|
||||
def get_size(
|
||||
mm_kwargs: dict[str, object],
|
||||
default_size: dict[str, object] | None = None,
|
||||
) -> dict[str, object]:
|
||||
seen.append(mm_kwargs)
|
||||
return dict(default_size or {})
|
||||
|
||||
monkeypatch.setattr(info, "_get_vision_size", get_size)
|
||||
image_processor = SimpleNamespace(size={"shortest_edge": 32, "longest_edge": 1024})
|
||||
|
||||
info._get_vision_info(
|
||||
image_width=28,
|
||||
image_height=28,
|
||||
num_frames=2,
|
||||
do_resize=False,
|
||||
image_processor=image_processor,
|
||||
mm_kwargs={},
|
||||
modality=modality, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert seen == [merged[scope]]
|
||||
|
||||
|
||||
def test_jina_vl_processing_order() -> None:
|
||||
"""Jina's document-first prompt keeps cached features and hashes aligned."""
|
||||
ctx = build_model_context(
|
||||
|
||||
@@ -1,19 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for Qwen3-VL processor.
|
||||
|
||||
Covers the fix for num_frames-based timestamp calculation
|
||||
(issue vllm-project/vllm#35909).
|
||||
"""
|
||||
"""Regression tests for Qwen3-VL multimodal processing."""
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers.models.qwen3_vl import Qwen3VLVideoProcessor
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.processing.context import BaseProcessingInfo
|
||||
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import build_model_context
|
||||
@@ -21,6 +23,32 @@ from ...utils import build_model_context
|
||||
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
|
||||
|
||||
|
||||
def test_qwen3_video_routing_schema_adds_vllm_compatibility_kwargs(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
native_video_kwargs = set(Qwen3VLVideoProcessor.valid_kwargs.__annotations__)
|
||||
assert "size" in native_video_kwargs
|
||||
assert "min_pixels" not in native_video_kwargs
|
||||
assert "max_pixels" not in native_video_kwargs
|
||||
|
||||
info = Qwen3VLProcessingInfo(SimpleNamespace())
|
||||
parent_supported = {
|
||||
"text_kwargs": {"padding"},
|
||||
"images_kwargs": {"size", "min_pixels", "max_pixels"},
|
||||
"videos_kwargs": set(native_video_kwargs),
|
||||
}
|
||||
monkeypatch.setattr(
|
||||
BaseProcessingInfo,
|
||||
"get_supported_mm_processor_kwargs",
|
||||
lambda self: {scope: set(kwargs) for scope, kwargs in parent_supported.items()},
|
||||
)
|
||||
|
||||
supported = info.supported_mm_processor_kwargs
|
||||
|
||||
assert {"min_pixels", "max_pixels"} <= supported["videos_kwargs"]
|
||||
assert parent_supported["videos_kwargs"] == native_video_kwargs
|
||||
|
||||
|
||||
def _build_video_mm_data(
|
||||
num_frames: int,
|
||||
width: int = 128,
|
||||
@@ -45,6 +73,122 @@ def _build_video_mm_data(
|
||||
return {"video": [(video, metadata)]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_partial_scoped_image_size(model_id: str) -> None:
|
||||
"""Partial image ``size`` must be completed and applied before the HF call.
|
||||
|
||||
Keep video compatibility and per-video kwargs in an image-only call as well:
|
||||
Qwen3 must sanitize the video scope before HF validates it. Regression for
|
||||
#56363.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1, "video": 0},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
image = np.zeros((128, 128, 3), dtype=np.uint8)
|
||||
min_pixels = 256 * 32 * 32
|
||||
default_size = dict(processor.info.get_image_processor().size)
|
||||
complete_size = default_size | {"shortest_edge": min_pixels}
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data({"image": [image]})
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"image_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
complete_grid = process({"images_kwargs": {"size": complete_size}})
|
||||
partial_grid = process(
|
||||
{
|
||||
"images_kwargs": {
|
||||
"size": {"shortest_edge": min_pixels},
|
||||
},
|
||||
"videos_kwargs": {
|
||||
"min_pixels": 4096,
|
||||
"max_pixels": 469_762_048,
|
||||
"fps": [2.0],
|
||||
"num_frames": [8],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(partial_grid, complete_grid)
|
||||
assert not torch.equal(partial_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_partial_scoped_video_size(model_id: str) -> None:
|
||||
"""Partial scoped video ``size`` must be completed and applied before HF."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
min_pixels = 512 * 32 * 32
|
||||
default_size = dict(processor.info.get_video_processor().size)
|
||||
complete_size = default_size | {"shortest_edge": min_pixels}
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"video_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
complete_grid = process({"videos_kwargs": {"size": complete_size}})
|
||||
partial_grid = process(
|
||||
{
|
||||
"videos_kwargs": {
|
||||
"size": {"shortest_edge": min_pixels},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(partial_grid, complete_grid)
|
||||
assert not torch.equal(partial_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_processor_flat_video_pixel_aliases_match_scoped_size(model_id: str) -> None:
|
||||
"""Flat min/max pixel aliases keep their historical Qwen3 video effect."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
pixel_budget = 512 * 32 * 32
|
||||
|
||||
def process(hf_mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data(_build_video_mm_data(num_frames=8))
|
||||
return processor._apply_hf_processor_main(mm_items, hf_mm_kwargs)[
|
||||
"video_grid_thw"
|
||||
]
|
||||
|
||||
stock_grid = process({})
|
||||
scoped_grid = process(
|
||||
{
|
||||
"videos_kwargs": {
|
||||
"size": {
|
||||
"shortest_edge": pixel_budget,
|
||||
"longest_edge": pixel_budget,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
flat_grid = process(
|
||||
{
|
||||
"min_pixels": pixel_budget,
|
||||
"max_pixels": pixel_budget,
|
||||
}
|
||||
)
|
||||
|
||||
assert torch.equal(flat_grid, scoped_grid)
|
||||
assert not torch.equal(flat_grid, stock_grid)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"num_frames",
|
||||
@@ -189,20 +333,21 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"hf_mm_kwargs",
|
||||
[{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
|
||||
("hf_mm_kwargs", "scalar_kwargs"),
|
||||
[
|
||||
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
|
||||
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
|
||||
],
|
||||
)
|
||||
def test_processor_multi_video_list_kwargs(
|
||||
model_id: str,
|
||||
hf_mm_kwargs: dict[str, Any],
|
||||
scalar_kwargs: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Regression test: a multi-video request with list-valued per-video
|
||||
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
|
||||
crash.
|
||||
"""List-valued video kwargs must be scalarized per item without mutation.
|
||||
|
||||
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
|
||||
video without slicing, so ``_get_video_second_idx`` received the list
|
||||
where a scalar was expected and raised ``TypeError``.
|
||||
Each video in the multi-video request must match processing the same input
|
||||
independently with its corresponding scalar ``fps``/``num_frames`` value.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
@@ -216,17 +361,36 @@ def test_processor_multi_video_list_kwargs(
|
||||
)
|
||||
mm_data = {
|
||||
"video": [
|
||||
_build_video_mm_data(num_frames=16)["video"][0],
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
]
|
||||
}
|
||||
|
||||
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_mm_kwargs,
|
||||
)
|
||||
|
||||
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
single_mm_data = _build_video_mm_data(num_frames=32)
|
||||
single = processor(
|
||||
single_prompt,
|
||||
mm_items=processor.info.parse_mm_data(single_mm_data),
|
||||
hf_processor_mm_kwargs=hf_kwargs,
|
||||
)
|
||||
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
|
||||
|
||||
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
|
||||
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
|
||||
|
||||
assert not torch.equal(expected_grids[0], expected_grids[1])
|
||||
assert torch.equal(multi_grids[0], expected_grids[0])
|
||||
assert torch.equal(multi_grids[1], expected_grids[1])
|
||||
assert hf_mm_kwargs == hf_mm_kwargs_before
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == 2, (
|
||||
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
|
||||
@@ -336,7 +500,12 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
|
||||
|
||||
capped_ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
|
||||
mm_processor_kwargs={
|
||||
"videos_kwargs": {
|
||||
"size": size,
|
||||
"cap_pixels_per_frame": True,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
|
||||
@@ -350,7 +519,7 @@ def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> Non
|
||||
|
||||
uncapped_ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"size": size},
|
||||
mm_processor_kwargs={"videos_kwargs": {"size": size}},
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
|
||||
|
||||
Reference in New Issue
Block a user