[Bugfix][Multimodal] Fix per-video cache option reuse (#55203)

Signed-off-by: bvolpato <brunocvcunha@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Bruno Volpato
2026-10-11 06:21:55 +00:00
committed by GitHub
co-authored by Codex
parent 2eb491e18b
commit 61f084b5f3
3 changed files with 197 additions and 45 deletions
@@ -15,6 +15,7 @@ from vllm.config import ModelConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.processing.context import BaseProcessingInfo
from ...registry import HF_EXAMPLE_MODELS
@@ -336,65 +337,136 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
("hf_mm_kwargs", "scalar_kwargs"),
[
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
({"fps": [2.0, 4.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
({"num_frames": [8, 16, 32]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [2.0, 4.0, 8.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
],
)
@pytest.mark.parametrize("model_kwargs", [False, True])
@pytest.mark.parametrize("use_uuids", [False, True])
@pytest.mark.parametrize("scoped", [False, True])
@pytest.mark.skip_global_cleanup
def test_processor_multi_video_list_kwargs(
model_id: str,
hf_mm_kwargs: dict[str, Any],
scalar_kwargs: list[dict[str, Any]],
model_kwargs: bool,
use_uuids: bool,
scoped: bool,
) -> None:
"""List-valued video kwargs must be scalarized per item without mutation.
Each video in the multi-video request must match processing the same input
independently with its corresponding scalar ``fps``/``num_frames`` value.
"""
"""Per-video options must survive duplicate items and partial cache hits."""
if scoped:
hf_mm_kwargs = {"videos_kwargs": hf_mm_kwargs}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 2},
mm_processor_kwargs=hf_mm_kwargs if model_kwargs else None,
mm_processor_cache_gb=1,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = (
"<|vision_start|><|video_pad|><|vision_end|>"
"<|vision_start|><|video_pad|><|vision_end|>"
)
mm_data = {
"video": [
_build_video_mm_data(num_frames=32)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
]
}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
videos = [
_build_video_mm_data(num_frames=32, original_fps=8.0)["video"][0]
for _ in range(3)
]
for i, (frames, _) in enumerate(videos):
frames.fill(i)
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
single_mm_data = _build_video_mm_data(num_frames=32)
single = processor(
single_prompt,
mm_items=processor.info.parse_mm_data(single_mm_data),
mm_items=processor.info.parse_mm_data({"video": [videos[0]]}),
hf_processor_mm_kwargs=hf_kwargs,
)
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
assert not torch.equal(expected_grids[0], expected_grids[1])
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
def run(indices, *, cache=None, cached_only=False):
return processor(
prompt,
mm_items=processor.info.parse_mm_data(
{"video": [None if cached_only else videos[i] for i in indices]}
),
mm_uuid_items=(
{"video": [f"video-{i}" for i in indices]} if use_uuids else None
),
hf_processor_mm_kwargs={} if model_kwargs else hf_mm_kwargs,
cache=cache,
)
for indices in ([0, 0], [0, 1], [0, 2], [2, 0]):
expected = run(indices)
actual = run(indices, cache=cache)
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
assert actual["mm_placeholders"] == expected["mm_placeholders"]
assert actual["mm_kwargs"] == expected["mm_kwargs"]
assert actual["mm_hashes"] == expected["mm_hashes"]
assert len(set(actual["mm_hashes"]["video"])) == 2
multi_grids = actual["mm_kwargs"].get_data()["video_grid_thw"]
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
if use_uuids:
cached_only = run([2, 0], cache=cache, cached_only=True)
assert cached_only["prompt_token_ids"] == actual["prompt_token_ids"]
assert cached_only["mm_kwargs"] == actual["mm_kwargs"]
assert hf_mm_kwargs == hf_mm_kwargs_before
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 2, (
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
@pytest.mark.parametrize(("key", "value"), [("fps", 2.0), ("num_frames", 8)])
@pytest.mark.parametrize("length", [0, 1])
@pytest.mark.parametrize("use_cache", [False, True])
@pytest.mark.parametrize("scoped", [False, True])
def test_processor_rejects_short_per_video_options(
key: str, value: float | int, length: int, use_cache: bool, scoped: bool
) -> None:
"""Reject malformed option lists with and without a warm processor cache."""
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 0, "video": 2},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
cache = MultiModalProcessorOnlyCache(ctx.model_config) if use_cache else None
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
videos = _build_video_mm_data(num_frames=32, original_fps=8.0)["video"] * 2
def process(values: list[float | int]) -> None:
kwargs = {key: values}
processor(
prompt,
mm_items=processor.info.parse_mm_data({"video": videos}),
hf_processor_mm_kwargs={"videos_kwargs": kwargs} if scoped else kwargs,
cache=cache,
)
if use_cache:
process([value, value])
with pytest.raises(ValueError, match=f"{key} must have at least 2 entries"):
process([value] * length)
@pytest.mark.parametrize("kwargs", [{"fps": [2.0, 4.0]}, {"num_frames": [8, 16]}])
def test_processor_profiles_fewer_videos_than_configured_options(
kwargs: dict[str, list[float] | list[int]],
) -> None:
"""Profiling may use fewer videos than the configured per-video list."""
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 0, "video": 2},
mm_processor_kwargs=kwargs,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
profiled = processor.get_dummy_mm_inputs({"video": 1})
assert len(profiled["mm_placeholders"]["video"]) == 1
assert profiled["mm_kwargs"].get_data()["video_grid_thw"].shape[0] == 1
def _build_video_embeds_mm_data(