[Bugfix][Multimodal] Fix per-video cache option reuse (#55203)

Signed-off-by: bvolpato <brunocvcunha@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Bruno Volpato
2026-10-11 06:21:55 +00:00
committed by GitHub
co-authored by Codex
parent 2eb491e18b
commit 61f084b5f3
3 changed files with 197 additions and 45 deletions
@@ -15,6 +15,7 @@ from vllm.config import ModelConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.processing.context import BaseProcessingInfo
from ...registry import HF_EXAMPLE_MODELS
@@ -336,65 +337,136 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
("hf_mm_kwargs", "scalar_kwargs"),
[
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
({"fps": [2.0, 4.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
({"num_frames": [8, 16, 32]}, [{"num_frames": 8}, {"num_frames": 16}]),
({"fps": [2.0, 4.0, 8.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
],
)
@pytest.mark.parametrize("model_kwargs", [False, True])
@pytest.mark.parametrize("use_uuids", [False, True])
@pytest.mark.parametrize("scoped", [False, True])
@pytest.mark.skip_global_cleanup
def test_processor_multi_video_list_kwargs(
model_id: str,
hf_mm_kwargs: dict[str, Any],
scalar_kwargs: list[dict[str, Any]],
model_kwargs: bool,
use_uuids: bool,
scoped: bool,
) -> None:
"""List-valued video kwargs must be scalarized per item without mutation.
Each video in the multi-video request must match processing the same input
independently with its corresponding scalar ``fps``/``num_frames`` value.
"""
"""Per-video options must survive duplicate items and partial cache hits."""
if scoped:
hf_mm_kwargs = {"videos_kwargs": hf_mm_kwargs}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 2},
mm_processor_kwargs=hf_mm_kwargs if model_kwargs else None,
mm_processor_cache_gb=1,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = (
"<|vision_start|><|video_pad|><|vision_end|>"
"<|vision_start|><|video_pad|><|vision_end|>"
)
mm_data = {
"video": [
_build_video_mm_data(num_frames=32)["video"][0],
_build_video_mm_data(num_frames=32)["video"][0],
]
}
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=hf_mm_kwargs,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
videos = [
_build_video_mm_data(num_frames=32, original_fps=8.0)["video"][0]
for _ in range(3)
]
for i, (frames, _) in enumerate(videos):
frames.fill(i)
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
single_mm_data = _build_video_mm_data(num_frames=32)
single = processor(
single_prompt,
mm_items=processor.info.parse_mm_data(single_mm_data),
mm_items=processor.info.parse_mm_data({"video": [videos[0]]}),
hf_processor_mm_kwargs=hf_kwargs,
)
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
assert not torch.equal(expected_grids[0], expected_grids[1])
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
def run(indices, *, cache=None, cached_only=False):
return processor(
prompt,
mm_items=processor.info.parse_mm_data(
{"video": [None if cached_only else videos[i] for i in indices]}
),
mm_uuid_items=(
{"video": [f"video-{i}" for i in indices]} if use_uuids else None
),
hf_processor_mm_kwargs={} if model_kwargs else hf_mm_kwargs,
cache=cache,
)
for indices in ([0, 0], [0, 1], [0, 2], [2, 0]):
expected = run(indices)
actual = run(indices, cache=cache)
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
assert actual["mm_placeholders"] == expected["mm_placeholders"]
assert actual["mm_kwargs"] == expected["mm_kwargs"]
assert actual["mm_hashes"] == expected["mm_hashes"]
assert len(set(actual["mm_hashes"]["video"])) == 2
multi_grids = actual["mm_kwargs"].get_data()["video_grid_thw"]
assert torch.equal(multi_grids[0], expected_grids[0])
assert torch.equal(multi_grids[1], expected_grids[1])
if use_uuids:
cached_only = run([2, 0], cache=cache, cached_only=True)
assert cached_only["prompt_token_ids"] == actual["prompt_token_ids"]
assert cached_only["mm_kwargs"] == actual["mm_kwargs"]
assert hf_mm_kwargs == hf_mm_kwargs_before
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 2, (
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
@pytest.mark.parametrize(("key", "value"), [("fps", 2.0), ("num_frames", 8)])
@pytest.mark.parametrize("length", [0, 1])
@pytest.mark.parametrize("use_cache", [False, True])
@pytest.mark.parametrize("scoped", [False, True])
def test_processor_rejects_short_per_video_options(
key: str, value: float | int, length: int, use_cache: bool, scoped: bool
) -> None:
"""Reject malformed option lists with and without a warm processor cache."""
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 0, "video": 2},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
cache = MultiModalProcessorOnlyCache(ctx.model_config) if use_cache else None
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
videos = _build_video_mm_data(num_frames=32, original_fps=8.0)["video"] * 2
def process(values: list[float | int]) -> None:
kwargs = {key: values}
processor(
prompt,
mm_items=processor.info.parse_mm_data({"video": videos}),
hf_processor_mm_kwargs={"videos_kwargs": kwargs} if scoped else kwargs,
cache=cache,
)
if use_cache:
process([value, value])
with pytest.raises(ValueError, match=f"{key} must have at least 2 entries"):
process([value] * length)
@pytest.mark.parametrize("kwargs", [{"fps": [2.0, 4.0]}, {"num_frames": [8, 16]}])
def test_processor_profiles_fewer_videos_than_configured_options(
kwargs: dict[str, list[float] | list[int]],
) -> None:
"""Profiling may use fewer videos than the configured per-video list."""
ctx = build_model_context(
MODEL_ID,
limit_mm_per_prompt={"image": 0, "video": 2},
mm_processor_kwargs=kwargs,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
profiled = processor.get_dummy_mm_inputs({"video": 1})
assert len(profiled["mm_placeholders"]["video"]) == 1
assert profiled["mm_kwargs"].get_data()["video_grid_thw"].shape[0] == 1
def _build_video_embeds_mm_data(
+68 -1
View File
@@ -60,7 +60,7 @@ from vllm.config.multimodal import (
VideoPruningMethod,
)
from vllm.distributed import get_pp_group, parallel_state
from vllm.inputs import MultiModalDataDict
from vllm.inputs import MultiModalDataDict, MultiModalHashes
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
from vllm.model_executor.layers.attention.mm_encoder_attention import (
@@ -81,6 +81,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
@@ -94,6 +95,7 @@ from vllm.multimodal.parse import ImageSize, MultiModalDataItems
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
@@ -1342,6 +1344,67 @@ def _replace_video_token_placeholders(
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
def _get_per_video_kwargs(
self, kwargs: Mapping[str, object], num_videos: int | None = None
) -> dict[str, list[Any]]:
merged = self.info._merge_and_resolve_mm_processor_kwargs(kwargs)
per_video_kwargs = {
key: value
for key, value in merged.get("videos_kwargs", {}).items()
if key in ("fps", "num_frames") and isinstance(value, list)
}
if num_videos:
for key, values in per_video_kwargs.items():
if len(values) < num_videos:
raise ValueError(
f"{key} must have at least {num_videos} entries "
"(one per video); "
f"got {len(values)}."
)
return per_video_kwargs
def _get_mm_hashes(self, inputs: ProcessorInputs) -> MultiModalHashes:
hashes = dict(super()._get_mm_hashes(inputs))
per_video_kwargs = self._get_per_video_kwargs(
inputs.hf_processor_mm_kwargs, len(hashes.get("video", []))
)
if per_video_kwargs and "video" in hashes:
algorithm = self.info.ctx.get_mm_config().mm_hasher_algorithm
hashes["video"] = [
MultiModalHasher.hash_kwargs(
algorithm,
mm_hash=mm_hash,
**{key: values[i] for key, values in per_video_kwargs.items()},
)
for i, mm_hash in enumerate(hashes["video"])
]
return hashes
def _get_cache_missing_processor_kwargs(
self,
inputs: ProcessorInputs,
mm_is_cached: Mapping[str, list[bool]],
) -> Mapping[str, object]:
per_video_kwargs = self._get_per_video_kwargs(
inputs.hf_processor_mm_kwargs, len(mm_is_cached.get("video", []))
)
if not per_video_kwargs or "video" not in mm_is_cached:
return inputs.hf_processor_mm_kwargs
kwargs = self.info._merge_and_resolve_mm_processor_kwargs(
inputs.hf_processor_mm_kwargs
)
kwargs["videos_kwargs"].update(
{
key: [
value
for value, hit in zip(values, mm_is_cached["video"])
if not hit
]
for key, values in per_video_kwargs.items()
}
)
return kwargs
@staticmethod
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
"""Transformers>=5.10 processors override `replace_video_token`
@@ -1412,6 +1475,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
video_input_ids_lst: list[list[int]] = []
if videos := hf_data.pop("videos", []):
assert isinstance(videos, list)
self._get_per_video_kwargs(hf_kwargs, len(videos))
video_grid_thw_lst = []
pixel_values_videos_lst = []
timestamps_per_video = []
@@ -1544,6 +1608,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
non_video_scoped_kwargs.pop("fps", None)
non_video_scoped_kwargs.pop("num_frames", None)
# The context merges model defaults again when calling HF.
for key in self._get_per_video_kwargs({}):
non_video_scoped_kwargs[key] = None
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**non_video_mm_kwargs),
+24 -11
View File
@@ -1393,6 +1393,21 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
"""
return replace(cached_update, item_idx=new_item_idx)
def _get_mm_hashes(self, inputs: ProcessorInputs) -> MultiModalHashes:
"""Get each item's processor and encoder cache identity."""
return inputs.get_mm_hashes(
self.info.model_id,
self.info.ctx.get_mm_config().mm_hasher_algorithm,
)
def _get_cache_missing_processor_kwargs(
self,
inputs: ProcessorInputs,
mm_is_cached: MultiModalIsCached,
) -> Mapping[str, object]:
"""Select processor options for the items missing from the cache."""
return inputs.hf_processor_mm_kwargs
def _merge_mm_kwargs(
self,
cache: BaseMultiModalProcessorCache,
@@ -1464,10 +1479,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
# Use overrides if provided; fallback to data-dependent hashing.
with timing_ctx.record("get_mm_hashes"):
mm_hashes = inputs.get_mm_hashes(
self.info.model_id,
self.info.ctx.get_mm_config().mm_hasher_algorithm,
)
mm_hashes = self._get_mm_hashes(inputs)
mm_prompt_updates = self._get_mm_prompt_updates(
inputs.mm_data_items,
@@ -1500,10 +1512,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
return self._apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("get_mm_hashes"):
mm_hashes = inputs.get_mm_hashes(
self.info.model_id,
self.info.ctx.get_mm_config().mm_hasher_algorithm,
)
mm_hashes = self._get_mm_hashes(inputs)
with timing_ctx.record("get_cache_missing_items"):
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
@@ -1512,25 +1521,29 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
mm_hashes=mm_hashes,
)
missing_processor_kwargs = self._get_cache_missing_processor_kwargs(
inputs, mm_is_cached
)
# NOTE: The prompt does not correspond to `mm_missing_data_items`,
# so we can't apply prompt updates until the new multimodal
# items are combined with the cached multimodal items
with timing_ctx.record("apply_hf_processor"):
mm_missing_processed_data = self._apply_hf_processor_main(
mm_items=mm_missing_data_items,
hf_kwargs=inputs.hf_processor_mm_kwargs,
hf_kwargs=missing_processor_kwargs,
)
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
mm_missing_processed_data,
self._get_mm_fields_config(
mm_missing_processed_data, inputs.hf_processor_mm_kwargs
mm_missing_processed_data, missing_processor_kwargs
),
)
mm_missing_prompt_updates = self._get_mm_prompt_updates(
mm_missing_data_items,
inputs.hf_processor_mm_kwargs,
missing_processor_kwargs,
mm_missing_kwargs,
)