From 61f084b5f3e58aa3d3eb5c86aeaf4686c460c478 Mon Sep 17 00:00:00 2001 From: Bruno Volpato Date: Sun, 11 Oct 2026 02:21:55 -0400 Subject: [PATCH] [Bugfix][Multimodal] Fix per-video cache option reuse (#55203) Signed-off-by: bvolpato Co-authored-by: Codex --- .../multimodal/processing/test_qwen3_vl.py | 138 +++++++++++++----- vllm/model_executor/models/qwen3_vl.py | 69 ++++++++- vllm/multimodal/processing/processor.py | 35 +++-- 3 files changed, 197 insertions(+), 45 deletions(-) diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index 2428109815..0c74a86ff0 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -15,6 +15,7 @@ from vllm.config import ModelConfig from vllm.config.multimodal import MultiModalDummyOptions from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.cache import MultiModalProcessorOnlyCache from vllm.multimodal.processing.context import BaseProcessingInfo from ...registry import HF_EXAMPLE_MODELS @@ -336,65 +337,136 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget( ("hf_mm_kwargs", "scalar_kwargs"), [ ({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]), - ({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]), + ({"fps": [2.0, 4.0]}, [{"fps": 2.0}, {"fps": 4.0}]), + ({"num_frames": [8, 16, 32]}, [{"num_frames": 8}, {"num_frames": 16}]), + ({"fps": [2.0, 4.0, 8.0]}, [{"fps": 2.0}, {"fps": 4.0}]), ], ) +@pytest.mark.parametrize("model_kwargs", [False, True]) +@pytest.mark.parametrize("use_uuids", [False, True]) +@pytest.mark.parametrize("scoped", [False, True]) +@pytest.mark.skip_global_cleanup def test_processor_multi_video_list_kwargs( model_id: str, hf_mm_kwargs: dict[str, Any], scalar_kwargs: list[dict[str, Any]], + model_kwargs: bool, + use_uuids: bool, + scoped: bool, ) -> None: - """List-valued video kwargs must be scalarized per item without mutation. - - Each video in the multi-video request must match processing the same input - independently with its corresponding scalar ``fps``/``num_frames`` value. - """ + """Per-video options must survive duplicate items and partial cache hits.""" + if scoped: + hf_mm_kwargs = {"videos_kwargs": hf_mm_kwargs} + hf_mm_kwargs_before = deepcopy(hf_mm_kwargs) ctx = build_model_context( model_id, limit_mm_per_prompt={"image": 0, "video": 2}, + mm_processor_kwargs=hf_mm_kwargs if model_kwargs else None, + mm_processor_cache_gb=1, ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) - - prompt = ( - "<|vision_start|><|video_pad|><|vision_end|>" - "<|vision_start|><|video_pad|><|vision_end|>" - ) - mm_data = { - "video": [ - _build_video_mm_data(num_frames=32)["video"][0], - _build_video_mm_data(num_frames=32)["video"][0], - ] - } - - hf_mm_kwargs_before = deepcopy(hf_mm_kwargs) - processed = processor( - prompt, - mm_items=processor.info.parse_mm_data(mm_data), - hf_processor_mm_kwargs=hf_mm_kwargs, - ) + cache = MultiModalProcessorOnlyCache(ctx.model_config) + prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2 + videos = [ + _build_video_mm_data(num_frames=32, original_fps=8.0)["video"][0] + for _ in range(3) + ] + for i, (frames, _) in enumerate(videos): + frames.fill(i) def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor: single_prompt = "<|vision_start|><|video_pad|><|vision_end|>" - single_mm_data = _build_video_mm_data(num_frames=32) single = processor( single_prompt, - mm_items=processor.info.parse_mm_data(single_mm_data), + mm_items=processor.info.parse_mm_data({"video": [videos[0]]}), hf_processor_mm_kwargs=hf_kwargs, ) return single["mm_kwargs"].get_data()["video_grid_thw"][0] expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs] - multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"] - assert not torch.equal(expected_grids[0], expected_grids[1]) - assert torch.equal(multi_grids[0], expected_grids[0]) - assert torch.equal(multi_grids[1], expected_grids[1]) + + def run(indices, *, cache=None, cached_only=False): + return processor( + prompt, + mm_items=processor.info.parse_mm_data( + {"video": [None if cached_only else videos[i] for i in indices]} + ), + mm_uuid_items=( + {"video": [f"video-{i}" for i in indices]} if use_uuids else None + ), + hf_processor_mm_kwargs={} if model_kwargs else hf_mm_kwargs, + cache=cache, + ) + + for indices in ([0, 0], [0, 1], [0, 2], [2, 0]): + expected = run(indices) + actual = run(indices, cache=cache) + assert actual["prompt_token_ids"] == expected["prompt_token_ids"] + assert actual["mm_placeholders"] == expected["mm_placeholders"] + assert actual["mm_kwargs"] == expected["mm_kwargs"] + assert actual["mm_hashes"] == expected["mm_hashes"] + assert len(set(actual["mm_hashes"]["video"])) == 2 + multi_grids = actual["mm_kwargs"].get_data()["video_grid_thw"] + assert torch.equal(multi_grids[0], expected_grids[0]) + assert torch.equal(multi_grids[1], expected_grids[1]) + + if use_uuids: + cached_only = run([2, 0], cache=cache, cached_only=True) + assert cached_only["prompt_token_ids"] == actual["prompt_token_ids"] + assert cached_only["mm_kwargs"] == actual["mm_kwargs"] assert hf_mm_kwargs == hf_mm_kwargs_before - video_phs = processed["mm_placeholders"].get("video", []) - assert len(video_phs) == 2, ( - f"Expected exactly 2 video placeholders, got {len(video_phs)}" + +@pytest.mark.parametrize(("key", "value"), [("fps", 2.0), ("num_frames", 8)]) +@pytest.mark.parametrize("length", [0, 1]) +@pytest.mark.parametrize("use_cache", [False, True]) +@pytest.mark.parametrize("scoped", [False, True]) +def test_processor_rejects_short_per_video_options( + key: str, value: float | int, length: int, use_cache: bool, scoped: bool +) -> None: + """Reject malformed option lists with and without a warm processor cache.""" + ctx = build_model_context( + MODEL_ID, + limit_mm_per_prompt={"image": 0, "video": 2}, ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + cache = MultiModalProcessorOnlyCache(ctx.model_config) if use_cache else None + prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2 + videos = _build_video_mm_data(num_frames=32, original_fps=8.0)["video"] * 2 + + def process(values: list[float | int]) -> None: + kwargs = {key: values} + processor( + prompt, + mm_items=processor.info.parse_mm_data({"video": videos}), + hf_processor_mm_kwargs={"videos_kwargs": kwargs} if scoped else kwargs, + cache=cache, + ) + + if use_cache: + process([value, value]) + + with pytest.raises(ValueError, match=f"{key} must have at least 2 entries"): + process([value] * length) + + +@pytest.mark.parametrize("kwargs", [{"fps": [2.0, 4.0]}, {"num_frames": [8, 16]}]) +def test_processor_profiles_fewer_videos_than_configured_options( + kwargs: dict[str, list[float] | list[int]], +) -> None: + """Profiling may use fewer videos than the configured per-video list.""" + ctx = build_model_context( + MODEL_ID, + limit_mm_per_prompt={"image": 0, "video": 2}, + mm_processor_kwargs=kwargs, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + profiled = processor.get_dummy_mm_inputs({"video": 1}) + + assert len(profiled["mm_placeholders"]["video"]) == 1 + assert profiled["mm_kwargs"].get_data()["video_grid_thw"].shape[0] == 1 def _build_video_embeds_mm_data( diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 77780b3d1d..527e2b0f3e 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -60,7 +60,7 @@ from vllm.config.multimodal import ( VideoPruningMethod, ) from vllm.distributed import get_pp_group, parallel_state -from vllm.inputs import MultiModalDataDict +from vllm.inputs import MultiModalDataDict, MultiModalHashes from vllm.logger import init_logger from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY from vllm.model_executor.layers.attention.mm_encoder_attention import ( @@ -81,6 +81,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.hasher import MultiModalHasher from vllm.multimodal.inputs import ( MultiModalFeatureSpec, MultiModalFieldConfig, @@ -94,6 +95,7 @@ from vllm.multimodal.parse import ImageSize, MultiModalDataItems from vllm.multimodal.processing import ( BaseDummyInputsBuilder, BaseMultiModalProcessor, + ProcessorInputs, PromptReplacement, PromptUpdate, PromptUpdateDetails, @@ -1342,6 +1344,67 @@ def _replace_video_token_placeholders( class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]): + def _get_per_video_kwargs( + self, kwargs: Mapping[str, object], num_videos: int | None = None + ) -> dict[str, list[Any]]: + merged = self.info._merge_and_resolve_mm_processor_kwargs(kwargs) + per_video_kwargs = { + key: value + for key, value in merged.get("videos_kwargs", {}).items() + if key in ("fps", "num_frames") and isinstance(value, list) + } + if num_videos: + for key, values in per_video_kwargs.items(): + if len(values) < num_videos: + raise ValueError( + f"{key} must have at least {num_videos} entries " + "(one per video); " + f"got {len(values)}." + ) + return per_video_kwargs + + def _get_mm_hashes(self, inputs: ProcessorInputs) -> MultiModalHashes: + hashes = dict(super()._get_mm_hashes(inputs)) + per_video_kwargs = self._get_per_video_kwargs( + inputs.hf_processor_mm_kwargs, len(hashes.get("video", [])) + ) + if per_video_kwargs and "video" in hashes: + algorithm = self.info.ctx.get_mm_config().mm_hasher_algorithm + hashes["video"] = [ + MultiModalHasher.hash_kwargs( + algorithm, + mm_hash=mm_hash, + **{key: values[i] for key, values in per_video_kwargs.items()}, + ) + for i, mm_hash in enumerate(hashes["video"]) + ] + return hashes + + def _get_cache_missing_processor_kwargs( + self, + inputs: ProcessorInputs, + mm_is_cached: Mapping[str, list[bool]], + ) -> Mapping[str, object]: + per_video_kwargs = self._get_per_video_kwargs( + inputs.hf_processor_mm_kwargs, len(mm_is_cached.get("video", [])) + ) + if not per_video_kwargs or "video" not in mm_is_cached: + return inputs.hf_processor_mm_kwargs + kwargs = self.info._merge_and_resolve_mm_processor_kwargs( + inputs.hf_processor_mm_kwargs + ) + kwargs["videos_kwargs"].update( + { + key: [ + value + for value, hit in zip(values, mm_is_cached["video"]) + if not hit + ] + for key, values in per_video_kwargs.items() + } + ) + return kwargs + @staticmethod def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool: """Transformers>=5.10 processors override `replace_video_token` @@ -1412,6 +1475,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) video_input_ids_lst: list[list[int]] = [] if videos := hf_data.pop("videos", []): assert isinstance(videos, list) + self._get_per_video_kwargs(hf_kwargs, len(videos)) video_grid_thw_lst = [] pixel_values_videos_lst = [] timestamps_per_video = [] @@ -1544,6 +1608,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"]) non_video_scoped_kwargs.pop("fps", None) non_video_scoped_kwargs.pop("num_frames", None) + # The context merges model defaults again when calling HF. + for key in self._get_per_video_kwargs({}): + non_video_scoped_kwargs[key] = None non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs processed_data = self.info.ctx.call_hf_processor( self.info.get_hf_processor(**non_video_mm_kwargs), diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index abdb585402..5b46267dff 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -1393,6 +1393,21 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): """ return replace(cached_update, item_idx=new_item_idx) + def _get_mm_hashes(self, inputs: ProcessorInputs) -> MultiModalHashes: + """Get each item's processor and encoder cache identity.""" + return inputs.get_mm_hashes( + self.info.model_id, + self.info.ctx.get_mm_config().mm_hasher_algorithm, + ) + + def _get_cache_missing_processor_kwargs( + self, + inputs: ProcessorInputs, + mm_is_cached: MultiModalIsCached, + ) -> Mapping[str, object]: + """Select processor options for the items missing from the cache.""" + return inputs.hf_processor_mm_kwargs + def _merge_mm_kwargs( self, cache: BaseMultiModalProcessorCache, @@ -1464,10 +1479,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): # Use overrides if provided; fallback to data-dependent hashing. with timing_ctx.record("get_mm_hashes"): - mm_hashes = inputs.get_mm_hashes( - self.info.model_id, - self.info.ctx.get_mm_config().mm_hasher_algorithm, - ) + mm_hashes = self._get_mm_hashes(inputs) mm_prompt_updates = self._get_mm_prompt_updates( inputs.mm_data_items, @@ -1500,10 +1512,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): return self._apply_hf_processor(inputs, timing_ctx) with timing_ctx.record("get_mm_hashes"): - mm_hashes = inputs.get_mm_hashes( - self.info.model_id, - self.info.ctx.get_mm_config().mm_hasher_algorithm, - ) + mm_hashes = self._get_mm_hashes(inputs) with timing_ctx.record("get_cache_missing_items"): mm_is_cached, mm_missing_data_items = self._get_cache_missing_items( @@ -1512,25 +1521,29 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): mm_hashes=mm_hashes, ) + missing_processor_kwargs = self._get_cache_missing_processor_kwargs( + inputs, mm_is_cached + ) + # NOTE: The prompt does not correspond to `mm_missing_data_items`, # so we can't apply prompt updates until the new multimodal # items are combined with the cached multimodal items with timing_ctx.record("apply_hf_processor"): mm_missing_processed_data = self._apply_hf_processor_main( mm_items=mm_missing_data_items, - hf_kwargs=inputs.hf_processor_mm_kwargs, + hf_kwargs=missing_processor_kwargs, ) mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs( mm_missing_processed_data, self._get_mm_fields_config( - mm_missing_processed_data, inputs.hf_processor_mm_kwargs + mm_missing_processed_data, missing_processor_kwargs ), ) mm_missing_prompt_updates = self._get_mm_prompt_updates( mm_missing_data_items, - inputs.hf_processor_mm_kwargs, + missing_processor_kwargs, mm_missing_kwargs, )