[Bugfix][Multimodal] Fix per-video cache option reuse (#55203)
Signed-off-by: bvolpato <brunocvcunha@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -15,6 +15,7 @@ from vllm.config import ModelConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.model_executor.models.qwen3_vl import Qwen3VLProcessingInfo
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.processing.context import BaseProcessingInfo
|
||||
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
@@ -336,65 +337,136 @@ def test_processor_kwargs_videos_kwargs_does_not_leak_into_image_budget(
|
||||
("hf_mm_kwargs", "scalar_kwargs"),
|
||||
[
|
||||
({"num_frames": [8, 16]}, [{"num_frames": 8}, {"num_frames": 16}]),
|
||||
({"fps": [4.0, 12.0]}, [{"fps": 4.0}, {"fps": 12.0}]),
|
||||
({"fps": [2.0, 4.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
|
||||
({"num_frames": [8, 16, 32]}, [{"num_frames": 8}, {"num_frames": 16}]),
|
||||
({"fps": [2.0, 4.0, 8.0]}, [{"fps": 2.0}, {"fps": 4.0}]),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_kwargs", [False, True])
|
||||
@pytest.mark.parametrize("use_uuids", [False, True])
|
||||
@pytest.mark.parametrize("scoped", [False, True])
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_processor_multi_video_list_kwargs(
|
||||
model_id: str,
|
||||
hf_mm_kwargs: dict[str, Any],
|
||||
scalar_kwargs: list[dict[str, Any]],
|
||||
model_kwargs: bool,
|
||||
use_uuids: bool,
|
||||
scoped: bool,
|
||||
) -> None:
|
||||
"""List-valued video kwargs must be scalarized per item without mutation.
|
||||
|
||||
Each video in the multi-video request must match processing the same input
|
||||
independently with its corresponding scalar ``fps``/``num_frames`` value.
|
||||
"""
|
||||
"""Per-video options must survive duplicate items and partial cache hits."""
|
||||
if scoped:
|
||||
hf_mm_kwargs = {"videos_kwargs": hf_mm_kwargs}
|
||||
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 2},
|
||||
mm_processor_kwargs=hf_mm_kwargs if model_kwargs else None,
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = (
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
)
|
||||
mm_data = {
|
||||
"video": [
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
]
|
||||
}
|
||||
|
||||
hf_mm_kwargs_before = deepcopy(hf_mm_kwargs)
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_mm_kwargs,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
|
||||
videos = [
|
||||
_build_video_mm_data(num_frames=32, original_fps=8.0)["video"][0]
|
||||
for _ in range(3)
|
||||
]
|
||||
for i, (frames, _) in enumerate(videos):
|
||||
frames.fill(i)
|
||||
|
||||
def process_single(hf_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
single_prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
single_mm_data = _build_video_mm_data(num_frames=32)
|
||||
single = processor(
|
||||
single_prompt,
|
||||
mm_items=processor.info.parse_mm_data(single_mm_data),
|
||||
mm_items=processor.info.parse_mm_data({"video": [videos[0]]}),
|
||||
hf_processor_mm_kwargs=hf_kwargs,
|
||||
)
|
||||
return single["mm_kwargs"].get_data()["video_grid_thw"][0]
|
||||
|
||||
expected_grids = [process_single(kwargs) for kwargs in scalar_kwargs]
|
||||
multi_grids = processed["mm_kwargs"].get_data()["video_grid_thw"]
|
||||
|
||||
assert not torch.equal(expected_grids[0], expected_grids[1])
|
||||
assert torch.equal(multi_grids[0], expected_grids[0])
|
||||
assert torch.equal(multi_grids[1], expected_grids[1])
|
||||
|
||||
def run(indices, *, cache=None, cached_only=False):
|
||||
return processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(
|
||||
{"video": [None if cached_only else videos[i] for i in indices]}
|
||||
),
|
||||
mm_uuid_items=(
|
||||
{"video": [f"video-{i}" for i in indices]} if use_uuids else None
|
||||
),
|
||||
hf_processor_mm_kwargs={} if model_kwargs else hf_mm_kwargs,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
for indices in ([0, 0], [0, 1], [0, 2], [2, 0]):
|
||||
expected = run(indices)
|
||||
actual = run(indices, cache=cache)
|
||||
assert actual["prompt_token_ids"] == expected["prompt_token_ids"]
|
||||
assert actual["mm_placeholders"] == expected["mm_placeholders"]
|
||||
assert actual["mm_kwargs"] == expected["mm_kwargs"]
|
||||
assert actual["mm_hashes"] == expected["mm_hashes"]
|
||||
assert len(set(actual["mm_hashes"]["video"])) == 2
|
||||
multi_grids = actual["mm_kwargs"].get_data()["video_grid_thw"]
|
||||
assert torch.equal(multi_grids[0], expected_grids[0])
|
||||
assert torch.equal(multi_grids[1], expected_grids[1])
|
||||
|
||||
if use_uuids:
|
||||
cached_only = run([2, 0], cache=cache, cached_only=True)
|
||||
assert cached_only["prompt_token_ids"] == actual["prompt_token_ids"]
|
||||
assert cached_only["mm_kwargs"] == actual["mm_kwargs"]
|
||||
assert hf_mm_kwargs == hf_mm_kwargs_before
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == 2, (
|
||||
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
|
||||
|
||||
@pytest.mark.parametrize(("key", "value"), [("fps", 2.0), ("num_frames", 8)])
|
||||
@pytest.mark.parametrize("length", [0, 1])
|
||||
@pytest.mark.parametrize("use_cache", [False, True])
|
||||
@pytest.mark.parametrize("scoped", [False, True])
|
||||
def test_processor_rejects_short_per_video_options(
|
||||
key: str, value: float | int, length: int, use_cache: bool, scoped: bool
|
||||
) -> None:
|
||||
"""Reject malformed option lists with and without a warm processor cache."""
|
||||
ctx = build_model_context(
|
||||
MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 0, "video": 2},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config) if use_cache else None
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * 2
|
||||
videos = _build_video_mm_data(num_frames=32, original_fps=8.0)["video"] * 2
|
||||
|
||||
def process(values: list[float | int]) -> None:
|
||||
kwargs = {key: values}
|
||||
processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"video": videos}),
|
||||
hf_processor_mm_kwargs={"videos_kwargs": kwargs} if scoped else kwargs,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
if use_cache:
|
||||
process([value, value])
|
||||
|
||||
with pytest.raises(ValueError, match=f"{key} must have at least 2 entries"):
|
||||
process([value] * length)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kwargs", [{"fps": [2.0, 4.0]}, {"num_frames": [8, 16]}])
|
||||
def test_processor_profiles_fewer_videos_than_configured_options(
|
||||
kwargs: dict[str, list[float] | list[int]],
|
||||
) -> None:
|
||||
"""Profiling may use fewer videos than the configured per-video list."""
|
||||
ctx = build_model_context(
|
||||
MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 0, "video": 2},
|
||||
mm_processor_kwargs=kwargs,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
profiled = processor.get_dummy_mm_inputs({"video": 1})
|
||||
|
||||
assert len(profiled["mm_placeholders"]["video"]) == 1
|
||||
assert profiled["mm_kwargs"].get_data()["video_grid_thw"].shape[0] == 1
|
||||
|
||||
|
||||
def _build_video_embeds_mm_data(
|
||||
|
||||
@@ -60,7 +60,7 @@ from vllm.config.multimodal import (
|
||||
VideoPruningMethod,
|
||||
)
|
||||
from vllm.distributed import get_pp_group, parallel_state
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalHashes
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
@@ -81,6 +81,7 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
@@ -94,6 +95,7 @@ from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
@@ -1342,6 +1344,67 @@ def _replace_video_token_placeholders(
|
||||
|
||||
|
||||
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
|
||||
def _get_per_video_kwargs(
|
||||
self, kwargs: Mapping[str, object], num_videos: int | None = None
|
||||
) -> dict[str, list[Any]]:
|
||||
merged = self.info._merge_and_resolve_mm_processor_kwargs(kwargs)
|
||||
per_video_kwargs = {
|
||||
key: value
|
||||
for key, value in merged.get("videos_kwargs", {}).items()
|
||||
if key in ("fps", "num_frames") and isinstance(value, list)
|
||||
}
|
||||
if num_videos:
|
||||
for key, values in per_video_kwargs.items():
|
||||
if len(values) < num_videos:
|
||||
raise ValueError(
|
||||
f"{key} must have at least {num_videos} entries "
|
||||
"(one per video); "
|
||||
f"got {len(values)}."
|
||||
)
|
||||
return per_video_kwargs
|
||||
|
||||
def _get_mm_hashes(self, inputs: ProcessorInputs) -> MultiModalHashes:
|
||||
hashes = dict(super()._get_mm_hashes(inputs))
|
||||
per_video_kwargs = self._get_per_video_kwargs(
|
||||
inputs.hf_processor_mm_kwargs, len(hashes.get("video", []))
|
||||
)
|
||||
if per_video_kwargs and "video" in hashes:
|
||||
algorithm = self.info.ctx.get_mm_config().mm_hasher_algorithm
|
||||
hashes["video"] = [
|
||||
MultiModalHasher.hash_kwargs(
|
||||
algorithm,
|
||||
mm_hash=mm_hash,
|
||||
**{key: values[i] for key, values in per_video_kwargs.items()},
|
||||
)
|
||||
for i, mm_hash in enumerate(hashes["video"])
|
||||
]
|
||||
return hashes
|
||||
|
||||
def _get_cache_missing_processor_kwargs(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
mm_is_cached: Mapping[str, list[bool]],
|
||||
) -> Mapping[str, object]:
|
||||
per_video_kwargs = self._get_per_video_kwargs(
|
||||
inputs.hf_processor_mm_kwargs, len(mm_is_cached.get("video", []))
|
||||
)
|
||||
if not per_video_kwargs or "video" not in mm_is_cached:
|
||||
return inputs.hf_processor_mm_kwargs
|
||||
kwargs = self.info._merge_and_resolve_mm_processor_kwargs(
|
||||
inputs.hf_processor_mm_kwargs
|
||||
)
|
||||
kwargs["videos_kwargs"].update(
|
||||
{
|
||||
key: [
|
||||
value
|
||||
for value, hit in zip(values, mm_is_cached["video"])
|
||||
if not hit
|
||||
]
|
||||
for key, values in per_video_kwargs.items()
|
||||
}
|
||||
)
|
||||
return kwargs
|
||||
|
||||
@staticmethod
|
||||
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
|
||||
"""Transformers>=5.10 processors override `replace_video_token`
|
||||
@@ -1412,6 +1475,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
video_input_ids_lst: list[list[int]] = []
|
||||
if videos := hf_data.pop("videos", []):
|
||||
assert isinstance(videos, list)
|
||||
self._get_per_video_kwargs(hf_kwargs, len(videos))
|
||||
video_grid_thw_lst = []
|
||||
pixel_values_videos_lst = []
|
||||
timestamps_per_video = []
|
||||
@@ -1544,6 +1608,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
|
||||
non_video_scoped_kwargs.pop("fps", None)
|
||||
non_video_scoped_kwargs.pop("num_frames", None)
|
||||
# The context merges model defaults again when calling HF.
|
||||
for key in self._get_per_video_kwargs({}):
|
||||
non_video_scoped_kwargs[key] = None
|
||||
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**non_video_mm_kwargs),
|
||||
|
||||
@@ -1393,6 +1393,21 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""
|
||||
return replace(cached_update, item_idx=new_item_idx)
|
||||
|
||||
def _get_mm_hashes(self, inputs: ProcessorInputs) -> MultiModalHashes:
|
||||
"""Get each item's processor and encoder cache identity."""
|
||||
return inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
def _get_cache_missing_processor_kwargs(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
mm_is_cached: MultiModalIsCached,
|
||||
) -> Mapping[str, object]:
|
||||
"""Select processor options for the items missing from the cache."""
|
||||
return inputs.hf_processor_mm_kwargs
|
||||
|
||||
def _merge_mm_kwargs(
|
||||
self,
|
||||
cache: BaseMultiModalProcessorCache,
|
||||
@@ -1464,10 +1479,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
mm_hashes = self._get_mm_hashes(inputs)
|
||||
|
||||
mm_prompt_updates = self._get_mm_prompt_updates(
|
||||
inputs.mm_data_items,
|
||||
@@ -1500,10 +1512,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
mm_hashes = self._get_mm_hashes(inputs)
|
||||
|
||||
with timing_ctx.record("get_cache_missing_items"):
|
||||
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
|
||||
@@ -1512,25 +1521,29 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_hashes=mm_hashes,
|
||||
)
|
||||
|
||||
missing_processor_kwargs = self._get_cache_missing_processor_kwargs(
|
||||
inputs, mm_is_cached
|
||||
)
|
||||
|
||||
# NOTE: The prompt does not correspond to `mm_missing_data_items`,
|
||||
# so we can't apply prompt updates until the new multimodal
|
||||
# items are combined with the cached multimodal items
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
mm_missing_processed_data = self._apply_hf_processor_main(
|
||||
mm_items=mm_missing_data_items,
|
||||
hf_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
hf_kwargs=missing_processor_kwargs,
|
||||
)
|
||||
|
||||
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
mm_missing_processed_data,
|
||||
self._get_mm_fields_config(
|
||||
mm_missing_processed_data, inputs.hf_processor_mm_kwargs
|
||||
mm_missing_processed_data, missing_processor_kwargs
|
||||
),
|
||||
)
|
||||
|
||||
mm_missing_prompt_updates = self._get_mm_prompt_updates(
|
||||
mm_missing_data_items,
|
||||
inputs.hf_processor_mm_kwargs,
|
||||
missing_processor_kwargs,
|
||||
mm_missing_kwargs,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user