diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index 2465a528ef..6f92e71dd4 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -184,3 +184,46 @@ def test_processor_multi_video_list_kwargs( assert len(video_phs) == 2, ( f"Expected exactly 2 video placeholders, got {len(video_phs)}" ) + + +@pytest.mark.parametrize("model_id", [MODEL_ID]) +def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None: + """Regression test for memory profiling with ``cap_pixels_per_frame``. + + With the HF per-frame pixel cap enabled (transformers#48071) and a + raised video budget, a 2-frame profiling dummy would be processed at + only 2 * cap pixels, underestimating the largest possible video (a + fully sampled one still fills the whole ``longest_edge`` budget). The + dummy builder must spread the budget over enough frames that the cap + is not binding. + """ + # 16 capped frames' worth of budget on top of the default per-frame + # ceiling (max_video_tokens=768 at patch 16, merge 2 -> 786,432 + # pixels per frame). + per_frame_cap = 768 * (16 * 2) ** 2 + budget = per_frame_cap * 16 + size = {"longest_edge": budget, "shortest_edge": 4096} + + capped_ctx = build_model_context( + model_id, + mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True}, + limit_mm_per_prompt={"image": 0, "video": 1}, + ) + capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config) + capped_dummy = capped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {}) + capped_frames = capped_dummy["video"][0][0].shape[0] + assert capped_frames == 16, ( + f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}" + ) + + uncapped_ctx = build_model_context( + model_id, + mm_processor_kwargs={"size": size}, + limit_mm_per_prompt={"image": 0, "video": 1}, + ) + uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config) + uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {}) + uncapped_frames = uncapped_dummy["video"][0][0].shape[0] + assert uncapped_frames == 2, ( + f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}" + ) diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index d0fad8fcdb..0aade09076 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -112,7 +112,7 @@ from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.triton_utils import HAS_TRITON, tl, triton from vllm.utils.cache import LRUCache from vllm.utils.collection_utils import is_list_of -from vllm.utils.math_utils import round_up +from vllm.utils.math_utils import cdiv, round_up from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers @@ -1146,6 +1146,28 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): # video_max_pixels contains the temporal compression factor, # so we divide by 2 to get the maximum number of image pixels. video_max_pixels = video_size["longest_edge"] + + # With the HF processor's per-frame pixel cap enabled + # (cap_pixels_per_frame, huggingface/transformers#48071), a 2-frame + # dummy is processed at only 2 * cap pixels, so memory profiling + # underestimates the largest possible item: a fully sampled video + # still reaches the whole longest_edge budget. Spread the budget + # over enough frames that the cap is not binding for the dummy. + # This takes precedence over a shrinking num_frames override, which + # would otherwise reintroduce the under-profiling. + if mm_kwargs.get("cap_pixels_per_frame"): + max_video_tokens = mm_kwargs.get( + "max_video_tokens", + getattr(video_processor, "max_video_tokens", 768), + ) + patch_size = mm_kwargs.get("patch_size", video_processor.patch_size) + merge_size = mm_kwargs.get("merge_size", video_processor.merge_size) + per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2 + target_num_frames = max( + target_num_frames, + min(video_processor.max_frames, cdiv(video_max_pixels, per_frame_cap)), + ) + target_video_width, target_video_height = ( self.info.get_image_size_with_most_features( max_pixels=video_max_pixels // temporal_patch_size