[Model] Honor cap_pixels_per_frame in Qwen3-VL memory profiling (#54380)
Signed-off-by: Devon Krisman <git@krisman.dev> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
b383e16396
commit
4f78a8fdd0
@@ -184,3 +184,46 @@ def test_processor_multi_video_list_kwargs(
|
||||
assert len(video_phs) == 2, (
|
||||
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None:
|
||||
"""Regression test for memory profiling with ``cap_pixels_per_frame``.
|
||||
|
||||
With the HF per-frame pixel cap enabled (transformers#48071) and a
|
||||
raised video budget, a 2-frame profiling dummy would be processed at
|
||||
only 2 * cap pixels, underestimating the largest possible video (a
|
||||
fully sampled one still fills the whole ``longest_edge`` budget). The
|
||||
dummy builder must spread the budget over enough frames that the cap
|
||||
is not binding.
|
||||
"""
|
||||
# 16 capped frames' worth of budget on top of the default per-frame
|
||||
# ceiling (max_video_tokens=768 at patch 16, merge 2 -> 786,432
|
||||
# pixels per frame).
|
||||
per_frame_cap = 768 * (16 * 2) ** 2
|
||||
budget = per_frame_cap * 16
|
||||
size = {"longest_edge": budget, "shortest_edge": 4096}
|
||||
|
||||
capped_ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"size": size, "cap_pixels_per_frame": True},
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
capped = MULTIMODAL_REGISTRY.create_processor(capped_ctx.model_config)
|
||||
capped_dummy = capped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {})
|
||||
capped_frames = capped_dummy["video"][0][0].shape[0]
|
||||
assert capped_frames == 16, (
|
||||
f"Expected the dummy to spread the budget over 16 frames, got {capped_frames}"
|
||||
)
|
||||
|
||||
uncapped_ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"size": size},
|
||||
limit_mm_per_prompt={"image": 0, "video": 1},
|
||||
)
|
||||
uncapped = MULTIMODAL_REGISTRY.create_processor(uncapped_ctx.model_config)
|
||||
uncapped_dummy = uncapped.dummy_inputs.get_dummy_mm_data(1024, {"video": 1}, {})
|
||||
uncapped_frames = uncapped_dummy["video"][0][0].shape[0]
|
||||
assert uncapped_frames == 2, (
|
||||
f"Expected the uncapped dummy to keep 2 frames, got {uncapped_frames}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user