[Bugfix][Qwen3-VL] Fix multi-video crash with list-valued fps/num_frames (#46305)
Signed-off-by: Ting Sun <suntcrick@gmail.com>
This commit is contained in:
@@ -138,3 +138,49 @@ def test_processor_multi_video(
|
||||
assert video_phs[i].offset >= prev_end, (
|
||||
f"Placeholder {i} overlaps with placeholder {i - 1}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"hf_mm_kwargs",
|
||||
[{"num_frames": [8, 16]}, {"fps": [2.0, 4.0]}],
|
||||
)
|
||||
def test_processor_multi_video_list_kwargs(
|
||||
model_id: str,
|
||||
hf_mm_kwargs: dict[str, Any],
|
||||
) -> None:
|
||||
"""Regression test: a multi-video request with list-valued per-video
|
||||
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
|
||||
crash.
|
||||
|
||||
Before the fix, ``_call_hf_processor`` copied the whole kwargs to every
|
||||
video without slicing, so ``_get_video_second_idx`` received the list
|
||||
where a scalar was expected and raised ``TypeError``.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": 2},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = (
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
)
|
||||
mm_data = {
|
||||
"video": [
|
||||
_build_video_mm_data(num_frames=16)["video"][0],
|
||||
_build_video_mm_data(num_frames=32)["video"][0],
|
||||
]
|
||||
}
|
||||
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=hf_mm_kwargs,
|
||||
)
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == 2, (
|
||||
f"Expected exactly 2 video placeholders, got {len(video_phs)}"
|
||||
)
|
||||
|
||||
@@ -1271,7 +1271,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
vision_end_token_id = hf_config.vision_end_token_id
|
||||
video_token_id = hf_config.video_token_id
|
||||
|
||||
for item in videos:
|
||||
for item_idx, item in enumerate(videos):
|
||||
video_array, metadata = item
|
||||
|
||||
# NOTE: @JJJYmmm new attr metadata.frames_indices indicates
|
||||
@@ -1282,6 +1282,12 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# NOTE: a copy of is created to update do_sample_frames,
|
||||
# otherwise mm_hash for the object will be incorrect.
|
||||
video_mm_kwargs = dict(**mm_kwargs)
|
||||
sampled_fps = video_mm_kwargs.get("fps")
|
||||
if is_list_of(sampled_fps, float):
|
||||
video_mm_kwargs["fps"] = sampled_fps[item_idx]
|
||||
sampled_num_frames = video_mm_kwargs.get("num_frames")
|
||||
if is_list_of(sampled_num_frames, int):
|
||||
video_mm_kwargs["num_frames"] = sampled_num_frames[item_idx]
|
||||
if "do_sample_frames" not in video_mm_kwargs:
|
||||
# qwen_vl_utils already has "do_sample_frames" in
|
||||
# mm_kwargs, don't overwrite it.
|
||||
@@ -1363,10 +1369,15 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
else:
|
||||
video_outputs = dict()
|
||||
|
||||
# fps/num_frames are video-only kwargs already consumed by the loop;
|
||||
# exclude them so the text/image processor call below never gets a list.
|
||||
non_video_mm_kwargs = {
|
||||
k: v for k, v in mm_kwargs.items() if k not in ("fps", "num_frames")
|
||||
}
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
mm_data=mm_data,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_kwargs=non_video_mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user