[Bugfix][Multimodal] Fix video temporal padding estimates (#49030)

Signed-off-by: xiaoming <1259730330@qq.com>
This commit is contained in:
labAxiaoming
2026-07-29 01:57:31 +08:00
committed by GitHub
parent 8a7b3c2990
commit 1db989bbf1
7 changed files with 34 additions and 9 deletions
@@ -63,6 +63,31 @@ def test_encoder_cudagraph_uses_model_video_frame_limit():
assert Glm4vForConditionalGeneration.get_max_frames_per_video(model) == 600
@pytest.mark.parametrize(
("temporal_patch_size", "expected_grid_t"),
[(2, 9), (4, 5), (8, 3)],
)
def test_vision_info_rounds_up_temporal_frames(
temporal_patch_size: int,
expected_grid_t: int,
):
info = Mock(spec=Glm4vProcessingInfo)
vision_config = info.get_hf_config.return_value.vision_config
vision_config.patch_size = 14
vision_config.spatial_merge_size = 2
vision_config.temporal_patch_size = temporal_patch_size
_, num_vision_tokens = Glm4vProcessingInfo._get_vision_info(
info,
image_width=28,
image_height=28,
num_frames=17,
do_resize=False,
)
assert num_vision_tokens == expected_grid_t
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
@pytest.mark.parametrize("expected_toks_per_frame", [299])
@pytest.mark.parametrize(
+2 -2
View File
@@ -1082,8 +1082,8 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
preprocessed_size = ImageSize(width=image_width, height=image_height)
# NOTE: Frames are padded to be divisible by `temporal_patch_size`
# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294
padded_num_frames = num_frames + num_frames % temporal_patch_size
# https://github.com/huggingface/transformers/blob/v5.13.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L249-L252
padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
grid_t = max(padded_num_frames // temporal_patch_size, 1)
grid_h = preprocessed_size.height // patch_size
+2 -2
View File
@@ -409,8 +409,8 @@ class KananaVProcessingInfo(BaseProcessingInfo):
preprocessed_size = ImageSize(width=image_width, height=image_height)
# NOTE: Frames are padded to be divisible by `temporal_patch_size`
# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294
padded_num_frames = num_frames + num_frames % temporal_patch_size
# https://github.com/huggingface/transformers/blob/v5.13.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L249-L252
padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
grid_t = max(padded_num_frames // temporal_patch_size, 1)
grid_h = preprocessed_size.height // patch_size
+1 -1
View File
@@ -983,7 +983,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
else:
preprocessed_size = ImageSize(width=image_width, height=image_height)
padded_num_frames = num_frames + num_frames % temporal_patch_size
padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
grid_t = max(padded_num_frames // temporal_patch_size, 1)
grid_h = preprocessed_size.height // patch_size
@@ -1345,7 +1345,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
preprocessed = ImageSize(width=rw, height=rh)
else:
preprocessed = ImageSize(width=image_width, height=image_height)
padded_frames = num_frames + num_frames % temporal_patch_size
padded_frames = num_frames + (-num_frames % temporal_patch_size)
grid_t = max(padded_frames // temporal_patch_size, 1)
grid_h = preprocessed.height // patch_size
grid_w = preprocessed.width // patch_size
+1 -1
View File
@@ -715,7 +715,7 @@ class MiMoV2OmniProcessingInfo(BaseProcessingInfo):
effective_frames = num_frames * tokens_per_second
else:
effective_frames = num_frames
padded_num_frames = effective_frames + effective_frames % temporal_patch_size
padded_num_frames = effective_frames + (-effective_frames % temporal_patch_size)
grid_t = max(padded_num_frames // temporal_patch_size, 1)
grid_h = preprocessed_size.height // patch_size
grid_w = preprocessed_size.width // patch_size
+2 -2
View File
@@ -898,8 +898,8 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
preprocessed_size = ImageSize(width=image_width, height=image_height)
# NOTE: Frames are padded to be divisible by `temporal_patch_size`
# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294
padded_num_frames = num_frames + num_frames % temporal_patch_size
# https://github.com/huggingface/transformers/blob/v5.13.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L249-L252
padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
grid_t = max(padded_num_frames // temporal_patch_size, 1)
grid_h = preprocessed_size.height // patch_size