diff --git a/tests/models/multimodal/processing/test_glm4_1v.py b/tests/models/multimodal/processing/test_glm4_1v.py index 5798c56634..0cafa261a3 100644 --- a/tests/models/multimodal/processing/test_glm4_1v.py +++ b/tests/models/multimodal/processing/test_glm4_1v.py @@ -1,9 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from unittest.mock import Mock + import pytest from vllm.assets.video import VideoAsset +from vllm.model_executor.models.glm4_1v import ( + Glm4vForConditionalGeneration, + Glm4vProcessingInfo, +) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import batched_tensors_equal from vllm.multimodal.video import DynamicVideoBackend, VideoBackend @@ -11,6 +17,52 @@ from vllm.multimodal.video import DynamicVideoBackend, VideoBackend from ...utils import build_model_context +@pytest.mark.parametrize( + ( + "max_video_pixels", + "max_tokens", + "expected_num_frames", + ), + [ + (47_040_000, 124_988, 11), + (47_040_000, 30_000, 24), + (100_352_000, 124_988, 21), + (100_352_000, 30_000, 7), + (100_352_000, 0, 1), + ], +) +def test_get_max_video_frames_matches_glm_resize( + max_video_pixels: int, + max_tokens: int, + expected_num_frames: int, +): + info = Mock(spec=Glm4vProcessingInfo) + info.get_image_size_with_most_features.return_value = (2184, 2184) + info._get_video_max_pixels.return_value = max_video_pixels + vision_config = info.get_hf_config.return_value.vision_config + vision_config.patch_size = 14 + vision_config.spatial_merge_size = 2 + vision_config.temporal_patch_size = 2 + info._get_vision_info.side_effect = lambda **kwargs: ( + Glm4vProcessingInfo._get_vision_info(info, **kwargs) + ) + + num_frames = Glm4vProcessingInfo._get_max_video_frames( + info, + max_tokens=max_tokens, + ) + + assert num_frames == expected_num_frames + assert info._get_video_max_pixels.call_count == 1 + assert info._get_vision_info.call_count == 600 + + +def test_encoder_cudagraph_uses_model_video_frame_limit(): + model = Mock() + + assert Glm4vForConditionalGeneration.get_max_frames_per_video(model) == 600 + + @pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"]) @pytest.mark.parametrize("expected_toks_per_frame", [299]) @pytest.mark.parametrize( diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index d64a4ef513..810d9de87b 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -40,6 +40,7 @@ import transformers from einops import rearrange from packaging.version import Version from transformers import BatchFeature, Glm4vProcessor +from transformers.image_processing_base import ImageProcessingMixin from transformers.models.glm4v.configuration_glm4v import ( Glm4vTextConfig, Glm4vVisionConfig, @@ -49,6 +50,7 @@ from transformers.models.glm4v.image_processing_glm4v import ( smart_resize, ) from transformers.models.glm4v.video_processing_glm4v import Glm4vVideoProcessor +from transformers.video_processing_utils import BaseVideoProcessor from transformers.video_utils import VideoMetadata from vllm.config import VllmConfig @@ -94,6 +96,8 @@ from vllm.multimodal.processing import ( PromptUpdateDetails, ) from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.processor import get_processor_cls_name_from_config +from vllm.transformers_utils.utils import convert_model_repo_to_path from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers @@ -982,11 +986,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo): return self.get_hf_processor(**kwargs).video_processor def _get_processor_class_name(self) -> str | None: - from vllm.transformers_utils.processor import ( - get_processor_cls_name_from_config, - ) - from vllm.transformers_utils.utils import convert_model_repo_to_path - return get_processor_cls_name_from_config( convert_model_repo_to_path(self.ctx.model_config.model), revision=self.ctx.model_config.revision, @@ -1109,8 +1108,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo): image_processor_config = self.ctx.get_hf_image_processor_config() if not image_processor_config.get("size"): - from transformers.image_processing_base import ImageProcessingMixin - image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict( self.ctx.model_config.model, revision=self.ctx.model_config.revision, @@ -1122,8 +1119,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo): return self._get_longest_edge(size, "GLM4V image processor size") def _get_video_max_pixels(self) -> int: - from transformers.video_processing_utils import BaseVideoProcessor - mm_kwargs = self.ctx.get_merged_mm_kwargs({}) if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None: return int(override_max_pixels) @@ -1175,39 +1170,29 @@ class Glm4vProcessingInfo(BaseProcessingInfo): image_height=target_height, ) - def get_num_video_tokens( - self, - *, - image_width: int, - image_height: int, - num_frames: int, - ) -> int: - _, num_video_tokens = self._get_vision_info( - image_width=image_width, - image_height=image_height, - num_frames=num_frames, - max_image_pixels=28 * 28 * 2 * 30000, - ) - return num_video_tokens - def _get_max_video_frames(self, max_tokens: int) -> int: target_width, target_height = self.get_image_size_with_most_features() - num_frames = 0 + max_video_pixels = self._get_video_max_pixels() + num_frames_with_most_features = 1 + max_vision_tokens = 0 - while True: - next_num_frames = num_frames + 1 - next_max_tokens = self.get_num_video_tokens( + for num_frames in range(1, _MAX_FRAMES_PER_VIDEO + 1): + _, num_vision_tokens = self._get_vision_info( image_width=target_width, image_height=target_height, - num_frames=next_num_frames, + num_frames=num_frames, + max_image_pixels=max_video_pixels, ) - if next_max_tokens > max_tokens or next_max_tokens == 0: - break - num_frames = next_num_frames + if ( + 0 < num_vision_tokens <= max_tokens + and num_vision_tokens > max_vision_tokens + ): + num_frames_with_most_features = num_frames + max_vision_tokens = num_vision_tokens - return num_frames + return num_frames_with_most_features def get_num_frames_with_most_features( self, @@ -1215,15 +1200,9 @@ class Glm4vProcessingInfo(BaseProcessingInfo): mm_counts: Mapping[str, int], ) -> int: max_images = mm_counts.get("image", 0) - max_videos = mm_counts.get("video", 0) max_image_tokens = self.get_max_image_tokens() * max_images - max_total_frames = self._get_max_video_frames(seq_len - max_image_tokens) - max_frames_per_video = min( - max_total_frames // max(max_videos, 1), _MAX_FRAMES_PER_VIDEO - ) - - return max(max_frames_per_video, 1) + return self._get_max_video_frames(seq_len - max_image_tokens) def _get_video_second_idx_glm4v( self, metadata: dict[str, Any], total_frames: int @@ -2001,15 +1980,7 @@ class Glm4vForConditionalGeneration( raise AssertionError("This line should be unreachable.") def get_max_frames_per_video(self) -> int: - mm_registry = MULTIMODAL_REGISTRY - info = mm_registry.get_processing_info(self.model_config) - max_frames_per_video = info.get_num_frames_with_most_features( - seq_len=self.model_config.max_model_len, - mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")}, - ) - # Small 'max_frames_per_video' will cause 'tensor mismatch' in PR#43403 - # 16 is the default 'num_frames' of '_get_vision_info' - return max(max_frames_per_video, 16) + return _MAX_FRAMES_PER_VIDEO def get_encoder_cudagraph_budget_range( self,