[Bugfix][GLM4V] Fix video dummy profiling and memory usage (#48729)

Signed-off-by: xiaoming <1259730330@qq.com>

tests/models/multimodal/processing/test_glm4_1v.py

@@ -1,9 +1,15 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
+from unittest.mock import Mock
+
 import pytest
 
 from vllm.assets.video import VideoAsset
+from vllm.model_executor.models.glm4_1v import (
+    Glm4vForConditionalGeneration,
+    Glm4vProcessingInfo,
+)
 from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.multimodal.inputs import batched_tensors_equal
 from vllm.multimodal.video import DynamicVideoBackend, VideoBackend
@@ -11,6 +17,52 @@ from vllm.multimodal.video import DynamicVideoBackend, VideoBackend
 from ...utils import build_model_context
 
 
+@pytest.mark.parametrize(
+    (
+        "max_video_pixels",
+        "max_tokens",
+        "expected_num_frames",
+    ),
+    [
+        (47_040_000, 124_988, 11),
+        (47_040_000, 30_000, 24),
+        (100_352_000, 124_988, 21),
+        (100_352_000, 30_000, 7),
+        (100_352_000, 0, 1),
+    ],
+)
+def test_get_max_video_frames_matches_glm_resize(
+    max_video_pixels: int,
+    max_tokens: int,
+    expected_num_frames: int,
+):
+    info = Mock(spec=Glm4vProcessingInfo)
+    info.get_image_size_with_most_features.return_value = (2184, 2184)
+    info._get_video_max_pixels.return_value = max_video_pixels
+    vision_config = info.get_hf_config.return_value.vision_config
+    vision_config.patch_size = 14
+    vision_config.spatial_merge_size = 2
+    vision_config.temporal_patch_size = 2
+    info._get_vision_info.side_effect = lambda **kwargs: (
+        Glm4vProcessingInfo._get_vision_info(info, **kwargs)
+    )
+
+    num_frames = Glm4vProcessingInfo._get_max_video_frames(
+        info,
+        max_tokens=max_tokens,
+    )
+
+    assert num_frames == expected_num_frames
+    assert info._get_video_max_pixels.call_count == 1
+    assert info._get_vision_info.call_count == 600
+
+
+def test_encoder_cudagraph_uses_model_video_frame_limit():
+    model = Mock()
+
+    assert Glm4vForConditionalGeneration.get_max_frames_per_video(model) == 600
+
+
 @pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
 @pytest.mark.parametrize("expected_toks_per_frame", [299])
 @pytest.mark.parametrize(

vllm/model_executor/models/glm4_1v.py

@@ -40,6 +40,7 @@ import transformers
 from einops import rearrange
 from packaging.version import Version
 from transformers import BatchFeature, Glm4vProcessor
+from transformers.image_processing_base import ImageProcessingMixin
 from transformers.models.glm4v.configuration_glm4v import (
     Glm4vTextConfig,
     Glm4vVisionConfig,
@@ -49,6 +50,7 @@ from transformers.models.glm4v.image_processing_glm4v import (
     smart_resize,
 )
 from transformers.models.glm4v.video_processing_glm4v import Glm4vVideoProcessor
+from transformers.video_processing_utils import BaseVideoProcessor
 from transformers.video_utils import VideoMetadata
 
 from vllm.config import VllmConfig
@@ -94,6 +96,8 @@ from vllm.multimodal.processing import (
     PromptUpdateDetails,
 )
 from vllm.sequence import IntermediateTensors
+from vllm.transformers_utils.processor import get_processor_cls_name_from_config
+from vllm.transformers_utils.utils import convert_model_repo_to_path
 from vllm.utils.tensor_schema import TensorSchema, TensorShape
 from vllm.v1.attention.backends.registry import AttentionBackendEnum
 from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
@@ -982,11 +986,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
         return self.get_hf_processor(**kwargs).video_processor
 
     def _get_processor_class_name(self) -> str | None:
-        from vllm.transformers_utils.processor import (
-            get_processor_cls_name_from_config,
-        )
-        from vllm.transformers_utils.utils import convert_model_repo_to_path
-
         return get_processor_cls_name_from_config(
             convert_model_repo_to_path(self.ctx.model_config.model),
             revision=self.ctx.model_config.revision,
@@ -1109,8 +1108,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
 
         image_processor_config = self.ctx.get_hf_image_processor_config()
         if not image_processor_config.get("size"):
-            from transformers.image_processing_base import ImageProcessingMixin
-
             image_processor_config, _ = ImageProcessingMixin.get_image_processor_dict(
                 self.ctx.model_config.model,
                 revision=self.ctx.model_config.revision,
@@ -1122,8 +1119,6 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
         return self._get_longest_edge(size, "GLM4V image processor size")
 
     def _get_video_max_pixels(self) -> int:
-        from transformers.video_processing_utils import BaseVideoProcessor
-
         mm_kwargs = self.ctx.get_merged_mm_kwargs({})
         if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
             return int(override_max_pixels)
@@ -1175,39 +1170,29 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
             image_height=target_height,
         )
 
-    def get_num_video_tokens(
-        self,
-        *,
-        image_width: int,
-        image_height: int,
-        num_frames: int,
-    ) -> int:
-        _, num_video_tokens = self._get_vision_info(
-            image_width=image_width,
-            image_height=image_height,
-            num_frames=num_frames,
-            max_image_pixels=28 * 28 * 2 * 30000,
-        )
-        return num_video_tokens
-
     def _get_max_video_frames(self, max_tokens: int) -> int:
         target_width, target_height = self.get_image_size_with_most_features()
 
-        num_frames = 0
+        max_video_pixels = self._get_video_max_pixels()
+        num_frames_with_most_features = 1
+        max_vision_tokens = 0
 
-        while True:
-            next_num_frames = num_frames + 1
-            next_max_tokens = self.get_num_video_tokens(
+        for num_frames in range(1, _MAX_FRAMES_PER_VIDEO + 1):
+            _, num_vision_tokens = self._get_vision_info(
                 image_width=target_width,
                 image_height=target_height,
-                num_frames=next_num_frames,
+                num_frames=num_frames,
+                max_image_pixels=max_video_pixels,
             )
-            if next_max_tokens > max_tokens or next_max_tokens == 0:
-                break
 
-            num_frames = next_num_frames
+            if (
+                0 < num_vision_tokens <= max_tokens
+                and num_vision_tokens > max_vision_tokens
+            ):
+                num_frames_with_most_features = num_frames
+                max_vision_tokens = num_vision_tokens
 
-        return num_frames
+        return num_frames_with_most_features
 
     def get_num_frames_with_most_features(
         self,
@@ -1215,15 +1200,9 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
         mm_counts: Mapping[str, int],
     ) -> int:
         max_images = mm_counts.get("image", 0)
-        max_videos = mm_counts.get("video", 0)
 
         max_image_tokens = self.get_max_image_tokens() * max_images
-        max_total_frames = self._get_max_video_frames(seq_len - max_image_tokens)
-        max_frames_per_video = min(
-            max_total_frames // max(max_videos, 1), _MAX_FRAMES_PER_VIDEO
-        )
-
-        return max(max_frames_per_video, 1)
+        return self._get_max_video_frames(seq_len - max_image_tokens)
 
     def _get_video_second_idx_glm4v(
         self, metadata: dict[str, Any], total_frames: int
@@ -2001,15 +1980,7 @@ class Glm4vForConditionalGeneration(
         raise AssertionError("This line should be unreachable.")
 
     def get_max_frames_per_video(self) -> int:
-        mm_registry = MULTIMODAL_REGISTRY
-        info = mm_registry.get_processing_info(self.model_config)
-        max_frames_per_video = info.get_num_frames_with_most_features(
-            seq_len=self.model_config.max_model_len,
-            mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
-        )
-        # Small 'max_frames_per_video' will cause 'tensor mismatch' in PR#43403
-        # 16 is the default 'num_frames' of '_get_vision_info'
-        return max(max_frames_per_video, 16)
+        return _MAX_FRAMES_PER_VIDEO
 
     def get_encoder_cudagraph_budget_range(
         self,