[Bugfix][Multimodal] Keep Gemma 4 video frame counts on CPU (#52441)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>

tests/models/multimodal/generation/test_vit_cudagraph.py

@@ -286,6 +286,12 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
             "<bos><start_of_turn>user\n<|video|>\nDescribe this video in one sentence."
             "<end_of_turn>\n<start_of_turn>model\n"
         ),
+        # The 16-frame test video produces 1056 vision tokens. Capture only
+        # the smallest supported bucket that covers it instead of all default
+        # buckets through max_model_len, which adds unrelated memory pressure.
+        compilation_config_overrides={
+            "encoder_cudagraph_token_budgets": [1120],
+        },
         needs_video_metadata=True,
         marks=[pytest.mark.core_model],
     ),

vllm/model_executor/models/gemma4_mm.py

@@ -801,7 +801,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
                     MultiModalFieldConfig.flat_from_sizes("video", vfc)
                 ),
                 video_frame_counts=MultiModalFieldConfig.batched(
-                    "video",
+                    "video", keep_on_cpu=True
                 ),
                 video_num_soft_tokens=MultiModalFieldConfig.batched(
                     "video", keep_on_cpu=True