From 1e47ec00d2e6ad911969508068b72f7d1aae1968 Mon Sep 17 00:00:00 2001 From: Abhinav Verma Date: Tue, 15 Sep 2026 20:27:32 +0530 Subject: [PATCH] [Bugfix][Gemma 4] Don't read fft_length when profiling unified audio (#56721) Signed-off-by: Abhinav Verma Co-authored-by: Claude Opus 5 --- .../processing/test_gemma4_unified.py | 44 ++++++++++++++++++- vllm/model_executor/models/gemma4_mm.py | 25 ++++++++++- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/tests/models/multimodal/processing/test_gemma4_unified.py b/tests/models/multimodal/processing/test_gemma4_unified.py index 67a81ddb7b..fe544e2338 100644 --- a/tests/models/multimodal/processing/test_gemma4_unified.py +++ b/tests/models/multimodal/processing/test_gemma4_unified.py @@ -6,9 +6,13 @@ from collections.abc import Mapping import pytest import torch from PIL import Image as PILImage +from transformers.models.gemma4 import Gemma4AudioFeatureExtractor from vllm.exceptions import VLLMValidationError -from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs +from vllm.model_executor.models.gemma4_mm import ( + Gemma4ImagePixelInputs, + _dummy_audio_num_samples, +) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalFieldConfig @@ -204,3 +208,41 @@ def test_limit_mm_per_prompt( mm_items=processor.info.parse_mm_data(mm_data), hf_processor_mm_kwargs={}, ) + + +def test_dummy_audio_length_falls_back_for_the_unified_extractor(): + """The unified audio feature extractor does not define ``fft_length``. + + ``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute + unconditionally, so memory profiling raised ``AttributeError`` for every + unified Gemma 4 deployment in which audio is the modality with the largest + per-item token count -- which is the case as soon as ``audio_seq_length`` + is raised above the video budget. + """ + + class _UnifiedFeatureExtractor: + sampling_rate = 16000 + + class _UnifiedProcessor: + feature_extractor = _UnifiedFeatureExtractor() + audio_seq_length = 3000 + audio_ms_per_token = 40 + + # 3000 tokens * 40 ms = 120 s at 16 kHz + assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000 + + +def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor(): + """The mel/tower extractor keeps its existing behaviour.""" + + class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor): + def __init__(self): # noqa: D107 - bypass the real feature-extractor init + self.fft_length = 512 + self.sampling_rate = 16000 + + class _TowerProcessor: + feature_extractor = _TowerFeatureExtractor() + audio_seq_length = 750 + audio_ms_per_token = 40 + + assert _dummy_audio_num_samples(_TowerProcessor()) == 512 diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index 2dddc72b21..2fe2752475 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -24,6 +24,7 @@ from PIL import Image as PILImage from torch import nn from transformers import AutoModel, BatchFeature from transformers.models.gemma4 import ( + Gemma4AudioFeatureExtractor, Gemma4Config, Gemma4Processor, Gemma4VisionConfig, @@ -456,6 +457,28 @@ class Gemma4ProcessingInfo(BaseProcessingInfo): # --------------------------------------------------------------------------- +def _dummy_audio_num_samples(processor: Any) -> int: + """Length in samples of a worst-case dummy audio item. + + ``fft_length`` is set in ``Gemma4AudioFeatureExtractor.__init__`` and is not + defined by ``Gemma4UnifiedAudioFeatureExtractor``, which the encoder-free + Gemma 4 variant uses. Reading it unconditionally therefore raises + ``AttributeError`` during memory profiling on that variant, which is reached + whenever audio is the modality with the largest per-item token count. + + For any other extractor, fall back to the audio budget the processor itself + declares -- the same pair of values used to warn about over-long audio + elsewhere in this file. + """ + feature_extractor = processor.feature_extractor + if isinstance(feature_extractor, Gemma4AudioFeatureExtractor): + return feature_extractor.fft_length + + sampling_rate = feature_extractor.sampling_rate + max_duration_s = processor.audio_seq_length * processor.audio_ms_per_token / 1000.0 + return int(max_duration_s * sampling_rate) + + class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: num_images = mm_counts.get("image", 0) @@ -508,7 +531,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): } if num_audios > 0: - audio_len = processor.feature_extractor.fft_length + audio_len = _dummy_audio_num_samples(processor) data["audio"] = self._get_dummy_audios( length=audio_len, num_audios=num_audios,