[Bugfix][Gemma 4] Don't read fft_length when profiling unified audio (#56721)

Signed-off-by: Abhinav Verma <verma.abhinav275@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Abhinav Verma
2026-09-15 14:57:32 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent f0fbc98c48
commit 1e47ec00d2
2 changed files with 67 additions and 2 deletions
@@ -6,9 +6,13 @@ from collections.abc import Mapping
import pytest import pytest
import torch import torch
from PIL import Image as PILImage from PIL import Image as PILImage
from transformers.models.gemma4 import Gemma4AudioFeatureExtractor
from vllm.exceptions import VLLMValidationError from vllm.exceptions import VLLMValidationError
from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs from vllm.model_executor.models.gemma4_mm import (
Gemma4ImagePixelInputs,
_dummy_audio_num_samples,
)
from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig from vllm.multimodal.inputs import MultiModalFieldConfig
@@ -204,3 +208,41 @@ def test_limit_mm_per_prompt(
mm_items=processor.info.parse_mm_data(mm_data), mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={}, hf_processor_mm_kwargs={},
) )
def test_dummy_audio_length_falls_back_for_the_unified_extractor():
"""The unified audio feature extractor does not define ``fft_length``.
``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute
unconditionally, so memory profiling raised ``AttributeError`` for every
unified Gemma 4 deployment in which audio is the modality with the largest
per-item token count -- which is the case as soon as ``audio_seq_length``
is raised above the video budget.
"""
class _UnifiedFeatureExtractor:
sampling_rate = 16000
class _UnifiedProcessor:
feature_extractor = _UnifiedFeatureExtractor()
audio_seq_length = 3000
audio_ms_per_token = 40
# 3000 tokens * 40 ms = 120 s at 16 kHz
assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000
def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor():
"""The mel/tower extractor keeps its existing behaviour."""
class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor):
def __init__(self): # noqa: D107 - bypass the real feature-extractor init
self.fft_length = 512
self.sampling_rate = 16000
class _TowerProcessor:
feature_extractor = _TowerFeatureExtractor()
audio_seq_length = 750
audio_ms_per_token = 40
assert _dummy_audio_num_samples(_TowerProcessor()) == 512
+24 -1
View File
@@ -24,6 +24,7 @@ from PIL import Image as PILImage
from torch import nn from torch import nn
from transformers import AutoModel, BatchFeature from transformers import AutoModel, BatchFeature
from transformers.models.gemma4 import ( from transformers.models.gemma4 import (
Gemma4AudioFeatureExtractor,
Gemma4Config, Gemma4Config,
Gemma4Processor, Gemma4Processor,
Gemma4VisionConfig, Gemma4VisionConfig,
@@ -456,6 +457,28 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def _dummy_audio_num_samples(processor: Any) -> int:
"""Length in samples of a worst-case dummy audio item.
``fft_length`` is set in ``Gemma4AudioFeatureExtractor.__init__`` and is not
defined by ``Gemma4UnifiedAudioFeatureExtractor``, which the encoder-free
Gemma 4 variant uses. Reading it unconditionally therefore raises
``AttributeError`` during memory profiling on that variant, which is reached
whenever audio is the modality with the largest per-item token count.
For any other extractor, fall back to the audio budget the processor itself
declares -- the same pair of values used to warn about over-long audio
elsewhere in this file.
"""
feature_extractor = processor.feature_extractor
if isinstance(feature_extractor, Gemma4AudioFeatureExtractor):
return feature_extractor.fft_length
sampling_rate = feature_extractor.sampling_rate
max_duration_s = processor.audio_seq_length * processor.audio_ms_per_token / 1000.0
return int(max_duration_s * sampling_rate)
class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]): class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0) num_images = mm_counts.get("image", 0)
@@ -508,7 +531,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
} }
if num_audios > 0: if num_audios > 0:
audio_len = processor.feature_extractor.fft_length audio_len = _dummy_audio_num_samples(processor)
data["audio"] = self._get_dummy_audios( data["audio"] = self._get_dummy_audios(
length=audio_len, length=audio_len,
num_audios=num_audios, num_audios=num_audios,