[Bugfix][Gemma 4] Don't read fft_length when profiling unified audio (#56721)
Signed-off-by: Abhinav Verma <verma.abhinav275@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
f0fbc98c48
commit
1e47ec00d2
@@ -6,9 +6,13 @@ from collections.abc import Mapping
|
|||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
from PIL import Image as PILImage
|
from PIL import Image as PILImage
|
||||||
|
from transformers.models.gemma4 import Gemma4AudioFeatureExtractor
|
||||||
|
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.model_executor.models.gemma4_mm import Gemma4ImagePixelInputs
|
from vllm.model_executor.models.gemma4_mm import (
|
||||||
|
Gemma4ImagePixelInputs,
|
||||||
|
_dummy_audio_num_samples,
|
||||||
|
)
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||||
|
|
||||||
@@ -204,3 +208,41 @@ def test_limit_mm_per_prompt(
|
|||||||
mm_items=processor.info.parse_mm_data(mm_data),
|
mm_items=processor.info.parse_mm_data(mm_data),
|
||||||
hf_processor_mm_kwargs={},
|
hf_processor_mm_kwargs={},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_dummy_audio_length_falls_back_for_the_unified_extractor():
|
||||||
|
"""The unified audio feature extractor does not define ``fft_length``.
|
||||||
|
|
||||||
|
``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute
|
||||||
|
unconditionally, so memory profiling raised ``AttributeError`` for every
|
||||||
|
unified Gemma 4 deployment in which audio is the modality with the largest
|
||||||
|
per-item token count -- which is the case as soon as ``audio_seq_length``
|
||||||
|
is raised above the video budget.
|
||||||
|
"""
|
||||||
|
|
||||||
|
class _UnifiedFeatureExtractor:
|
||||||
|
sampling_rate = 16000
|
||||||
|
|
||||||
|
class _UnifiedProcessor:
|
||||||
|
feature_extractor = _UnifiedFeatureExtractor()
|
||||||
|
audio_seq_length = 3000
|
||||||
|
audio_ms_per_token = 40
|
||||||
|
|
||||||
|
# 3000 tokens * 40 ms = 120 s at 16 kHz
|
||||||
|
assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000
|
||||||
|
|
||||||
|
|
||||||
|
def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor():
|
||||||
|
"""The mel/tower extractor keeps its existing behaviour."""
|
||||||
|
|
||||||
|
class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor):
|
||||||
|
def __init__(self): # noqa: D107 - bypass the real feature-extractor init
|
||||||
|
self.fft_length = 512
|
||||||
|
self.sampling_rate = 16000
|
||||||
|
|
||||||
|
class _TowerProcessor:
|
||||||
|
feature_extractor = _TowerFeatureExtractor()
|
||||||
|
audio_seq_length = 750
|
||||||
|
audio_ms_per_token = 40
|
||||||
|
|
||||||
|
assert _dummy_audio_num_samples(_TowerProcessor()) == 512
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ from PIL import Image as PILImage
|
|||||||
from torch import nn
|
from torch import nn
|
||||||
from transformers import AutoModel, BatchFeature
|
from transformers import AutoModel, BatchFeature
|
||||||
from transformers.models.gemma4 import (
|
from transformers.models.gemma4 import (
|
||||||
|
Gemma4AudioFeatureExtractor,
|
||||||
Gemma4Config,
|
Gemma4Config,
|
||||||
Gemma4Processor,
|
Gemma4Processor,
|
||||||
Gemma4VisionConfig,
|
Gemma4VisionConfig,
|
||||||
@@ -456,6 +457,28 @@ class Gemma4ProcessingInfo(BaseProcessingInfo):
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _dummy_audio_num_samples(processor: Any) -> int:
|
||||||
|
"""Length in samples of a worst-case dummy audio item.
|
||||||
|
|
||||||
|
``fft_length`` is set in ``Gemma4AudioFeatureExtractor.__init__`` and is not
|
||||||
|
defined by ``Gemma4UnifiedAudioFeatureExtractor``, which the encoder-free
|
||||||
|
Gemma 4 variant uses. Reading it unconditionally therefore raises
|
||||||
|
``AttributeError`` during memory profiling on that variant, which is reached
|
||||||
|
whenever audio is the modality with the largest per-item token count.
|
||||||
|
|
||||||
|
For any other extractor, fall back to the audio budget the processor itself
|
||||||
|
declares -- the same pair of values used to warn about over-long audio
|
||||||
|
elsewhere in this file.
|
||||||
|
"""
|
||||||
|
feature_extractor = processor.feature_extractor
|
||||||
|
if isinstance(feature_extractor, Gemma4AudioFeatureExtractor):
|
||||||
|
return feature_extractor.fft_length
|
||||||
|
|
||||||
|
sampling_rate = feature_extractor.sampling_rate
|
||||||
|
max_duration_s = processor.audio_seq_length * processor.audio_ms_per_token / 1000.0
|
||||||
|
return int(max_duration_s * sampling_rate)
|
||||||
|
|
||||||
|
|
||||||
class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
|
class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
|
||||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||||
num_images = mm_counts.get("image", 0)
|
num_images = mm_counts.get("image", 0)
|
||||||
@@ -508,7 +531,7 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
|
|||||||
}
|
}
|
||||||
|
|
||||||
if num_audios > 0:
|
if num_audios > 0:
|
||||||
audio_len = processor.feature_extractor.fft_length
|
audio_len = _dummy_audio_num_samples(processor)
|
||||||
data["audio"] = self._get_dummy_audios(
|
data["audio"] = self._get_dummy_audios(
|
||||||
length=audio_len,
|
length=audio_len,
|
||||||
num_audios=num_audios,
|
num_audios=num_audios,
|
||||||
|
|||||||
Reference in New Issue
Block a user