diff --git a/tests/models/multimodal/processing/test_whisper.py b/tests/models/multimodal/processing/test_whisper.py new file mode 100644 index 0000000000..1627b48914 --- /dev/null +++ b/tests/models/multimodal/processing/test_whisper.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for Whisper's multimodal preprocessing.""" + +import numpy as np +import pytest + +from vllm.multimodal import MULTIMODAL_REGISTRY + +from ...utils import build_model_context + + +@pytest.mark.parametrize("model_id", ["openai/whisper-large-v3-turbo"]) +@pytest.mark.parametrize("audio_duration_s", [5, 30, 35]) +def test_audio_features_fit_encoder_window( + model_id: str, + audio_duration_s: int, +) -> None: + """The encoder only has positions for one 30 s window, so a longer clip + has to be cut to that window like the HF feature extractor does by default. + """ + ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": 1}) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + feature_extractor = processor.info.get_feature_extractor() + + sampling_rate = feature_extractor.sampling_rate + rng = np.random.RandomState(0) + audio = rng.rand(sampling_rate * audio_duration_s).astype(np.float32) + + processed_inputs = processor( + "<|startoftranscript|>", + mm_items=processor.info.parse_mm_data({"audio": [(audio, sampling_rate)]}), + hf_processor_mm_kwargs={}, + ) + + input_features = processed_inputs["mm_kwargs"].get_data()["input_features"] + assert input_features.shape[-1] == feature_extractor.nb_max_frames diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index 59268619b0..f67e3f0cb9 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -767,6 +767,7 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo hf_kwargs=dict( hf_inputs.hf_kwargs, sampling_rate=feature_extractor.sampling_rate, + truncation=True, ) )