diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 58e6027468..387049f7cc 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -421,6 +421,7 @@ full_transcription = " ".join(transcriptions) The `split_audio` function: +- Expects 1D mono audio (`load_audio` downmixes by default) - Splits audio at quiet points to avoid cutting through speech - Uses RMS energy to find low-amplitude regions within the overlap window - Preserves all audio samples (no data loss) diff --git a/tests/multimodal/test_audio.py b/tests/multimodal/test_audio.py index 40d0197a41..a4a4a94767 100644 --- a/tests/multimodal/test_audio.py +++ b/tests/multimodal/test_audio.py @@ -744,6 +744,23 @@ class TestAudioChunking: # from start_idx, so the quietest scanned window starts at 19200. assert split_idx == 19200 + def test_split_audio_rejects_multi_channel(self): + """Chunking is mono-only; stereo must fail loudly rather than silently. + + find_split_point searches axis 0, so (channels, time) input would skip + the energy search entirely and split at fixed boundaries instead. + """ + stereo = np.ones((2, 16000 * 65), dtype=np.float32) + + with pytest.raises(ValueError, match="expects mono audio"): + split_audio( + audio_data=stereo, + sample_rate=16000, + max_clip_duration_s=30.0, + overlap_duration_s=1.0, + min_energy_window_size=1600, + ) + def test_split_audio_preserves_boundaries(self): """Verify first and last samples are preserved when chunking.""" diff --git a/vllm/entrypoints/speech_to_text/base/serving.py b/vllm/entrypoints/speech_to_text/base/serving.py index e98816a132..8065c13d43 100644 --- a/vllm/entrypoints/speech_to_text/base/serving.py +++ b/vllm/entrypoints/speech_to_text/base/serving.py @@ -178,6 +178,7 @@ class SpeechToTextBaseServing(GenerateBaseServing): y, sr = load_audio( buf, sr=self.asr_config.sample_rate, + mono=True, max_duration_s=self.max_audio_decode_duration_s, ) except ValueError: diff --git a/vllm/multimodal/audio.py b/vllm/multimodal/audio.py index e9470c6e02..50834efe5c 100644 --- a/vllm/multimodal/audio.py +++ b/vllm/multimodal/audio.py @@ -336,8 +336,8 @@ def split_audio( for splitting. Args: - audio_data: Audio array to split. Can be 1D (mono) or multi-dimensional. - Splits along the last dimension (time axis). + audio_data: 1D mono audio array to split. ASR models consume mono, so + callers must downmix before chunking. sample_rate: Sample rate of the audio in Hz. max_clip_duration_s: Maximum duration of each chunk in seconds. overlap_duration_s: Overlap duration in seconds between consecutive chunks. @@ -345,8 +345,10 @@ def split_audio( min_energy_window_size: Window size in samples for finding low-energy regions. Returns: - List of audio chunks. Each chunk is a numpy array with the same shape - as the input except for the last (time) dimension. + List of 1D audio chunks. + + Raises: + AssertionError: If ``audio_data`` is not 1D. Example: >>> audio = np.random.randn(1040000) # 65 seconds at 16kHz @@ -360,6 +362,11 @@ def split_audio( >>> len(chunks) 3 """ + if audio_data.ndim > 1: + raise ValueError( + f"split_audio expects mono audio, got shape {audio_data.shape}" + ) + chunk_size = int(sample_rate * max_clip_duration_s) overlap_size = int(sample_rate * overlap_duration_s) chunks = [] @@ -402,7 +409,7 @@ def find_split_point( RMS energy in sliding windows. Args: - wav: Audio array. Can be 1D or multi-dimensional. + wav: 1D mono audio array. start_idx: Start index of search region (inclusive). end_idx: End index of search region (exclusive). min_energy_window: Window size in samples for energy calculation.