From 15d65f86694ac647b139179f88c9d7aea0820108 Mon Sep 17 00:00:00 2001 From: TobyJBell <51262478+TobyB1702@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:06:46 -0400 Subject: [PATCH] [Bugfix] Changed speech to text chunk timestamp to cumulative approach (#41131) Signed-off-by: Toby Bell --- docs/contributing/model/transcription.md | 2 +- .../test_speech_to_text_cancellation.py | 12 +++- .../test_chunk_timestamp_offset.py | 70 +++++++++++++++++++ .../test_transcription_inter_chunk_spacing.py | 4 +- .../speech_to_text/base/serving.py | 24 +++++-- 5 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py diff --git a/docs/contributing/model/transcription.md b/docs/contributing/model/transcription.md index b076ef84a4..f82bafa44d 100644 --- a/docs/contributing/model/transcription.md +++ b/docs/contributing/model/transcription.md @@ -195,7 +195,7 @@ Provide a fast duration→token estimate to improve streaming usage statistics: The API server takes care of basic audio I/O and optional chunking before building prompts: - Resampling: Input audio is resampled to `SpeechToTextConfig.sample_rate` using `AudioResampler`. -- Chunking: If `SpeechToTextConfig.allow_audio_chunking` is True and the duration exceeds `max_audio_clip_s`, the server splits the audio into overlapping chunks and generates a prompt per chunk. Overlap is controlled by `overlap_chunk_second`. +- Chunking: If `SpeechToTextConfig.allow_audio_chunking` is True and the duration exceeds `max_audio_clip_s`, the server splits the audio into chunks and generates a prompt per chunk. There is no overlap between chunks, overlap_chunk_second controls the size of the search window used to find the split point. - Energy-aware splitting: When `min_energy_split_window_size` is set, the server finds low-energy regions to minimize cutting within words. Relevant server logic: diff --git a/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py b/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py index 040fc1a48f..45772708d9 100644 --- a/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py +++ b/tests/entrypoints/speech_to_text/test_speech_to_text_cancellation.py @@ -53,7 +53,13 @@ async def test_non_streaming_cancel_aborts_engine_requests( server.asr_config = SimpleNamespace(max_audio_clip_s=30) server._check_model = AsyncMock(return_value=None) server._maybe_get_adapters = Mock(return_value=None) - server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 40.0)) + server._preprocess_speech_to_text = AsyncMock( + return_value=( + engine_inputs, + 40.0, + [30.0 * i for i in range(len(engine_inputs))], + ) + ) server._log_inputs = Mock() request = SimpleNamespace( @@ -122,7 +128,9 @@ async def test_non_streaming_cancel_advances_all_chunk_generators(): server.asr_config = SimpleNamespace(max_audio_clip_s=30) server._check_model = AsyncMock(return_value=None) server._maybe_get_adapters = Mock(return_value=None) - server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 90.0)) + server._preprocess_speech_to_text = AsyncMock( + return_value=(engine_inputs, 90.0, [0.0, 29.5, 29.5 + 29.7]) + ) server._log_inputs = Mock() request = SimpleNamespace( diff --git a/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py b/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py new file mode 100644 index 0000000000..22231e0d85 --- /dev/null +++ b/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regression test for chunk timestamp offset drift in _preprocess_speech_to_text.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import numpy as np +import pytest + +from vllm.config.speech_to_text import SpeechToTextConfig +from vllm.entrypoints.speech_to_text.base.serving import SpeechToTextBaseServing + +SR = 16_000 +_PATCH = "vllm.entrypoints.speech_to_text.base.serving" + + +@pytest.mark.asyncio +async def test_chunk_offsets_are_cumulative_not_nominal(): + """ + chunk_start_offsets must be cumulative actual chunk lengths, not the old approach of + 'idx * max_audio_clip_s'. When split_audio places a boundary before the + nominal 30 s mark, the old formula drifts; the fixed formula stays exact. + """ + # Chunks shorter than exactly 30 s, as split_audio produces when a quiet + # region falls inside the 1 s overlap window before the nominal boundary. + chunk_lengths = [int(29.5 * SR), int(29.7 * SR), int(5.0 * SR)] + chunks = [np.zeros(n, dtype=np.float32) for n in chunk_lengths] + + duration = sum(chunk_lengths) / SR + + expected_offsets = [0.0, 29.5, 29.5 + 29.7] # cumulative seconds + wrong_offsets = [0.0, 30.0, 60.0] # what the old bug produced + + serving = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing) + serving._decode_and_chunk_speech_async = AsyncMock(return_value=(chunks, duration)) + serving.asr_config = SpeechToTextConfig( + sample_rate=float(SR), + max_audio_clip_s=30, + overlap_chunk_second=1, + min_energy_split_window_size=1600, + ) + serving.max_audio_filesize_mb = 100.0 + serving.model_cls = MagicMock() + serving.model_cls.validate_language.side_effect = lambda lang: lang + serving.model_cls.supports_explicit_language_detection = False + serving.model_cls.get_generation_prompt.return_value = {} + serving.model_config = MagicMock() + serving.task_type = "transcribe" + serving.renderer = MagicMock() + serving.renderer.render_cmpl_async = AsyncMock( + return_value=[MagicMock()] * len(chunks) + ) + + request = MagicMock() + request.language = "en" + request.to_language = None + request.response_format = "json" + request.build_stt_params.return_value = MagicMock() + + with patch(f"{_PATCH}.parse_model_prompt", return_value=MagicMock()): + _, _, offsets = await serving._preprocess_speech_to_text( + request=request, + audio_data=b"\x00", + request_id="test", + ) + + assert offsets == pytest.approx(expected_offsets, abs=1e-6) + assert offsets != pytest.approx(wrong_offsets, abs=1e-6) diff --git a/tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py b/tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py index 7e51e5be44..8c99429af1 100644 --- a/tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py +++ b/tests/entrypoints/speech_to_text/transcription/test_transcription_inter_chunk_spacing.py @@ -342,7 +342,9 @@ async def test_create_transcription_non_streaming_joins_chunks_by_language(): models.lora_requests = {} models.is_base_model.return_value = True - preprocess_mock = AsyncMock(return_value=([MagicMock(), MagicMock()], 1.0)) + preprocess_mock = AsyncMock( + return_value=([MagicMock(), MagicMock()], 1.0, [0.0, 29.5]) + ) with ( patch( diff --git a/vllm/entrypoints/speech_to_text/base/serving.py b/vllm/entrypoints/speech_to_text/base/serving.py index 7c703f2535..c646f53383 100644 --- a/vllm/entrypoints/speech_to_text/base/serving.py +++ b/vllm/entrypoints/speech_to_text/base/serving.py @@ -258,7 +258,7 @@ class SpeechToTextBaseServing(GenerateBaseServing): request: SpeechToTextRequest, audio_data: bytes, request_id: str, - ) -> tuple[list[EngineInput], float]: + ) -> tuple[list[EngineInput], float, list[float]]: # Validate request request.language = self.model_cls.validate_language(request.language) request.to_language = ( @@ -277,6 +277,13 @@ class SpeechToTextBaseServing(GenerateBaseServing): # Run cpu intensive preprocess step in a separate thread pool executor. chunks, duration = await self._decode_and_chunk_speech_async(audio_data) + chunk_start_offsets: list[float] = [0.0] + + for chunk in chunks[:-1]: + chunk_start_offsets.append( + chunk_start_offsets[-1] + chunk.shape[-1] / self.asr_config.sample_rate + ) + if request.language is None and getattr( self.model_cls, "supports_explicit_language_detection", False ): @@ -306,7 +313,7 @@ class SpeechToTextBaseServing(GenerateBaseServing): engine_inputs = await self.renderer.render_cmpl_async(parsed_prompts) - return engine_inputs, duration + return engine_inputs, duration, chunk_start_offsets def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt): dec_prompt = prompt["decoder_prompt"] @@ -391,7 +398,7 @@ class SpeechToTextBaseServing(GenerateBaseServing): SpeechToTextSegment, segment_class( id=len(segments), - seek=start_time, + seek=int(start_time), start=start_time + BASE_OFFSET * start_timestamp, end=start_time + BASE_OFFSET * end_timestamp, temperature=request.temperature, @@ -467,7 +474,11 @@ class SpeechToTextBaseServing(GenerateBaseServing): lora_request = self._maybe_get_adapters(request) - engine_inputs, duration_s = await self._preprocess_speech_to_text( + ( + engine_inputs, + duration_s, + chunk_start_offsets, + ) = await self._preprocess_speech_to_text( request=request, audio_data=audio_data, request_id=request_id, @@ -587,11 +598,10 @@ class SpeechToTextBaseServing(GenerateBaseServing): assert len(list_result_generator) == 1, ( "`max_audio_clip_s` is set to None, audio cannot be chunked" ) + assert len(chunk_start_offsets) == len(list_result_generator) result_generator = merge_async_iterators(*list_result_generator) async for idx, op in result_generator: - start_time = ( - float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0 - ) + start_time = chunk_start_offsets[idx] if request.response_format == "verbose_json": assert op.outputs[0].logprobs segments: list[SpeechToTextSegment] = self._get_verbose_segments(