[Bugfix] Changed speech to text chunk timestamp to cumulative approach (#41131)

Signed-off-by: Toby Bell <toby.bell1702@hotmail.co.uk>
This commit is contained in:
TobyJBell
2026-07-27 17:06:46 +00:00
committed by GitHub
parent e3c2fc3b3c
commit 15d65f8669
5 changed files with 101 additions and 11 deletions
+1 -1
View File
@@ -195,7 +195,7 @@ Provide a fast duration→token estimate to improve streaming usage statistics:
The API server takes care of basic audio I/O and optional chunking before building prompts:
- Resampling: Input audio is resampled to `SpeechToTextConfig.sample_rate` using `AudioResampler`.
- Chunking: If `SpeechToTextConfig.allow_audio_chunking` is True and the duration exceeds `max_audio_clip_s`, the server splits the audio into overlapping chunks and generates a prompt per chunk. Overlap is controlled by `overlap_chunk_second`.
- Chunking: If `SpeechToTextConfig.allow_audio_chunking` is True and the duration exceeds `max_audio_clip_s`, the server splits the audio into chunks and generates a prompt per chunk. There is no overlap between chunks, overlap_chunk_second controls the size of the search window used to find the split point.
- Energy-aware splitting: When `min_energy_split_window_size` is set, the server finds low-energy regions to minimize cutting within words.
Relevant server logic:
@@ -53,7 +53,13 @@ async def test_non_streaming_cancel_aborts_engine_requests(
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
server._check_model = AsyncMock(return_value=None)
server._maybe_get_adapters = Mock(return_value=None)
server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 40.0))
server._preprocess_speech_to_text = AsyncMock(
return_value=(
engine_inputs,
40.0,
[30.0 * i for i in range(len(engine_inputs))],
)
)
server._log_inputs = Mock()
request = SimpleNamespace(
@@ -122,7 +128,9 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
server._check_model = AsyncMock(return_value=None)
server._maybe_get_adapters = Mock(return_value=None)
server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 90.0))
server._preprocess_speech_to_text = AsyncMock(
return_value=(engine_inputs, 90.0, [0.0, 29.5, 29.5 + 29.7])
)
server._log_inputs = Mock()
request = SimpleNamespace(
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for chunk timestamp offset drift in _preprocess_speech_to_text."""
from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock, patch
import numpy as np
import pytest
from vllm.config.speech_to_text import SpeechToTextConfig
from vllm.entrypoints.speech_to_text.base.serving import SpeechToTextBaseServing
SR = 16_000
_PATCH = "vllm.entrypoints.speech_to_text.base.serving"
@pytest.mark.asyncio
async def test_chunk_offsets_are_cumulative_not_nominal():
"""
chunk_start_offsets must be cumulative actual chunk lengths, not the old approach of
'idx * max_audio_clip_s'. When split_audio places a boundary before the
nominal 30 s mark, the old formula drifts; the fixed formula stays exact.
"""
# Chunks shorter than exactly 30 s, as split_audio produces when a quiet
# region falls inside the 1 s overlap window before the nominal boundary.
chunk_lengths = [int(29.5 * SR), int(29.7 * SR), int(5.0 * SR)]
chunks = [np.zeros(n, dtype=np.float32) for n in chunk_lengths]
duration = sum(chunk_lengths) / SR
expected_offsets = [0.0, 29.5, 29.5 + 29.7] # cumulative seconds
wrong_offsets = [0.0, 30.0, 60.0] # what the old bug produced
serving = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
serving._decode_and_chunk_speech_async = AsyncMock(return_value=(chunks, duration))
serving.asr_config = SpeechToTextConfig(
sample_rate=float(SR),
max_audio_clip_s=30,
overlap_chunk_second=1,
min_energy_split_window_size=1600,
)
serving.max_audio_filesize_mb = 100.0
serving.model_cls = MagicMock()
serving.model_cls.validate_language.side_effect = lambda lang: lang
serving.model_cls.supports_explicit_language_detection = False
serving.model_cls.get_generation_prompt.return_value = {}
serving.model_config = MagicMock()
serving.task_type = "transcribe"
serving.renderer = MagicMock()
serving.renderer.render_cmpl_async = AsyncMock(
return_value=[MagicMock()] * len(chunks)
)
request = MagicMock()
request.language = "en"
request.to_language = None
request.response_format = "json"
request.build_stt_params.return_value = MagicMock()
with patch(f"{_PATCH}.parse_model_prompt", return_value=MagicMock()):
_, _, offsets = await serving._preprocess_speech_to_text(
request=request,
audio_data=b"\x00",
request_id="test",
)
assert offsets == pytest.approx(expected_offsets, abs=1e-6)
assert offsets != pytest.approx(wrong_offsets, abs=1e-6)
@@ -342,7 +342,9 @@ async def test_create_transcription_non_streaming_joins_chunks_by_language():
models.lora_requests = {}
models.is_base_model.return_value = True
preprocess_mock = AsyncMock(return_value=([MagicMock(), MagicMock()], 1.0))
preprocess_mock = AsyncMock(
return_value=([MagicMock(), MagicMock()], 1.0, [0.0, 29.5])
)
with (
patch(
@@ -258,7 +258,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
request: SpeechToTextRequest,
audio_data: bytes,
request_id: str,
) -> tuple[list[EngineInput], float]:
) -> tuple[list[EngineInput], float, list[float]]:
# Validate request
request.language = self.model_cls.validate_language(request.language)
request.to_language = (
@@ -277,6 +277,13 @@ class SpeechToTextBaseServing(GenerateBaseServing):
# Run cpu intensive preprocess step in a separate thread pool executor.
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
chunk_start_offsets: list[float] = [0.0]
for chunk in chunks[:-1]:
chunk_start_offsets.append(
chunk_start_offsets[-1] + chunk.shape[-1] / self.asr_config.sample_rate
)
if request.language is None and getattr(
self.model_cls, "supports_explicit_language_detection", False
):
@@ -306,7 +313,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
engine_inputs = await self.renderer.render_cmpl_async(parsed_prompts)
return engine_inputs, duration
return engine_inputs, duration, chunk_start_offsets
def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):
dec_prompt = prompt["decoder_prompt"]
@@ -391,7 +398,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
SpeechToTextSegment,
segment_class(
id=len(segments),
seek=start_time,
seek=int(start_time),
start=start_time + BASE_OFFSET * start_timestamp,
end=start_time + BASE_OFFSET * end_timestamp,
temperature=request.temperature,
@@ -467,7 +474,11 @@ class SpeechToTextBaseServing(GenerateBaseServing):
lora_request = self._maybe_get_adapters(request)
engine_inputs, duration_s = await self._preprocess_speech_to_text(
(
engine_inputs,
duration_s,
chunk_start_offsets,
) = await self._preprocess_speech_to_text(
request=request,
audio_data=audio_data,
request_id=request_id,
@@ -587,11 +598,10 @@ class SpeechToTextBaseServing(GenerateBaseServing):
assert len(list_result_generator) == 1, (
"`max_audio_clip_s` is set to None, audio cannot be chunked"
)
assert len(chunk_start_offsets) == len(list_result_generator)
result_generator = merge_async_iterators(*list_result_generator)
async for idx, op in result_generator:
start_time = (
float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0
)
start_time = chunk_start_offsets[idx]
if request.response_format == "verbose_json":
assert op.outputs[0].logprobs
segments: list[SpeechToTextSegment] = self._get_verbose_segments(