[Bugfix] Changed speech to text chunk timestamp to cumulative approach (#41131)
Signed-off-by: Toby Bell <toby.bell1702@hotmail.co.uk>
This commit is contained in:
@@ -195,7 +195,7 @@ Provide a fast duration→token estimate to improve streaming usage statistics:
|
||||
The API server takes care of basic audio I/O and optional chunking before building prompts:
|
||||
|
||||
- Resampling: Input audio is resampled to `SpeechToTextConfig.sample_rate` using `AudioResampler`.
|
||||
- Chunking: If `SpeechToTextConfig.allow_audio_chunking` is True and the duration exceeds `max_audio_clip_s`, the server splits the audio into overlapping chunks and generates a prompt per chunk. Overlap is controlled by `overlap_chunk_second`.
|
||||
- Chunking: If `SpeechToTextConfig.allow_audio_chunking` is True and the duration exceeds `max_audio_clip_s`, the server splits the audio into chunks and generates a prompt per chunk. There is no overlap between chunks, overlap_chunk_second controls the size of the search window used to find the split point.
|
||||
- Energy-aware splitting: When `min_energy_split_window_size` is set, the server finds low-energy regions to minimize cutting within words.
|
||||
|
||||
Relevant server logic:
|
||||
|
||||
@@ -53,7 +53,13 @@ async def test_non_streaming_cancel_aborts_engine_requests(
|
||||
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
|
||||
server._check_model = AsyncMock(return_value=None)
|
||||
server._maybe_get_adapters = Mock(return_value=None)
|
||||
server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 40.0))
|
||||
server._preprocess_speech_to_text = AsyncMock(
|
||||
return_value=(
|
||||
engine_inputs,
|
||||
40.0,
|
||||
[30.0 * i for i in range(len(engine_inputs))],
|
||||
)
|
||||
)
|
||||
server._log_inputs = Mock()
|
||||
|
||||
request = SimpleNamespace(
|
||||
@@ -122,7 +128,9 @@ async def test_non_streaming_cancel_advances_all_chunk_generators():
|
||||
server.asr_config = SimpleNamespace(max_audio_clip_s=30)
|
||||
server._check_model = AsyncMock(return_value=None)
|
||||
server._maybe_get_adapters = Mock(return_value=None)
|
||||
server._preprocess_speech_to_text = AsyncMock(return_value=(engine_inputs, 90.0))
|
||||
server._preprocess_speech_to_text = AsyncMock(
|
||||
return_value=(engine_inputs, 90.0, [0.0, 29.5, 29.5 + 29.7])
|
||||
)
|
||||
server._log_inputs = Mock()
|
||||
|
||||
request = SimpleNamespace(
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for chunk timestamp offset drift in _preprocess_speech_to_text."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config.speech_to_text import SpeechToTextConfig
|
||||
from vllm.entrypoints.speech_to_text.base.serving import SpeechToTextBaseServing
|
||||
|
||||
SR = 16_000
|
||||
_PATCH = "vllm.entrypoints.speech_to_text.base.serving"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chunk_offsets_are_cumulative_not_nominal():
|
||||
"""
|
||||
chunk_start_offsets must be cumulative actual chunk lengths, not the old approach of
|
||||
'idx * max_audio_clip_s'. When split_audio places a boundary before the
|
||||
nominal 30 s mark, the old formula drifts; the fixed formula stays exact.
|
||||
"""
|
||||
# Chunks shorter than exactly 30 s, as split_audio produces when a quiet
|
||||
# region falls inside the 1 s overlap window before the nominal boundary.
|
||||
chunk_lengths = [int(29.5 * SR), int(29.7 * SR), int(5.0 * SR)]
|
||||
chunks = [np.zeros(n, dtype=np.float32) for n in chunk_lengths]
|
||||
|
||||
duration = sum(chunk_lengths) / SR
|
||||
|
||||
expected_offsets = [0.0, 29.5, 29.5 + 29.7] # cumulative seconds
|
||||
wrong_offsets = [0.0, 30.0, 60.0] # what the old bug produced
|
||||
|
||||
serving = SpeechToTextBaseServing.__new__(SpeechToTextBaseServing)
|
||||
serving._decode_and_chunk_speech_async = AsyncMock(return_value=(chunks, duration))
|
||||
serving.asr_config = SpeechToTextConfig(
|
||||
sample_rate=float(SR),
|
||||
max_audio_clip_s=30,
|
||||
overlap_chunk_second=1,
|
||||
min_energy_split_window_size=1600,
|
||||
)
|
||||
serving.max_audio_filesize_mb = 100.0
|
||||
serving.model_cls = MagicMock()
|
||||
serving.model_cls.validate_language.side_effect = lambda lang: lang
|
||||
serving.model_cls.supports_explicit_language_detection = False
|
||||
serving.model_cls.get_generation_prompt.return_value = {}
|
||||
serving.model_config = MagicMock()
|
||||
serving.task_type = "transcribe"
|
||||
serving.renderer = MagicMock()
|
||||
serving.renderer.render_cmpl_async = AsyncMock(
|
||||
return_value=[MagicMock()] * len(chunks)
|
||||
)
|
||||
|
||||
request = MagicMock()
|
||||
request.language = "en"
|
||||
request.to_language = None
|
||||
request.response_format = "json"
|
||||
request.build_stt_params.return_value = MagicMock()
|
||||
|
||||
with patch(f"{_PATCH}.parse_model_prompt", return_value=MagicMock()):
|
||||
_, _, offsets = await serving._preprocess_speech_to_text(
|
||||
request=request,
|
||||
audio_data=b"\x00",
|
||||
request_id="test",
|
||||
)
|
||||
|
||||
assert offsets == pytest.approx(expected_offsets, abs=1e-6)
|
||||
assert offsets != pytest.approx(wrong_offsets, abs=1e-6)
|
||||
+3
-1
@@ -342,7 +342,9 @@ async def test_create_transcription_non_streaming_joins_chunks_by_language():
|
||||
models.lora_requests = {}
|
||||
models.is_base_model.return_value = True
|
||||
|
||||
preprocess_mock = AsyncMock(return_value=([MagicMock(), MagicMock()], 1.0))
|
||||
preprocess_mock = AsyncMock(
|
||||
return_value=([MagicMock(), MagicMock()], 1.0, [0.0, 29.5])
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
|
||||
@@ -258,7 +258,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
request: SpeechToTextRequest,
|
||||
audio_data: bytes,
|
||||
request_id: str,
|
||||
) -> tuple[list[EngineInput], float]:
|
||||
) -> tuple[list[EngineInput], float, list[float]]:
|
||||
# Validate request
|
||||
request.language = self.model_cls.validate_language(request.language)
|
||||
request.to_language = (
|
||||
@@ -277,6 +277,13 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
# Run cpu intensive preprocess step in a separate thread pool executor.
|
||||
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
|
||||
|
||||
chunk_start_offsets: list[float] = [0.0]
|
||||
|
||||
for chunk in chunks[:-1]:
|
||||
chunk_start_offsets.append(
|
||||
chunk_start_offsets[-1] + chunk.shape[-1] / self.asr_config.sample_rate
|
||||
)
|
||||
|
||||
if request.language is None and getattr(
|
||||
self.model_cls, "supports_explicit_language_detection", False
|
||||
):
|
||||
@@ -306,7 +313,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
|
||||
engine_inputs = await self.renderer.render_cmpl_async(parsed_prompts)
|
||||
|
||||
return engine_inputs, duration
|
||||
return engine_inputs, duration, chunk_start_offsets
|
||||
|
||||
def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):
|
||||
dec_prompt = prompt["decoder_prompt"]
|
||||
@@ -391,7 +398,7 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
SpeechToTextSegment,
|
||||
segment_class(
|
||||
id=len(segments),
|
||||
seek=start_time,
|
||||
seek=int(start_time),
|
||||
start=start_time + BASE_OFFSET * start_timestamp,
|
||||
end=start_time + BASE_OFFSET * end_timestamp,
|
||||
temperature=request.temperature,
|
||||
@@ -467,7 +474,11 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
|
||||
engine_inputs, duration_s = await self._preprocess_speech_to_text(
|
||||
(
|
||||
engine_inputs,
|
||||
duration_s,
|
||||
chunk_start_offsets,
|
||||
) = await self._preprocess_speech_to_text(
|
||||
request=request,
|
||||
audio_data=audio_data,
|
||||
request_id=request_id,
|
||||
@@ -587,11 +598,10 @@ class SpeechToTextBaseServing(GenerateBaseServing):
|
||||
assert len(list_result_generator) == 1, (
|
||||
"`max_audio_clip_s` is set to None, audio cannot be chunked"
|
||||
)
|
||||
assert len(chunk_start_offsets) == len(list_result_generator)
|
||||
result_generator = merge_async_iterators(*list_result_generator)
|
||||
async for idx, op in result_generator:
|
||||
start_time = (
|
||||
float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0
|
||||
)
|
||||
start_time = chunk_start_offsets[idx]
|
||||
if request.response_format == "verbose_json":
|
||||
assert op.outputs[0].logprobs
|
||||
segments: list[SpeechToTextSegment] = self._get_verbose_segments(
|
||||
|
||||
Reference in New Issue
Block a user