[ASR] Optimize CPU preproc to get 2.5x RTFx via multi-threading (#44612)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ekagra Ranjan
2026-06-11 21:05:11 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 42ae5e7ac6
commit e0b9fb1290
4 changed files with 105 additions and 34 deletions
+61 -34
View File
@@ -6,6 +6,7 @@ import math
import time
import zlib
from collections.abc import AsyncGenerator, Callable, Set
from concurrent.futures import ThreadPoolExecutor
from functools import cached_property
from typing import Final, Literal, TypeAlias, TypeVar, cast
@@ -37,7 +38,7 @@ from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import get_tokenizer
from vllm.utils.async_utils import merge_async_iterators
from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterators
from ..transcription.protocol import (
TranscriptionResponse,
@@ -63,6 +64,7 @@ T = TypeVar("T", bound=SpeechToTextResponse)
V = TypeVar("V", bound=SpeechToTextResponseVerbose)
S = TypeVar("S", bound=SpeechToTextSegment)
ResponseType: TypeAlias = (
TranscriptionResponse
| TranslationResponse
@@ -131,6 +133,19 @@ class OpenAISpeechToText(OpenAIServing):
self.default_sampling_params,
)
# setup preprocess resources
# we keep separate thread pool for frontend preprocessing instead
# of reusing the one from Renderer which showed lower throughput
# https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781
num_audio_preprocess_workers = envs.VLLM_MAX_AUDIO_PREPROCESS_WORKERS
self._preprocess_executor = ThreadPoolExecutor(
max_workers=num_audio_preprocess_workers,
thread_name_prefix="stt-preprocess",
)
self._decode_and_chunk_speech_async = make_async_with_semaphore(
self._decode_and_chunk_speech, executor=self._preprocess_executor
)
@cached_property
def model_cls(self) -> type[SupportsTranscription]:
from vllm.model_executor.model_loader import get_model_cls
@@ -138,6 +153,49 @@ class OpenAISpeechToText(OpenAIServing):
model_cls = get_model_cls(self.model_config)
return cast(type[SupportsTranscription], model_cls)
def shutdown(self) -> None:
self._preprocess_executor.shutdown(wait=False)
def _decode_and_chunk_speech(
self,
audio_data: bytes,
) -> tuple[list[np.ndarray], float]:
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
# transparently falls back to ffmpeg via an in-memory fd.
# NOTE resample to model SR here for efficiency. This is also a
# pre-requisite for chunking, as it assumes Whisper SR.
try:
with io.BytesIO(audio_data) as buf:
y, sr = load_audio(
buf,
sr=self.asr_config.sample_rate,
max_duration_s=self.max_audio_decode_duration_s,
)
except Exception as exc:
raise ValueError("Invalid or unsupported audio file.") from exc
duration = get_audio_duration(y=y, sr=sr)
do_split_audio = self.asr_config.allow_audio_chunking and (
self.asr_config.max_audio_clip_s is not None
and duration > self.asr_config.max_audio_clip_s
)
if not do_split_audio:
chunks = [y]
else:
assert self.asr_config.max_audio_clip_s is not None
assert self.asr_config.min_energy_split_window_size is not None
chunks = split_audio(
audio_data=y,
sample_rate=int(sr),
max_clip_duration_s=self.asr_config.max_audio_clip_s,
overlap_duration_s=self.asr_config.overlap_chunk_second,
min_energy_window_size=self.asr_config.min_energy_split_window_size,
)
return chunks, duration
async def _detect_language(
self,
audio_chunk: np.ndarray,
@@ -210,39 +268,8 @@ class OpenAISpeechToText(OpenAIServing):
value=len(audio_data) / 1024**2,
)
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
# transparently falls back to ffmpeg via an in-memory fd.
# NOTE resample to model SR here for efficiency. This is also a
# pre-requisite for chunking, as it assumes Whisper SR.
try:
with io.BytesIO(audio_data) as buf:
y, sr = load_audio(
buf,
sr=self.asr_config.sample_rate,
max_duration_s=self.max_audio_decode_duration_s,
)
except Exception as exc:
raise ValueError("Invalid or unsupported audio file.") from exc
duration = get_audio_duration(y=y, sr=sr)
do_split_audio = self.asr_config.allow_audio_chunking and (
self.asr_config.max_audio_clip_s is not None
and duration > self.asr_config.max_audio_clip_s
)
if not do_split_audio:
chunks = [y]
else:
assert self.asr_config.max_audio_clip_s is not None
assert self.asr_config.min_energy_split_window_size is not None
chunks = split_audio(
audio_data=y,
sample_rate=int(sr),
max_clip_duration_s=self.asr_config.max_audio_clip_s,
overlap_duration_s=self.asr_config.overlap_chunk_second,
min_energy_window_size=self.asr_config.min_energy_split_window_size,
)
# Run cpu intensive preprocess step in a separate thread pool executor.
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
if request.language is None and getattr(
self.model_cls, "supports_explicit_language_detection", False