[ASR] Optimize CPU preproc to get 2.5x RTFx via multi-threading (#44612)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ekagra Ranjan
2026-06-11 21:05:11 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 42ae5e7ac6
commit e0b9fb1290
4 changed files with 105 additions and 34 deletions
@@ -474,6 +474,13 @@ async def lifespan(app: FastAPI):
finally:
if task is not None:
task.cancel()
for attr_name in (
"openai_serving_transcription",
"openai_serving_translation",
):
serving = getattr(app.state, attr_name, None)
if serving is not None and hasattr(serving, "shutdown"):
serving.shutdown()
finally:
# Ensure app state including engine ref is gc'd
del app.state
+61 -34
View File
@@ -6,6 +6,7 @@ import math
import time
import zlib
from collections.abc import AsyncGenerator, Callable, Set
from concurrent.futures import ThreadPoolExecutor
from functools import cached_property
from typing import Final, Literal, TypeAlias, TypeVar, cast
@@ -37,7 +38,7 @@ from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import get_tokenizer
from vllm.utils.async_utils import merge_async_iterators
from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterators
from ..transcription.protocol import (
TranscriptionResponse,
@@ -63,6 +64,7 @@ T = TypeVar("T", bound=SpeechToTextResponse)
V = TypeVar("V", bound=SpeechToTextResponseVerbose)
S = TypeVar("S", bound=SpeechToTextSegment)
ResponseType: TypeAlias = (
TranscriptionResponse
| TranslationResponse
@@ -131,6 +133,19 @@ class OpenAISpeechToText(OpenAIServing):
self.default_sampling_params,
)
# setup preprocess resources
# we keep separate thread pool for frontend preprocessing instead
# of reusing the one from Renderer which showed lower throughput
# https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781
num_audio_preprocess_workers = envs.VLLM_MAX_AUDIO_PREPROCESS_WORKERS
self._preprocess_executor = ThreadPoolExecutor(
max_workers=num_audio_preprocess_workers,
thread_name_prefix="stt-preprocess",
)
self._decode_and_chunk_speech_async = make_async_with_semaphore(
self._decode_and_chunk_speech, executor=self._preprocess_executor
)
@cached_property
def model_cls(self) -> type[SupportsTranscription]:
from vllm.model_executor.model_loader import get_model_cls
@@ -138,6 +153,49 @@ class OpenAISpeechToText(OpenAIServing):
model_cls = get_model_cls(self.model_config)
return cast(type[SupportsTranscription], model_cls)
def shutdown(self) -> None:
self._preprocess_executor.shutdown(wait=False)
def _decode_and_chunk_speech(
self,
audio_data: bytes,
) -> tuple[list[np.ndarray], float]:
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
# transparently falls back to ffmpeg via an in-memory fd.
# NOTE resample to model SR here for efficiency. This is also a
# pre-requisite for chunking, as it assumes Whisper SR.
try:
with io.BytesIO(audio_data) as buf:
y, sr = load_audio(
buf,
sr=self.asr_config.sample_rate,
max_duration_s=self.max_audio_decode_duration_s,
)
except Exception as exc:
raise ValueError("Invalid or unsupported audio file.") from exc
duration = get_audio_duration(y=y, sr=sr)
do_split_audio = self.asr_config.allow_audio_chunking and (
self.asr_config.max_audio_clip_s is not None
and duration > self.asr_config.max_audio_clip_s
)
if not do_split_audio:
chunks = [y]
else:
assert self.asr_config.max_audio_clip_s is not None
assert self.asr_config.min_energy_split_window_size is not None
chunks = split_audio(
audio_data=y,
sample_rate=int(sr),
max_clip_duration_s=self.asr_config.max_audio_clip_s,
overlap_duration_s=self.asr_config.overlap_chunk_second,
min_energy_window_size=self.asr_config.min_energy_split_window_size,
)
return chunks, duration
async def _detect_language(
self,
audio_chunk: np.ndarray,
@@ -210,39 +268,8 @@ class OpenAISpeechToText(OpenAIServing):
value=len(audio_data) / 1024**2,
)
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
# transparently falls back to ffmpeg via an in-memory fd.
# NOTE resample to model SR here for efficiency. This is also a
# pre-requisite for chunking, as it assumes Whisper SR.
try:
with io.BytesIO(audio_data) as buf:
y, sr = load_audio(
buf,
sr=self.asr_config.sample_rate,
max_duration_s=self.max_audio_decode_duration_s,
)
except Exception as exc:
raise ValueError("Invalid or unsupported audio file.") from exc
duration = get_audio_duration(y=y, sr=sr)
do_split_audio = self.asr_config.allow_audio_chunking and (
self.asr_config.max_audio_clip_s is not None
and duration > self.asr_config.max_audio_clip_s
)
if not do_split_audio:
chunks = [y]
else:
assert self.asr_config.max_audio_clip_s is not None
assert self.asr_config.min_energy_split_window_size is not None
chunks = split_audio(
audio_data=y,
sample_rate=int(sr),
max_clip_duration_s=self.asr_config.max_audio_clip_s,
overlap_duration_s=self.asr_config.overlap_chunk_second,
min_energy_window_size=self.asr_config.min_energy_split_window_size,
)
# Run cpu intensive preprocess step in a separate thread pool executor.
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
if request.language is None and getattr(
self.model_cls, "supports_explicit_language_detection", False
+11
View File
@@ -78,6 +78,7 @@ if TYPE_CHECKING:
VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
VLLM_MAX_AUDIO_DECODE_DURATION_S: int = 600
VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2))
VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
VLLM_MEDIA_CONNECTOR: str = "http"
VLLM_MM_HASHER_ALGORITHM: str = "blake3"
@@ -928,6 +929,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MAX_AUDIO_DECODE_DURATION_S": lambda: int(
os.getenv("VLLM_MAX_AUDIO_DECODE_DURATION_S", "600")
),
# Maximum number of worker threads used for STT preprocessing. The default
# intentionally caps at 2 because that performed best in profiling.
# https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781
"VLLM_MAX_AUDIO_PREPROCESS_WORKERS": lambda: int(
os.getenv(
"VLLM_MAX_AUDIO_PREPROCESS_WORKERS",
str(max(1, min(os.cpu_count() or 1, 2))),
)
),
# Backend for Video IO — selects the frame-sampling algorithm.
# - "opencv": uniform sampling.
# - "opencv_dynamic": duration-aware dynamic sampling.
@@ -1997,6 +2007,7 @@ def compile_factors() -> dict[str, object]:
"VLLM_MEDIA_LOADING_THREAD_COUNT",
"VLLM_MAX_AUDIO_CLIP_FILESIZE_MB",
"VLLM_MAX_AUDIO_DECODE_DURATION_S",
"VLLM_MAX_AUDIO_PREPROCESS_WORKERS",
"VLLM_VIDEO_LOADER_BACKEND",
"VLLM_MEDIA_CONNECTOR",
"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME",
+26
View File
@@ -248,6 +248,32 @@ def make_async(
return _async_wrapper
def make_async_with_semaphore(
func: Callable[P, T],
executor: ThreadPoolExecutor,
) -> Callable[P, Awaitable[T]]:
"""
Take a blocking function, and run it on in an executor thread.
This function prevents the blocking function from blocking the
asyncio event loop.
The code in this function needs to be thread safe.
The function is wrapped in a semaphore to limit the number of
concurrent executions making it easier to cancel tasks before they start.
"""
semaphore = asyncio.Semaphore(executor._max_workers)
async def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
loop = asyncio.get_event_loop()
p_func = partial(func, *args, **kwargs)
async with semaphore:
return await loop.run_in_executor(executor, p_func)
return _async_wrapper
def run_in_loop(loop: AbstractEventLoop, function: Callable, *args):
if in_loop(loop):
function(*args)