From e0b9fb12902b0bed54d2f1b866a7ae00b30aa814 Mon Sep 17 00:00:00 2001 From: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Date: Fri, 12 Jun 2026 00:05:11 -0400 Subject: [PATCH] [ASR] Optimize CPU preproc to get 2.5x RTFx via multi-threading (#44612) Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/entrypoints/serve/utils/server_utils.py | 7 ++ .../speech_to_text/base/serving.py | 95 ++++++++++++------- vllm/envs.py | 11 +++ vllm/utils/async_utils.py | 26 +++++ 4 files changed, 105 insertions(+), 34 deletions(-) diff --git a/vllm/entrypoints/serve/utils/server_utils.py b/vllm/entrypoints/serve/utils/server_utils.py index 3b6dfde447..d24d492b61 100644 --- a/vllm/entrypoints/serve/utils/server_utils.py +++ b/vllm/entrypoints/serve/utils/server_utils.py @@ -474,6 +474,13 @@ async def lifespan(app: FastAPI): finally: if task is not None: task.cancel() + for attr_name in ( + "openai_serving_transcription", + "openai_serving_translation", + ): + serving = getattr(app.state, attr_name, None) + if serving is not None and hasattr(serving, "shutdown"): + serving.shutdown() finally: # Ensure app state including engine ref is gc'd del app.state diff --git a/vllm/entrypoints/speech_to_text/base/serving.py b/vllm/entrypoints/speech_to_text/base/serving.py index 1c6a0d77fe..9c0ecac41c 100644 --- a/vllm/entrypoints/speech_to_text/base/serving.py +++ b/vllm/entrypoints/speech_to_text/base/serving.py @@ -6,6 +6,7 @@ import math import time import zlib from collections.abc import AsyncGenerator, Callable, Set +from concurrent.futures import ThreadPoolExecutor from functools import cached_property from typing import Final, Literal, TypeAlias, TypeVar, cast @@ -37,7 +38,7 @@ from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import get_tokenizer -from vllm.utils.async_utils import merge_async_iterators +from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterators from ..transcription.protocol import ( TranscriptionResponse, @@ -63,6 +64,7 @@ T = TypeVar("T", bound=SpeechToTextResponse) V = TypeVar("V", bound=SpeechToTextResponseVerbose) S = TypeVar("S", bound=SpeechToTextSegment) + ResponseType: TypeAlias = ( TranscriptionResponse | TranslationResponse @@ -131,6 +133,19 @@ class OpenAISpeechToText(OpenAIServing): self.default_sampling_params, ) + # setup preprocess resources + # we keep separate thread pool for frontend preprocessing instead + # of reusing the one from Renderer which showed lower throughput + # https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781 + num_audio_preprocess_workers = envs.VLLM_MAX_AUDIO_PREPROCESS_WORKERS + self._preprocess_executor = ThreadPoolExecutor( + max_workers=num_audio_preprocess_workers, + thread_name_prefix="stt-preprocess", + ) + self._decode_and_chunk_speech_async = make_async_with_semaphore( + self._decode_and_chunk_speech, executor=self._preprocess_executor + ) + @cached_property def model_cls(self) -> type[SupportsTranscription]: from vllm.model_executor.model_loader import get_model_cls @@ -138,6 +153,49 @@ class OpenAISpeechToText(OpenAIServing): model_cls = get_model_cls(self.model_config) return cast(type[SupportsTranscription], model_cls) + def shutdown(self) -> None: + self._preprocess_executor.shutdown(wait=False) + + def _decode_and_chunk_speech( + self, + audio_data: bytes, + ) -> tuple[list[np.ndarray], float]: + # Decode audio bytes. For container formats (MP4, M4A, WebM) that + # soundfile cannot detect from a BytesIO stream, _load_audio_bytes + # transparently falls back to ffmpeg via an in-memory fd. + # NOTE resample to model SR here for efficiency. This is also a + # pre-requisite for chunking, as it assumes Whisper SR. + try: + with io.BytesIO(audio_data) as buf: + y, sr = load_audio( + buf, + sr=self.asr_config.sample_rate, + max_duration_s=self.max_audio_decode_duration_s, + ) + except Exception as exc: + raise ValueError("Invalid or unsupported audio file.") from exc + + duration = get_audio_duration(y=y, sr=sr) + do_split_audio = self.asr_config.allow_audio_chunking and ( + self.asr_config.max_audio_clip_s is not None + and duration > self.asr_config.max_audio_clip_s + ) + + if not do_split_audio: + chunks = [y] + else: + assert self.asr_config.max_audio_clip_s is not None + assert self.asr_config.min_energy_split_window_size is not None + chunks = split_audio( + audio_data=y, + sample_rate=int(sr), + max_clip_duration_s=self.asr_config.max_audio_clip_s, + overlap_duration_s=self.asr_config.overlap_chunk_second, + min_energy_window_size=self.asr_config.min_energy_split_window_size, + ) + + return chunks, duration + async def _detect_language( self, audio_chunk: np.ndarray, @@ -210,39 +268,8 @@ class OpenAISpeechToText(OpenAIServing): value=len(audio_data) / 1024**2, ) - # Decode audio bytes. For container formats (MP4, M4A, WebM) that - # soundfile cannot detect from a BytesIO stream, _load_audio_bytes - # transparently falls back to ffmpeg via an in-memory fd. - # NOTE resample to model SR here for efficiency. This is also a - # pre-requisite for chunking, as it assumes Whisper SR. - try: - with io.BytesIO(audio_data) as buf: - y, sr = load_audio( - buf, - sr=self.asr_config.sample_rate, - max_duration_s=self.max_audio_decode_duration_s, - ) - except Exception as exc: - raise ValueError("Invalid or unsupported audio file.") from exc - - duration = get_audio_duration(y=y, sr=sr) - do_split_audio = self.asr_config.allow_audio_chunking and ( - self.asr_config.max_audio_clip_s is not None - and duration > self.asr_config.max_audio_clip_s - ) - - if not do_split_audio: - chunks = [y] - else: - assert self.asr_config.max_audio_clip_s is not None - assert self.asr_config.min_energy_split_window_size is not None - chunks = split_audio( - audio_data=y, - sample_rate=int(sr), - max_clip_duration_s=self.asr_config.max_audio_clip_s, - overlap_duration_s=self.asr_config.overlap_chunk_second, - min_energy_window_size=self.asr_config.min_energy_split_window_size, - ) + # Run cpu intensive preprocess step in a separate thread pool executor. + chunks, duration = await self._decode_and_chunk_speech_async(audio_data) if request.language is None and getattr( self.model_cls, "supports_explicit_language_detection", False diff --git a/vllm/envs.py b/vllm/envs.py index d0133638f1..479aab2323 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -78,6 +78,7 @@ if TYPE_CHECKING: VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8 VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25 VLLM_MAX_AUDIO_DECODE_DURATION_S: int = 600 + VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2)) VLLM_VIDEO_LOADER_BACKEND: str = "opencv" VLLM_MEDIA_CONNECTOR: str = "http" VLLM_MM_HASHER_ALGORITHM: str = "blake3" @@ -928,6 +929,15 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_MAX_AUDIO_DECODE_DURATION_S": lambda: int( os.getenv("VLLM_MAX_AUDIO_DECODE_DURATION_S", "600") ), + # Maximum number of worker threads used for STT preprocessing. The default + # intentionally caps at 2 because that performed best in profiling. + # https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781 + "VLLM_MAX_AUDIO_PREPROCESS_WORKERS": lambda: int( + os.getenv( + "VLLM_MAX_AUDIO_PREPROCESS_WORKERS", + str(max(1, min(os.cpu_count() or 1, 2))), + ) + ), # Backend for Video IO — selects the frame-sampling algorithm. # - "opencv": uniform sampling. # - "opencv_dynamic": duration-aware dynamic sampling. @@ -1997,6 +2007,7 @@ def compile_factors() -> dict[str, object]: "VLLM_MEDIA_LOADING_THREAD_COUNT", "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", "VLLM_MAX_AUDIO_DECODE_DURATION_S", + "VLLM_MAX_AUDIO_PREPROCESS_WORKERS", "VLLM_VIDEO_LOADER_BACKEND", "VLLM_MEDIA_CONNECTOR", "VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME", diff --git a/vllm/utils/async_utils.py b/vllm/utils/async_utils.py index 725868c39a..9f368be7b2 100644 --- a/vllm/utils/async_utils.py +++ b/vllm/utils/async_utils.py @@ -248,6 +248,32 @@ def make_async( return _async_wrapper +def make_async_with_semaphore( + func: Callable[P, T], + executor: ThreadPoolExecutor, +) -> Callable[P, Awaitable[T]]: + """ + Take a blocking function, and run it on in an executor thread. + + This function prevents the blocking function from blocking the + asyncio event loop. + The code in this function needs to be thread safe. + + The function is wrapped in a semaphore to limit the number of + concurrent executions making it easier to cancel tasks before they start. + """ + + semaphore = asyncio.Semaphore(executor._max_workers) + + async def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + loop = asyncio.get_event_loop() + p_func = partial(func, *args, **kwargs) + async with semaphore: + return await loop.run_in_executor(executor, p_func) + + return _async_wrapper + + def run_in_loop(loop: AbstractEventLoop, function: Callable, *args): if in_loop(loop): function(*args)