[ASR] Optimize CPU preproc to get 2.5x RTFx via multi-threading (#44612)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
42ae5e7ac6
commit
e0b9fb1290
@@ -474,6 +474,13 @@ async def lifespan(app: FastAPI):
|
||||
finally:
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
for attr_name in (
|
||||
"openai_serving_transcription",
|
||||
"openai_serving_translation",
|
||||
):
|
||||
serving = getattr(app.state, attr_name, None)
|
||||
if serving is not None and hasattr(serving, "shutdown"):
|
||||
serving.shutdown()
|
||||
finally:
|
||||
# Ensure app state including engine ref is gc'd
|
||||
del app.state
|
||||
|
||||
@@ -6,6 +6,7 @@ import math
|
||||
import time
|
||||
import zlib
|
||||
from collections.abc import AsyncGenerator, Callable, Set
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import cached_property
|
||||
from typing import Final, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
@@ -37,7 +38,7 @@ from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
|
||||
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterators
|
||||
|
||||
from ..transcription.protocol import (
|
||||
TranscriptionResponse,
|
||||
@@ -63,6 +64,7 @@ T = TypeVar("T", bound=SpeechToTextResponse)
|
||||
V = TypeVar("V", bound=SpeechToTextResponseVerbose)
|
||||
S = TypeVar("S", bound=SpeechToTextSegment)
|
||||
|
||||
|
||||
ResponseType: TypeAlias = (
|
||||
TranscriptionResponse
|
||||
| TranslationResponse
|
||||
@@ -131,6 +133,19 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
self.default_sampling_params,
|
||||
)
|
||||
|
||||
# setup preprocess resources
|
||||
# we keep separate thread pool for frontend preprocessing instead
|
||||
# of reusing the one from Renderer which showed lower throughput
|
||||
# https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781
|
||||
num_audio_preprocess_workers = envs.VLLM_MAX_AUDIO_PREPROCESS_WORKERS
|
||||
self._preprocess_executor = ThreadPoolExecutor(
|
||||
max_workers=num_audio_preprocess_workers,
|
||||
thread_name_prefix="stt-preprocess",
|
||||
)
|
||||
self._decode_and_chunk_speech_async = make_async_with_semaphore(
|
||||
self._decode_and_chunk_speech, executor=self._preprocess_executor
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def model_cls(self) -> type[SupportsTranscription]:
|
||||
from vllm.model_executor.model_loader import get_model_cls
|
||||
@@ -138,6 +153,49 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
model_cls = get_model_cls(self.model_config)
|
||||
return cast(type[SupportsTranscription], model_cls)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._preprocess_executor.shutdown(wait=False)
|
||||
|
||||
def _decode_and_chunk_speech(
|
||||
self,
|
||||
audio_data: bytes,
|
||||
) -> tuple[list[np.ndarray], float]:
|
||||
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
|
||||
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
|
||||
# transparently falls back to ffmpeg via an in-memory fd.
|
||||
# NOTE resample to model SR here for efficiency. This is also a
|
||||
# pre-requisite for chunking, as it assumes Whisper SR.
|
||||
try:
|
||||
with io.BytesIO(audio_data) as buf:
|
||||
y, sr = load_audio(
|
||||
buf,
|
||||
sr=self.asr_config.sample_rate,
|
||||
max_duration_s=self.max_audio_decode_duration_s,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ValueError("Invalid or unsupported audio file.") from exc
|
||||
|
||||
duration = get_audio_duration(y=y, sr=sr)
|
||||
do_split_audio = self.asr_config.allow_audio_chunking and (
|
||||
self.asr_config.max_audio_clip_s is not None
|
||||
and duration > self.asr_config.max_audio_clip_s
|
||||
)
|
||||
|
||||
if not do_split_audio:
|
||||
chunks = [y]
|
||||
else:
|
||||
assert self.asr_config.max_audio_clip_s is not None
|
||||
assert self.asr_config.min_energy_split_window_size is not None
|
||||
chunks = split_audio(
|
||||
audio_data=y,
|
||||
sample_rate=int(sr),
|
||||
max_clip_duration_s=self.asr_config.max_audio_clip_s,
|
||||
overlap_duration_s=self.asr_config.overlap_chunk_second,
|
||||
min_energy_window_size=self.asr_config.min_energy_split_window_size,
|
||||
)
|
||||
|
||||
return chunks, duration
|
||||
|
||||
async def _detect_language(
|
||||
self,
|
||||
audio_chunk: np.ndarray,
|
||||
@@ -210,39 +268,8 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
value=len(audio_data) / 1024**2,
|
||||
)
|
||||
|
||||
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
|
||||
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
|
||||
# transparently falls back to ffmpeg via an in-memory fd.
|
||||
# NOTE resample to model SR here for efficiency. This is also a
|
||||
# pre-requisite for chunking, as it assumes Whisper SR.
|
||||
try:
|
||||
with io.BytesIO(audio_data) as buf:
|
||||
y, sr = load_audio(
|
||||
buf,
|
||||
sr=self.asr_config.sample_rate,
|
||||
max_duration_s=self.max_audio_decode_duration_s,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ValueError("Invalid or unsupported audio file.") from exc
|
||||
|
||||
duration = get_audio_duration(y=y, sr=sr)
|
||||
do_split_audio = self.asr_config.allow_audio_chunking and (
|
||||
self.asr_config.max_audio_clip_s is not None
|
||||
and duration > self.asr_config.max_audio_clip_s
|
||||
)
|
||||
|
||||
if not do_split_audio:
|
||||
chunks = [y]
|
||||
else:
|
||||
assert self.asr_config.max_audio_clip_s is not None
|
||||
assert self.asr_config.min_energy_split_window_size is not None
|
||||
chunks = split_audio(
|
||||
audio_data=y,
|
||||
sample_rate=int(sr),
|
||||
max_clip_duration_s=self.asr_config.max_audio_clip_s,
|
||||
overlap_duration_s=self.asr_config.overlap_chunk_second,
|
||||
min_energy_window_size=self.asr_config.min_energy_split_window_size,
|
||||
)
|
||||
# Run cpu intensive preprocess step in a separate thread pool executor.
|
||||
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
|
||||
|
||||
if request.language is None and getattr(
|
||||
self.model_cls, "supports_explicit_language_detection", False
|
||||
|
||||
@@ -78,6 +78,7 @@ if TYPE_CHECKING:
|
||||
VLLM_MEDIA_LOADING_THREAD_COUNT: int = 8
|
||||
VLLM_MAX_AUDIO_CLIP_FILESIZE_MB: int = 25
|
||||
VLLM_MAX_AUDIO_DECODE_DURATION_S: int = 600
|
||||
VLLM_MAX_AUDIO_PREPROCESS_WORKERS: int = max(1, min(os.cpu_count() or 1, 2))
|
||||
VLLM_VIDEO_LOADER_BACKEND: str = "opencv"
|
||||
VLLM_MEDIA_CONNECTOR: str = "http"
|
||||
VLLM_MM_HASHER_ALGORITHM: str = "blake3"
|
||||
@@ -928,6 +929,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_MAX_AUDIO_DECODE_DURATION_S": lambda: int(
|
||||
os.getenv("VLLM_MAX_AUDIO_DECODE_DURATION_S", "600")
|
||||
),
|
||||
# Maximum number of worker threads used for STT preprocessing. The default
|
||||
# intentionally caps at 2 because that performed best in profiling.
|
||||
# https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781
|
||||
"VLLM_MAX_AUDIO_PREPROCESS_WORKERS": lambda: int(
|
||||
os.getenv(
|
||||
"VLLM_MAX_AUDIO_PREPROCESS_WORKERS",
|
||||
str(max(1, min(os.cpu_count() or 1, 2))),
|
||||
)
|
||||
),
|
||||
# Backend for Video IO — selects the frame-sampling algorithm.
|
||||
# - "opencv": uniform sampling.
|
||||
# - "opencv_dynamic": duration-aware dynamic sampling.
|
||||
@@ -1997,6 +2007,7 @@ def compile_factors() -> dict[str, object]:
|
||||
"VLLM_MEDIA_LOADING_THREAD_COUNT",
|
||||
"VLLM_MAX_AUDIO_CLIP_FILESIZE_MB",
|
||||
"VLLM_MAX_AUDIO_DECODE_DURATION_S",
|
||||
"VLLM_MAX_AUDIO_PREPROCESS_WORKERS",
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"VLLM_MEDIA_CONNECTOR",
|
||||
"VLLM_OBJECT_STORAGE_SHM_BUFFER_NAME",
|
||||
|
||||
@@ -248,6 +248,32 @@ def make_async(
|
||||
return _async_wrapper
|
||||
|
||||
|
||||
def make_async_with_semaphore(
|
||||
func: Callable[P, T],
|
||||
executor: ThreadPoolExecutor,
|
||||
) -> Callable[P, Awaitable[T]]:
|
||||
"""
|
||||
Take a blocking function, and run it on in an executor thread.
|
||||
|
||||
This function prevents the blocking function from blocking the
|
||||
asyncio event loop.
|
||||
The code in this function needs to be thread safe.
|
||||
|
||||
The function is wrapped in a semaphore to limit the number of
|
||||
concurrent executions making it easier to cancel tasks before they start.
|
||||
"""
|
||||
|
||||
semaphore = asyncio.Semaphore(executor._max_workers)
|
||||
|
||||
async def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
|
||||
loop = asyncio.get_event_loop()
|
||||
p_func = partial(func, *args, **kwargs)
|
||||
async with semaphore:
|
||||
return await loop.run_in_executor(executor, p_func)
|
||||
|
||||
return _async_wrapper
|
||||
|
||||
|
||||
def run_in_loop(loop: AbstractEventLoop, function: Callable, *args):
|
||||
if in_loop(loop):
|
||||
function(*args)
|
||||
|
||||
Reference in New Issue
Block a user