[ASR] Optimize CPU preproc to get 2.5x RTFx via multi-threading (#44612)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
42ae5e7ac6
commit
e0b9fb1290
@@ -6,6 +6,7 @@ import math
|
||||
import time
|
||||
import zlib
|
||||
from collections.abc import AsyncGenerator, Callable, Set
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import cached_property
|
||||
from typing import Final, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
@@ -37,7 +38,7 @@ from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
|
||||
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
from vllm.utils.async_utils import make_async_with_semaphore, merge_async_iterators
|
||||
|
||||
from ..transcription.protocol import (
|
||||
TranscriptionResponse,
|
||||
@@ -63,6 +64,7 @@ T = TypeVar("T", bound=SpeechToTextResponse)
|
||||
V = TypeVar("V", bound=SpeechToTextResponseVerbose)
|
||||
S = TypeVar("S", bound=SpeechToTextSegment)
|
||||
|
||||
|
||||
ResponseType: TypeAlias = (
|
||||
TranscriptionResponse
|
||||
| TranslationResponse
|
||||
@@ -131,6 +133,19 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
self.default_sampling_params,
|
||||
)
|
||||
|
||||
# setup preprocess resources
|
||||
# we keep separate thread pool for frontend preprocessing instead
|
||||
# of reusing the one from Renderer which showed lower throughput
|
||||
# https://github.com/vllm-project/vllm/pull/44612#issuecomment-4662757781
|
||||
num_audio_preprocess_workers = envs.VLLM_MAX_AUDIO_PREPROCESS_WORKERS
|
||||
self._preprocess_executor = ThreadPoolExecutor(
|
||||
max_workers=num_audio_preprocess_workers,
|
||||
thread_name_prefix="stt-preprocess",
|
||||
)
|
||||
self._decode_and_chunk_speech_async = make_async_with_semaphore(
|
||||
self._decode_and_chunk_speech, executor=self._preprocess_executor
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def model_cls(self) -> type[SupportsTranscription]:
|
||||
from vllm.model_executor.model_loader import get_model_cls
|
||||
@@ -138,6 +153,49 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
model_cls = get_model_cls(self.model_config)
|
||||
return cast(type[SupportsTranscription], model_cls)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._preprocess_executor.shutdown(wait=False)
|
||||
|
||||
def _decode_and_chunk_speech(
|
||||
self,
|
||||
audio_data: bytes,
|
||||
) -> tuple[list[np.ndarray], float]:
|
||||
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
|
||||
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
|
||||
# transparently falls back to ffmpeg via an in-memory fd.
|
||||
# NOTE resample to model SR here for efficiency. This is also a
|
||||
# pre-requisite for chunking, as it assumes Whisper SR.
|
||||
try:
|
||||
with io.BytesIO(audio_data) as buf:
|
||||
y, sr = load_audio(
|
||||
buf,
|
||||
sr=self.asr_config.sample_rate,
|
||||
max_duration_s=self.max_audio_decode_duration_s,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ValueError("Invalid or unsupported audio file.") from exc
|
||||
|
||||
duration = get_audio_duration(y=y, sr=sr)
|
||||
do_split_audio = self.asr_config.allow_audio_chunking and (
|
||||
self.asr_config.max_audio_clip_s is not None
|
||||
and duration > self.asr_config.max_audio_clip_s
|
||||
)
|
||||
|
||||
if not do_split_audio:
|
||||
chunks = [y]
|
||||
else:
|
||||
assert self.asr_config.max_audio_clip_s is not None
|
||||
assert self.asr_config.min_energy_split_window_size is not None
|
||||
chunks = split_audio(
|
||||
audio_data=y,
|
||||
sample_rate=int(sr),
|
||||
max_clip_duration_s=self.asr_config.max_audio_clip_s,
|
||||
overlap_duration_s=self.asr_config.overlap_chunk_second,
|
||||
min_energy_window_size=self.asr_config.min_energy_split_window_size,
|
||||
)
|
||||
|
||||
return chunks, duration
|
||||
|
||||
async def _detect_language(
|
||||
self,
|
||||
audio_chunk: np.ndarray,
|
||||
@@ -210,39 +268,8 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
value=len(audio_data) / 1024**2,
|
||||
)
|
||||
|
||||
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
|
||||
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
|
||||
# transparently falls back to ffmpeg via an in-memory fd.
|
||||
# NOTE resample to model SR here for efficiency. This is also a
|
||||
# pre-requisite for chunking, as it assumes Whisper SR.
|
||||
try:
|
||||
with io.BytesIO(audio_data) as buf:
|
||||
y, sr = load_audio(
|
||||
buf,
|
||||
sr=self.asr_config.sample_rate,
|
||||
max_duration_s=self.max_audio_decode_duration_s,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ValueError("Invalid or unsupported audio file.") from exc
|
||||
|
||||
duration = get_audio_duration(y=y, sr=sr)
|
||||
do_split_audio = self.asr_config.allow_audio_chunking and (
|
||||
self.asr_config.max_audio_clip_s is not None
|
||||
and duration > self.asr_config.max_audio_clip_s
|
||||
)
|
||||
|
||||
if not do_split_audio:
|
||||
chunks = [y]
|
||||
else:
|
||||
assert self.asr_config.max_audio_clip_s is not None
|
||||
assert self.asr_config.min_energy_split_window_size is not None
|
||||
chunks = split_audio(
|
||||
audio_data=y,
|
||||
sample_rate=int(sr),
|
||||
max_clip_duration_s=self.asr_config.max_audio_clip_s,
|
||||
overlap_duration_s=self.asr_config.overlap_chunk_second,
|
||||
min_energy_window_size=self.asr_config.min_energy_split_window_size,
|
||||
)
|
||||
# Run cpu intensive preprocess step in a separate thread pool executor.
|
||||
chunks, duration = await self._decode_and_chunk_speech_async(audio_data)
|
||||
|
||||
if request.language is None and getattr(
|
||||
self.model_cls, "supports_explicit_language_detection", False
|
||||
|
||||
Reference in New Issue
Block a user