[Frontend] Consolidate Speech to Text entrypoints. (#42370)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
wang.yuqi
2026-05-12 07:06:57 +00:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent 8517cdaf90
commit d37e25ffbe
26 changed files with 681 additions and 607 deletions
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Literal, TypeAlias
import torch
## Protocols for Audio
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
_LONG_INFO = torch.iinfo(torch.long)
@@ -0,0 +1,756 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import io
import math
import time
import zlib
from collections.abc import AsyncGenerator, Callable, Set
from functools import cached_property
from typing import Final, Literal, TypeAlias, TypeVar, cast
import numpy as np
from fastapi import Request
from transformers import PreTrainedTokenizerBase
import vllm.envs as envs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
ErrorResponse,
RequestResponseMetadata,
UsageInfo,
)
from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.utils import get_max_tokens
from vllm.exceptions import VLLMValidationError
from vllm.inputs import EncoderDecoderInput, EngineInput
from vllm.logger import init_logger
from vllm.logprobs import FlatLogprobs, Logprob
from vllm.model_executor.models import SupportsTranscription
from vllm.multimodal.audio import get_audio_duration, split_audio
from vllm.multimodal.media.audio import load_audio
from vllm.outputs import RequestOutput
from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt
from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import get_tokenizer
from vllm.utils.async_utils import merge_async_iterators
from ..transcription.protocol import (
TranscriptionResponse,
TranscriptionResponseStreamChoice,
TranscriptionResponseVerbose,
TranscriptionSegment,
TranscriptionStreamResponse,
)
from ..translation.protocol import (
TranslationResponse,
TranslationResponseStreamChoice,
TranslationResponseVerbose,
TranslationSegment,
TranslationStreamResponse,
)
SpeechToTextResponse: TypeAlias = TranscriptionResponse | TranslationResponse
SpeechToTextResponseVerbose: TypeAlias = (
TranscriptionResponseVerbose | TranslationResponseVerbose
)
SpeechToTextSegment: TypeAlias = TranscriptionSegment | TranslationSegment
T = TypeVar("T", bound=SpeechToTextResponse)
V = TypeVar("V", bound=SpeechToTextResponseVerbose)
S = TypeVar("S", bound=SpeechToTextSegment)
ResponseType: TypeAlias = (
TranscriptionResponse
| TranslationResponse
| TranscriptionResponseVerbose
| TranslationResponseVerbose
)
logger = init_logger(__name__)
def asr_inter_chunk_separator(
language: str | None, no_space_languages: Set[str]
) -> str:
"""Space to insert between ASR text chunks for streaming and non-streaming join.
Languages in ``no_space_languages`` (e.g. Chinese, Japanese) use an empty
separator; others use a single ASCII space.
"""
return "" if language and language.lower() in no_space_languages else " "
class OpenAISpeechToText(OpenAIServing):
"""Base class for speech-to-text operations like transcription and
translation."""
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
task_type: Literal["transcribe", "translate"] = "transcribe",
enable_force_include_usage: bool = False,
):
super().__init__(
engine_client=engine_client,
models=models,
request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids,
)
self.default_sampling_params = self.model_config.get_diff_sampling_param()
self.task_type: Final = task_type
self.asr_config = self.model_cls.get_speech_to_text_config(
self.model_config, task_type
)
self.enable_force_include_usage = enable_force_include_usage
self.max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
if self.model_cls.supports_segment_timestamp:
self.tokenizer = cast(
PreTrainedTokenizerBase,
get_tokenizer(
tokenizer_name=self.model_config.tokenizer,
tokenizer_mode=self.model_config.tokenizer_mode,
),
)
if self.default_sampling_params:
logger.info(
"Overwriting default completion sampling param with: %s",
self.default_sampling_params,
)
@cached_property
def model_cls(self) -> type[SupportsTranscription]:
from vllm.model_executor.model_loader import get_model_cls
model_cls = get_model_cls(self.model_config)
return cast(type[SupportsTranscription], model_cls)
async def _detect_language(
self,
audio_chunk: np.ndarray,
request_id: str,
) -> str:
"""Auto-detect the spoken language from an audio chunk.
Delegates prompt construction and output parsing to the model class
via ``get_language_detection_prompt`` and
``parse_language_detection_output``.
"""
prompt = self.model_cls.get_language_detection_prompt(
audio_chunk,
self.asr_config,
)
allowed_token_ids = self.model_cls.get_language_token_ids(
self.tokenizer,
)
sampling_params = SamplingParams(
max_tokens=1,
temperature=0.0,
allowed_token_ids=allowed_token_ids,
)
result_generator = self.engine_client.generate(
prompt,
sampling_params,
request_id,
)
try:
final_output: RequestOutput
async for final_output in result_generator:
if final_output.finished:
break
except asyncio.CancelledError:
await asyncio.gather(
self.engine_client.abort(request_id),
return_exceptions=True,
)
raise
token_ids = list(final_output.outputs[0].token_ids)
lang = self.model_cls.parse_language_detection_output(
token_ids,
self.tokenizer,
)
logger.info("Auto-detected language: '%s'", lang)
return lang
async def _preprocess_speech_to_text(
self,
request: SpeechToTextRequest,
audio_data: bytes,
request_id: str,
) -> tuple[list[EngineInput], float]:
# Validate request
request.language = self.model_cls.validate_language(request.language)
request.to_language = (
self.model_cls.validate_language(request.to_language)
if request.to_language
else None
)
if len(audio_data) / 1024**2 > self.max_audio_filesize_mb:
raise VLLMValidationError(
"Maximum file size exceeded",
parameter="audio_filesize_mb",
value=len(audio_data) / 1024**2,
)
# Decode audio bytes. For container formats (MP4, M4A, WebM) that
# soundfile cannot detect from a BytesIO stream, _load_audio_bytes
# transparently falls back to ffmpeg via an in-memory fd.
# NOTE resample to model SR here for efficiency. This is also a
# pre-requisite for chunking, as it assumes Whisper SR.
try:
with io.BytesIO(audio_data) as buf:
y, sr = load_audio(buf, sr=self.asr_config.sample_rate)
except Exception as exc:
raise ValueError("Invalid or unsupported audio file.") from exc
duration = get_audio_duration(y=y, sr=sr)
do_split_audio = self.asr_config.allow_audio_chunking and (
self.asr_config.max_audio_clip_s is not None
and duration > self.asr_config.max_audio_clip_s
)
if not do_split_audio:
chunks = [y]
else:
assert self.asr_config.max_audio_clip_s is not None
assert self.asr_config.min_energy_split_window_size is not None
chunks = split_audio(
audio_data=y,
sample_rate=int(sr),
max_clip_duration_s=self.asr_config.max_audio_clip_s,
overlap_duration_s=self.asr_config.overlap_chunk_second,
min_energy_window_size=self.asr_config.min_energy_split_window_size,
)
if request.language is None and getattr(
self.model_cls, "supports_explicit_language_detection", False
):
# Auto-detect language from the first chunk.
request.language = await self._detect_language(
chunks[0], f"{request_id}-lang_detect"
)
parsed_prompts: list[DictPrompt] = []
for chunk in chunks:
stt_params = request.build_stt_params(
audio=chunk,
stt_config=self.asr_config,
model_config=self.model_config,
task_type=self.task_type,
)
prompt = self.model_cls.get_generation_prompt(stt_params)
parsed_prompt: DictPrompt
if request.response_format == "verbose_json":
parsed_prompt = parse_enc_dec_prompt(prompt)
parsed_prompt = self._preprocess_verbose_prompt(parsed_prompt)
else:
parsed_prompt = parse_model_prompt(self.model_config, prompt)
parsed_prompts.append(parsed_prompt)
engine_inputs = await self.renderer.render_cmpl_async(parsed_prompts)
return engine_inputs, duration
def _preprocess_verbose_prompt(self, prompt: EncoderDecoderDictPrompt):
dec_prompt = prompt["decoder_prompt"]
if not (isinstance(dec_prompt, dict) and "prompt" in dec_prompt):
raise VLLMValidationError(
"Expected decoder_prompt to contain text",
parameter="decoder_prompt",
value=type(dec_prompt).__name__,
)
dec_prompt["prompt"] = dec_prompt["prompt"].replace(
"<|notimestamps|>", "<|0.00|>"
)
return prompt
@staticmethod
def _get_decoder_prompt_len(engine_inputs: list[EngineInput]) -> int:
"""Get the length of the decoder prompt. Currently we need to offset
by the decoder prompt length when running beam search because the mm
encoder is not currently cached and runs on decode calls; because of
this, we need to make sure the redundant encoder calls won't exceed
the context :(
FIXME (Alex) - this will be removed in the very near future once the
encoder/decoder caching is implemented.
"""
input_len = 0
assert len(engine_inputs) > 0
first_input = engine_inputs[0]
if first_input.get("type") == "enc_dec":
first_input = cast(EncoderDecoderInput, first_input)
input_len = len(first_input["decoder_prompt"]["prompt_token_ids"])
return input_len
def _get_verbose_segments(
self,
tokens: tuple,
log_probs: FlatLogprobs | list[dict[int, Logprob]],
request: SpeechToTextRequest,
segment_class: type[SpeechToTextSegment],
start_time: float = 0,
) -> list[SpeechToTextSegment]:
"""
Convert tokens to verbose segments.
This method expects the model to produce
timestamps as tokens (similar to Whisper).
If the tokens do not include timestamp information,
the segments may not be generated correctly.
Note: No_speech_prob field is not supported
in this implementation and will be None. See docs for details.
"""
BASE_OFFSET = 0.02
init_token = self.tokenizer.encode("<|0.00|>", add_special_tokens=False)[0]
if tokens[-1] == self.tokenizer.eos_token_id:
tokens = tokens[:-1]
tokens_with_start = (init_token,) + tokens
segments: list[SpeechToTextSegment] = []
last_timestamp_start = 0
if tokens_with_start[-2] < init_token and tokens_with_start[-1] >= init_token:
tokens_with_start = tokens_with_start + (tokens_with_start[-1],)
avg_logprob = 0.0
for idx in range(1, len(tokens_with_start)):
# Timestamp tokens (e.g., <|0.00|>) are assumed to be sorted.
# If the ordering is violated, this slicing may produce incorrect results.
token = tokens_with_start[idx]
if token >= init_token and tokens_with_start[idx - 1] >= init_token:
sliced_timestamp_tokens = tokens_with_start[last_timestamp_start:idx]
start_timestamp = sliced_timestamp_tokens[0] - init_token
end_timestamp = sliced_timestamp_tokens[-1] - init_token
text = self.tokenizer.decode(sliced_timestamp_tokens[1:-1])
text_bytes = text.encode("utf-8")
casting_segment = cast(
SpeechToTextSegment,
segment_class(
id=len(segments),
seek=start_time,
start=start_time + BASE_OFFSET * start_timestamp,
end=start_time + BASE_OFFSET * end_timestamp,
temperature=request.temperature,
text=text,
# The compression ratio measures
# how compressible the generated text is.
# A higher ratio indicates more repetitive content,
# which is a strong sign of hallucination in outputs.
compression_ratio=len(text_bytes)
/ len(zlib.compress(text_bytes)),
tokens=sliced_timestamp_tokens[1:-1],
avg_logprob=avg_logprob / (idx - last_timestamp_start),
),
)
segments.append(casting_segment)
last_timestamp_start = idx
avg_logprob = 0
else:
avg_logprob += log_probs[idx - 1][token].logprob
return segments
async def _create_speech_to_text(
self,
audio_data: bytes,
request: SpeechToTextRequest,
raw_request: Request,
response_class: type[ResponseType],
stream_generator_method: Callable[..., AsyncGenerator[str, None]],
) -> T | V | AsyncGenerator[str, None] | ErrorResponse:
"""Base method for speech-to-text operations like transcription and
translation."""
if request.stream and request.use_beam_search:
return self.create_error_response(
"Streaming is not currently supported with beam search"
)
error_check_ret = await self._check_model(request)
if error_check_ret is not None:
return error_check_ret
if not request.model:
request.model = self.models.model_name()
# If the engine is dead, raise the engine's DEAD_ERROR.
# This is required for the streaming case, where we return a
# success status before we actually start generating text :).
if self.engine_client.errored:
raise self.engine_client.dead_error
if request.response_format not in ["text", "json", "verbose_json"]:
return self.create_error_response(
"Currently only support response_format: "
"`text`, `json` or `verbose_json`"
)
if (
request.response_format == "verbose_json"
and not self.model_cls.supports_segment_timestamp
):
return self.create_error_response(
f"Currently do not support verbose_json for {request.model}"
)
if request.response_format == "verbose_json" and request.stream:
return self.create_error_response(
"verbose_json format doesn't support streaming case"
)
request_id = f"{self.task_type}-{self._base_request_id(raw_request)}"
request_metadata = RequestResponseMetadata(request_id=request_id)
if raw_request:
raw_request.state.request_metadata = request_metadata
lora_request = self._maybe_get_adapters(request)
engine_inputs, duration_s = await self._preprocess_speech_to_text(
request=request,
audio_data=audio_data,
request_id=request_id,
)
# Schedule the request and get the result generator.
max_model_len = self.model_config.max_model_len
list_result_generator: list[AsyncGenerator[RequestOutput, None]] | None = None
input_len = (
OpenAISpeechToText._get_decoder_prompt_len(engine_inputs)
if request.use_beam_search
else 0
)
# Unlike most decoder-only models, whisper generation length is not
# constrained by the size of the input audio, which is mapped to a
# fixed-size log-mel-spectogram. Still, allow for fewer tokens to be
# generated by respecting the extra completion tokens arg.
max_tokens = get_max_tokens(
max_model_len,
request.max_completion_tokens,
input_len,
self.default_sampling_params,
)
if request.use_beam_search:
sampling_params = request.to_beam_search_params(
max_tokens, self.default_sampling_params
)
else:
sampling_params = request.to_sampling_params(
max_tokens,
self.default_sampling_params,
)
if request.response_format == "verbose_json":
sampling_params.logprobs = 1
engine_request_ids = [
request_id if len(engine_inputs) == 1 else f"{request_id}-{idx}"
for idx in range(len(engine_inputs))
]
list_result_generator = []
try:
for request_id_item, engine_input in zip(engine_request_ids, engine_inputs):
self._log_inputs(
request_id_item,
engine_input,
params=sampling_params,
lora_request=lora_request,
)
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
if isinstance(sampling_params, BeamSearchParams):
generator = self.beam_search(
prompt=engine_input,
params=sampling_params,
request_id=request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
else:
generator = self.engine_client.generate(
engine_input,
sampling_params,
request_id_item,
lora_request=lora_request,
trace_headers=trace_headers,
)
list_result_generator.append(generator)
except asyncio.CancelledError:
logger.info(
"Request %s cancelled; aborting %d transcription engine request(s).",
request_id,
len(engine_request_ids),
)
await asyncio.gather(
self.engine_client.abort(engine_request_ids),
return_exceptions=True,
)
raise
separator = asr_inter_chunk_separator(
request.language, self.model_cls.no_space_languages
)
if request.stream:
return stream_generator_method(
request,
list_result_generator,
request_id,
request_metadata,
duration_s,
separator,
)
# Non-streaming response.
try:
assert list_result_generator is not None
chunk_segment_parts: list[list[SpeechToTextSegment]] = [
[] for _ in list_result_generator
]
chunk_text_parts: list[list[str]] = [[] for _ in list_result_generator]
segments_types: dict[str, type[SpeechToTextSegment]] = {
"transcribe": TranscriptionSegment,
"translate": TranslationSegment,
}
segment_class: type[SpeechToTextSegment] = segments_types[self.task_type]
chunk_size_in_s = self.asr_config.max_audio_clip_s
if chunk_size_in_s is None:
assert len(list_result_generator) == 1, (
"`max_audio_clip_s` is set to None, audio cannot be chunked"
)
result_generator = merge_async_iterators(*list_result_generator)
async for idx, op in result_generator:
start_time = (
float(idx * chunk_size_in_s) if chunk_size_in_s is not None else 0.0
)
if request.response_format == "verbose_json":
assert op.outputs[0].logprobs
segments: list[SpeechToTextSegment] = self._get_verbose_segments(
tokens=tuple(op.outputs[0].token_ids),
segment_class=segment_class,
request=request,
start_time=start_time,
log_probs=op.outputs[0].logprobs,
)
chunk_segment_parts[idx].extend(segments)
chunk_text_parts[idx].extend([seg.text for seg in segments])
else:
raw_text = op.outputs[0].text
chunk_text_parts[idx].append(
self.model_cls.post_process_output(raw_text)
)
total_segments = [
segment
for segment_parts in chunk_segment_parts
for segment in segment_parts
]
text_parts = [text for text_part in chunk_text_parts for text in text_part]
text = separator.join(text_parts)
if self.task_type == "transcribe":
final_response: ResponseType
# add usage in TranscriptionResponse.
usage = {
"type": "duration",
# rounded up as per openAI specs
"seconds": int(math.ceil(duration_s)),
}
if request.response_format != "verbose_json":
final_response = cast(
T, TranscriptionResponse(text=text, usage=usage)
)
else:
final_response = cast(
V,
TranscriptionResponseVerbose(
text=text,
language=request.language,
duration=str(duration_s),
segments=total_segments,
),
)
else:
# no usage in response for translation task
if request.response_format != "verbose_json":
final_response = cast(T, TranslationResponse(text=text))
else:
final_response = cast(
V,
TranslationResponseVerbose(
text=text,
language=request.language,
duration=str(duration_s),
segments=total_segments,
),
)
return final_response
except asyncio.CancelledError:
logger.info(
"Request %s cancelled; aborting %d transcription engine request(s).",
request_id,
len(engine_request_ids),
)
await asyncio.gather(
self.engine_client.abort(engine_request_ids),
return_exceptions=True,
)
raise
async def _speech_to_text_stream_generator(
self,
request: SpeechToTextRequest,
list_result_generator: list[AsyncGenerator[RequestOutput, None]],
request_id: str,
request_metadata: RequestResponseMetadata,
audio_duration_s: float,
chunk_object_type: Literal["translation.chunk", "transcription.chunk"],
response_stream_choice_class: type[TranscriptionResponseStreamChoice]
| type[TranslationResponseStreamChoice],
stream_response_class: type[TranscriptionStreamResponse]
| type[TranslationStreamResponse],
separator: str,
) -> AsyncGenerator[str, None]:
created_time = int(time.time())
model_name = request.model
completion_tokens = 0
num_prompt_tokens = 0
include_usage = self.enable_force_include_usage or request.stream_include_usage
include_continuous_usage = (
request.stream_continuous_usage_stats
if include_usage and request.stream_continuous_usage_stats
else False
)
try:
for result_generator in list_result_generator:
beginning_of_chunk = True
async for res in result_generator:
# On first result.
if res.prompt_token_ids is not None:
num_prompt_tokens = len(res.prompt_token_ids)
if audio_tokens := self.model_cls.get_num_audio_tokens(
audio_duration_s, self.asr_config, self.model_config
):
num_prompt_tokens += audio_tokens
# We need to do it here, because if there are exceptions in
# the result_generator, it needs to be sent as the FIRST
# response (by the try...catch).
# Just one output (n=1) supported.
assert len(res.outputs) == 1
output = res.outputs[0]
# dont add separator to the first chunk
if (
result_generator is not list_result_generator[0]
and beginning_of_chunk
):
output.text = separator + output.text
beginning_of_chunk = False
# TODO: For models that output structured formats (e.g.,
# Qwen3-ASR with "language X<asr_text>" prefix), streaming
# would need buffering to strip the prefix properly since
# deltas may split the tag across chunks.
delta_message = DeltaMessage(content=output.text)
completion_tokens += len(output.token_ids)
if output.finish_reason is None:
# Still generating, send delta update.
choice_data = response_stream_choice_class(delta=delta_message)
else:
# Model is finished generating.
choice_data = response_stream_choice_class(
delta=delta_message,
finish_reason=output.finish_reason,
stop_reason=output.stop_reason,
)
chunk = stream_response_class(
id=request_id,
object=chunk_object_type,
created=created_time,
choices=[choice_data],
model=model_name,
)
# handle usage stats if requested & if continuous
if include_continuous_usage:
chunk.usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
)
data = chunk.model_dump_json(exclude_unset=True)
yield f"data: {data}\n\n"
# Once the final token is handled, if stream_options.include_usage
# is sent, send the usage.
if include_usage:
final_usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
)
final_usage_chunk = stream_response_class(
id=request_id,
object=chunk_object_type,
created=created_time,
choices=[],
model=model_name,
usage=final_usage,
)
final_usage_data = final_usage_chunk.model_dump_json(
exclude_unset=True, exclude_none=True
)
yield f"data: {final_usage_data}\n\n"
# report to FastAPI middleware aggregate usage across all choices
request_metadata.final_usage_info = UsageInfo(
prompt_tokens=num_prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=num_prompt_tokens + completion_tokens,
)
except Exception as e:
logger.exception("Error in %s stream generator.", self.task_type)
data = self.create_streaming_error_response(e)
yield f"data: {data}\n\n"
# Send the final done message after all response.n are finished
yield "data: [DONE]\n\n"
@@ -0,0 +1,79 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from fastapi import FastAPI
if TYPE_CHECKING:
from argparse import Namespace
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.logger import RequestLogger
from vllm.tasks import SupportedTask
else:
RequestLogger = object
def register_speech_to_text_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
):
if "realtime" in supported_tasks:
from .realtime.api_router import router as realtime_router
app.include_router(realtime_router)
if "transcription" in supported_tasks:
from .transcription.api_router import router as transcription_router
app.include_router(transcription_router)
from .translation.api_router import router as translation_router
app.include_router(translation_router)
def add_websocket_metrics_middleware(app: FastAPI):
from .realtime.metrics import WebSocketMetricsMiddleware
app.add_middleware(WebSocketMetricsMiddleware)
def init_speech_to_text_state(
engine_client: "EngineClient",
state: "State",
args: "Namespace",
request_logger: RequestLogger | None,
supported_tasks: tuple["SupportedTask", ...],
):
if "transcription" in supported_tasks:
from .transcription.serving import OpenAIServingTranscription
state.openai_serving_transcription = OpenAIServingTranscription(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
enable_force_include_usage=args.enable_force_include_usage,
)
from .translation.serving import OpenAIServingTranslation
state.openai_serving_translation = OpenAIServingTranslation(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
enable_force_include_usage=args.enable_force_include_usage,
)
if "realtime" in supported_tasks:
from .realtime.serving import OpenAIServingRealtime
state.openai_serving_realtime = OpenAIServingRealtime(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,39 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi import APIRouter, WebSocket
from vllm.logger import init_logger
from .connection import RealtimeConnection
logger = init_logger(__name__)
router = APIRouter()
@router.websocket("/v1/realtime")
async def realtime_endpoint(websocket: WebSocket):
"""WebSocket endpoint for realtime audio transcription.
Protocol:
1. Client connects to ws://host/v1/realtime
2. Server sends session.created event
3. Client optionally sends session.update with model/params
4. Client sends input_audio_buffer.commit when ready
5. Client sends input_audio_buffer.append events with base64 PCM16 chunks
6. Server processes and sends transcription.delta events
7. Server sends transcription.done with final text + usage
8. Repeat from step 5 for next utterance
9. Optionally, client sends input_audio_buffer.commit with final=True
to signal audio input is finished. Useful when streaming audio files
Audio format: PCM16, 16kHz, mono, base64-encoded
"""
app = websocket.app
serving = app.state.openai_serving_realtime
connection = RealtimeConnection(websocket, serving)
await connection.handle_connection()
@@ -0,0 +1,288 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import json
from collections.abc import AsyncGenerator
from http import HTTPStatus
from uuid import uuid4
import numpy as np
import pybase64 as base64
from fastapi import WebSocket
from starlette.websockets import WebSocketDisconnect
from vllm import envs
from vllm.entrypoints.openai.engine.protocol import ErrorResponse, UsageInfo
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from .protocol import (
ErrorEvent,
InputAudioBufferAppend,
InputAudioBufferCommit,
SessionCreated,
TranscriptionDelta,
TranscriptionDone,
)
from .serving import OpenAIServingRealtime
logger = init_logger(__name__)
class RealtimeConnection:
"""Manages WebSocket lifecycle and state for realtime transcription.
This class handles:
- WebSocket connection lifecycle (accept, receive, send, close)
- Event routing (session.update, append, commit)
- Audio buffering via asyncio.Queue
- Generation task management
- Error handling and cleanup
"""
def __init__(self, websocket: WebSocket, serving: OpenAIServingRealtime):
self.websocket = websocket
self.connection_id = f"ws-{uuid4()}"
self.serving = serving
self.audio_queue: asyncio.Queue[np.ndarray | None] = asyncio.Queue()
self.generation_task: asyncio.Task | None = None
self._is_connected = False
self._is_model_validated = False
self._max_audio_filesize_mb = envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB
async def handle_connection(self):
"""Main connection loop."""
await self.websocket.accept()
logger.debug("WebSocket connection accepted: %s", self.connection_id)
self._is_connected = True
# Send session created event
await self.send(SessionCreated())
try:
while True:
message = await self.websocket.receive_text()
try:
event = json.loads(message)
await self.handle_event(event)
except json.JSONDecodeError:
await self.send_error("Invalid JSON", "invalid_json")
except Exception as e:
logger.exception("Error handling event: %s", e)
await self.send_error(str(e), "processing_error")
except WebSocketDisconnect:
logger.debug("WebSocket disconnected: %s", self.connection_id)
self._is_connected = False
except Exception as e:
logger.exception("Unexpected error in connection: %s", e)
finally:
await self.cleanup()
def _check_model(self, model: str | None) -> None | ErrorResponse:
if self.serving._is_model_supported(model):
return None
return self.serving.create_error_response(
message=f"The model `{model}` does not exist.",
err_type="NotFoundError",
status_code=HTTPStatus.NOT_FOUND,
param="model",
)
async def handle_event(self, event: dict):
"""Route events to handlers.
Supported event types:
- session.update: Configure model
- input_audio_buffer.append: Add audio chunk to queue
- input_audio_buffer.commit: Start transcription generation
"""
event_type = event.get("type")
if event_type == "session.update":
logger.debug("Session updated: %s", event)
model = event.get("model")
if model is None:
await self.send_error("Missing required field: model", "invalid_event")
return
err = self._check_model(model)
if err is not None:
await self.send_error(err.error.message, "model_not_found")
return
self._is_model_validated = True
elif event_type == "input_audio_buffer.append":
append_event = InputAudioBufferAppend(**event)
try:
audio_bytes = base64.b64decode(append_event.audio)
# Convert PCM16 bytes to float32 numpy array
audio_array = (
np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32)
/ 32768.0
)
if len(audio_array) / 1024**2 > self._max_audio_filesize_mb:
raise VLLMValidationError(
"Maximum file size exceeded",
parameter="audio_filesize_mb",
value=len(audio_array) / 1024**2,
)
if len(audio_array) == 0:
raise VLLMValidationError("Can't process empty audio.")
# Put audio chunk in queue
self.audio_queue.put_nowait(audio_array)
except Exception as e:
logger.error("Failed to decode audio: %s", e)
await self.send_error("Invalid audio data", "invalid_audio")
elif event_type == "input_audio_buffer.commit":
if not self._is_model_validated:
err_msg = (
"Model not validated. Make sure to validate the"
" model by sending a session.update event."
)
await self.send_error(
err_msg,
"model_not_validated",
)
return
commit_event = InputAudioBufferCommit(**event)
# final signals that the audio is finished
if commit_event.final:
self.audio_queue.put_nowait(None)
else:
await self.start_generation()
else:
await self.send_error(f"Unknown event type: {event_type}", "unknown_event")
async def audio_stream_generator(self) -> AsyncGenerator[np.ndarray, None]:
"""Generator that yields audio chunks from the queue."""
while True:
audio_chunk = await self.audio_queue.get()
if audio_chunk is None: # Sentinel value to stop
break
yield audio_chunk
async def start_generation(self):
"""Start the transcription generation task."""
if self.generation_task is not None and not self.generation_task.done():
logger.warning("Generation already in progress, ignoring commit")
return
# Create audio stream generator
audio_stream = self.audio_stream_generator()
input_stream = asyncio.Queue[list[int]]()
# Transform to StreamingInput generator
streaming_input_gen = self.serving.transcribe_realtime(
audio_stream, input_stream
)
# Start generation task
self.generation_task = asyncio.create_task(
self._run_generation(streaming_input_gen, input_stream)
)
async def _run_generation(
self,
streaming_input_gen: AsyncGenerator,
input_stream: asyncio.Queue[list[int]],
):
"""Run the generation and stream results back to the client.
This method:
1. Creates sampling parameters from session config
2. Passes the streaming input generator to engine.generate()
3. Streams transcription.delta events as text is generated
4. Sends final transcription.done event with usage stats
5. Feeds generated token IDs back to input_stream for next iteration
6. Cleans up the audio queue
"""
request_id = f"rt-{self.connection_id}-{uuid4()}"
full_text = ""
prompt_token_ids_len: int = 0
completion_tokens_len: int = 0
try:
# Create sampling params
from vllm.sampling_params import RequestOutputKind, SamplingParams
sampling_params = SamplingParams.from_optional(
temperature=0.0,
max_tokens=self.serving.model_cls.realtime_max_tokens,
output_kind=RequestOutputKind.DELTA,
skip_clone=True,
)
# Pass the streaming input generator to the engine
# The engine will consume audio chunks as they arrive and
# stream back transcription results incrementally
result_gen = self.serving.engine_client.generate(
prompt=streaming_input_gen,
sampling_params=sampling_params,
request_id=request_id,
)
# Stream results back to client as they're generated
async for output in result_gen:
if output.outputs and len(output.outputs) > 0:
if not prompt_token_ids_len and output.prompt_token_ids:
prompt_token_ids_len = len(output.prompt_token_ids)
delta = output.outputs[0].text
full_text += delta
# append output to input
input_stream.put_nowait(list(output.outputs[0].token_ids))
await self.send(TranscriptionDelta(delta=delta))
completion_tokens_len += len(output.outputs[0].token_ids)
if not self._is_connected:
# finish because websocket connection was killed
break
usage = UsageInfo(
prompt_tokens=prompt_token_ids_len,
completion_tokens=completion_tokens_len,
total_tokens=prompt_token_ids_len + completion_tokens_len,
)
# Send final completion event
await self.send(TranscriptionDone(text=full_text, usage=usage))
# Clear queue for next utterance
while not self.audio_queue.empty():
self.audio_queue.get_nowait()
except Exception as e:
logger.exception("Error in generation: %s", e)
await self.send_error(str(e), "processing_error")
async def send(
self, event: SessionCreated | TranscriptionDelta | TranscriptionDone
):
"""Send event to client."""
data = event.model_dump_json()
await self.websocket.send_text(data)
async def send_error(self, message: str, code: str | None = None):
"""Send error event to client."""
error_event = ErrorEvent(error=message, code=code)
await self.websocket.send_text(error_event.model_dump_json())
async def cleanup(self):
"""Cleanup resources."""
# Signal audio stream to stop
self.audio_queue.put_nowait(None)
# Cancel generation task if running
if self.generation_task and not self.generation_task.done():
self.generation_task.cancel()
logger.debug("Connection cleanup complete: %s", self.connection_id)
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ASGI middleware for WebSocket Prometheus metrics.
Modeled after prometheus-fastapi-instrumentator, this middleware
transparently instruments WebSocket endpoints with standard metrics
without requiring changes to handler code.
NOTE: This module intentionally has zero vllm imports so that it can
be extracted into a standalone package (similar to
prometheus-fastapi-instrumentator) in the future. Please keep it that way.
"""
import time
from collections.abc import Awaitable
from prometheus_client import Counter, Gauge, Histogram
from starlette.types import ASGIApp, Message, Receive, Scope, Send
# Standard WebSocket metric names (not vllm-specific, following
# the same convention as prometheus-fastapi-instrumentator).
_active_sessions = Gauge(
name="vllm:websocket_connections_active",
documentation="Number of currently active WebSocket connections.",
multiprocess_mode="livesum",
)
_total_sessions = Counter(
name="vllm:websocket_connections_total",
documentation="Total number of WebSocket connections.",
)
_session_duration = Histogram(
name="vllm:websocket_connection_duration_seconds",
documentation="Duration of WebSocket connections in seconds.",
buckets=[0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1800],
)
class WebSocketMetricsMiddleware:
"""Pure ASGI middleware that instruments WebSocket connections.
Tracks active connections (gauge), total connections (counter),
and connection duration (histogram) for all WebSocket endpoints.
Usage::
app.add_middleware(WebSocketMetricsMiddleware)
"""
def __init__(self, app: ASGIApp) -> None:
self.app = app
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
if scope["type"] != "websocket":
return self.app(scope, receive, send)
return self._handle_websocket(scope, receive, send)
async def _handle_websocket(
self, scope: Scope, receive: Receive, send: Send
) -> None:
start_time: float | None = None
async def send_wrapper(message: Message) -> None:
nonlocal start_time
if message["type"] == "websocket.accept":
start_time = time.monotonic()
_active_sessions.inc()
_total_sessions.inc()
await send(message)
try:
await self.app(scope, receive, send_wrapper)
finally:
if start_time is not None:
_active_sessions.dec()
_session_duration.observe(time.monotonic() - start_time)
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import Literal
from pydantic import Field
from vllm.entrypoints.openai.engine.protocol import (
OpenAIBaseModel,
UsageInfo,
)
from vllm.utils import random_uuid
# Client -> Server Events
class InputAudioBufferAppend(OpenAIBaseModel):
"""Append audio chunk to buffer"""
type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append"
audio: str # base64-encoded PCM16 @ 16kHz
class InputAudioBufferCommit(OpenAIBaseModel):
"""Process accumulated audio buffer"""
type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit"
final: bool = False
# Server -> Client Events
class SessionUpdate(OpenAIBaseModel):
"""Configure session parameters"""
type: Literal["session.update"] = "session.update"
model: str | None = None
class SessionCreated(OpenAIBaseModel):
"""Connection established notification"""
type: Literal["session.created"] = "session.created"
id: str = Field(default_factory=lambda: f"sess-{random_uuid()}")
created: int = Field(default_factory=lambda: int(time.time()))
class TranscriptionDelta(OpenAIBaseModel):
"""Incremental transcription text"""
type: Literal["transcription.delta"] = "transcription.delta"
delta: str # Incremental text
class TranscriptionDone(OpenAIBaseModel):
"""Final transcription with usage stats"""
type: Literal["transcription.done"] = "transcription.done"
text: str # Complete transcription
usage: UsageInfo | None = None
class ErrorEvent(OpenAIBaseModel):
"""Error notification"""
type: Literal["error"] = "error"
error: str
code: str | None = None
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from collections.abc import AsyncGenerator
from functools import cached_property
from typing import Literal, cast
import numpy as np
from vllm.engine.protocol import EngineClient, StreamingInput
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.inputs import PromptType
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import SupportsRealtime
from vllm.renderers.inputs.preprocess import parse_model_prompt
logger = init_logger(__name__)
class OpenAIServingRealtime(OpenAIServing):
"""Realtime audio transcription service via WebSocket streaming.
Provides streaming audio-to-text transcription by transforming audio chunks
into StreamingInput objects that can be consumed by the engine.
"""
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
):
super().__init__(
engine_client=engine_client,
models=models,
request_logger=request_logger,
)
self.task_type: Literal["realtime"] = "realtime"
logger.info("OpenAIServingRealtime initialized for task: %s", self.task_type)
@cached_property
def model_cls(self) -> type[SupportsRealtime]:
"""Get the model class that supports transcription."""
from vllm.model_executor.model_loader import get_model_cls
model_cls = get_model_cls(self.model_config)
return cast(type[SupportsRealtime], model_cls)
async def transcribe_realtime(
self,
audio_stream: AsyncGenerator[np.ndarray, None],
input_stream: asyncio.Queue[list[int]],
) -> AsyncGenerator[StreamingInput, None]:
"""Transform audio stream into StreamingInput for engine.generate().
Args:
audio_stream: Async generator yielding float32 numpy audio arrays
input_stream: Queue containing context token IDs from previous
generation outputs. Used for autoregressive multi-turn
processing where each generation's output becomes the context
for the next iteration.
Yields:
StreamingInput objects containing audio prompts for the engine
"""
model_config = self.model_config
renderer = self.renderer
# mypy is being stupid
# TODO(Patrick) - fix this
stream_input_iter = cast(
AsyncGenerator[PromptType, None],
self.model_cls.buffer_realtime_audio(
audio_stream, input_stream, model_config
),
)
async for prompt in stream_input_iter:
parsed_prompt = parse_model_prompt(model_config, prompt)
(engine_input,) = await renderer.render_cmpl_async([parsed_prompt])
yield StreamingInput(prompt=engine_input)
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from typing import Annotated
from fastapi import APIRouter, Form, Request
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.utils import (
load_aware_call,
with_cancellation,
)
from vllm.logger import init_logger
from .protocol import TranscriptionRequest, TranscriptionResponseVariant
from .serving import OpenAIServingTranscription
logger = init_logger(__name__)
router = APIRouter()
def transcription(request: Request) -> OpenAIServingTranscription:
return request.app.state.openai_serving_transcription
@router.post(
"/v1/audio/transcriptions",
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_transcriptions(
raw_request: Request, request: Annotated[TranscriptionRequest, Form()]
):
handler = transcription(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Transcriptions API")
audio_data = await request.file.read()
generator = await handler.create_transcription(audio_data, request, raw_request)
if isinstance(generator, ErrorResponse):
return JSONResponse(
content=generator.model_dump(), status_code=generator.error.code
)
elif isinstance(generator, TranscriptionResponseVariant):
return JSONResponse(content=generator.model_dump())
return StreamingResponse(content=generator, media_type="text/event-stream")
@@ -0,0 +1,405 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import time
from http import HTTPStatus
from typing import TYPE_CHECKING, Literal, TypeAlias
from fastapi import HTTPException, UploadFile
from pydantic import (
Field,
model_validator,
)
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
OpenAIBaseModel,
UsageInfo,
)
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.sampling_params import (
BeamSearchParams,
RequestOutputKind,
SamplingParams,
)
from vllm.utils import random_uuid
from ..base.protocol import _LONG_INFO, AudioResponseFormat
if TYPE_CHECKING:
import numpy as np
from vllm.config import ModelConfig, SpeechToTextConfig
logger = init_logger(__name__)
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
delta: DeltaMessage
finish_reason: str | None = None
stop_reason: int | str | None = None
class TranscriptionStreamResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
object: Literal["transcription.chunk"] = "transcription.chunk"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
choices: list[TranscriptionResponseStreamChoice]
usage: UsageInfo | None = Field(default=None)
class TranscriptionRequest(OpenAIBaseModel):
# Ordered by official OpenAI API documentation
# https://platform.openai.com/docs/api-reference/audio/createTranscription
file: UploadFile
"""
The audio file object (not file name) to transcribe, in one of these
formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
"""
model: str | None = None
"""ID of the model to use.
"""
language: str | None = None
"""The language of the input audio.
Supplying the input language in
[ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
will improve accuracy and latency.
"""
hotwords: str | None = None
"""
hotwords refers to a list of important words or phrases that the model
should pay extra attention to during transcription.
"""
prompt: str = Field(default="")
"""An optional text to guide the model's style or continue a previous audio
segment.
The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
should match the audio language.
"""
response_format: AudioResponseFormat = Field(default="json")
"""
The format of the output, in one of these options: `json`, `text`, `srt`,
`verbose_json`, or `vtt`.
"""
## TODO (varun) : Support if set to 0, certain thresholds are met !!
timestamp_granularities: list[Literal["word", "segment"]] = Field(
alias="timestamp_granularities[]", default=[]
)
"""The timestamp granularities to populate for this transcription.
`response_format` must be set `verbose_json` to use timestamp granularities.
Either or both of these options are supported: `word`, or `segment`. Note:
There is no additional latency for segment timestamps, but generating word
timestamps incurs additional latency.
"""
stream: bool | None = False
"""When set, it will enable output to be streamed in a similar fashion
as the Chat Completion endpoint.
"""
# --8<-- [start:transcription-extra-params]
# Flattened stream option to simplify form data.
stream_include_usage: bool | None = False
stream_continuous_usage_stats: bool | None = False
vllm_xargs: dict[str, str | int | float | bool] | None = Field(
default=None,
description=(
"Additional request parameters with string or "
"numeric values, used by custom extensions."
),
)
# --8<-- [end:transcription-extra-params]
to_language: str | None = None
"""The language of the output audio we transcribe to.
Please note that this is not currently used by supported models at this
time, but it is a placeholder for future use, matching translation api.
"""
# --8<-- [start:transcription-sampling-params]
use_beam_search: bool = False
"""Whether or not beam search should be used."""
n: int = 1
"""The number of beams to be used in beam search."""
length_penalty: float = 1.0
"""Length penalty to be used for beam search."""
include_stop_str_in_output: bool = False
"""Whether to include the stop strings in output text."""
temperature: float = Field(default=0.0)
"""The sampling temperature, between 0 and 1.
Higher values like 0.8 will make the output more random, while lower values
like 0.2 will make it more focused / deterministic. If set to 0, the model
will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
to automatically increase the temperature until certain thresholds are hit.
"""
top_p: float | None = None
"""Enables nucleus (top-p) sampling, where tokens are selected from the
smallest possible set whose cumulative probability exceeds `p`.
"""
top_k: int | None = None
"""Limits sampling to the `k` most probable tokens at each step."""
min_p: float | None = None
"""Filters out tokens with a probability lower than `min_p`, ensuring a
minimum likelihood threshold during sampling.
"""
seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
"""The seed to use for sampling."""
frequency_penalty: float | None = 0.0
"""The frequency penalty to use for sampling."""
repetition_penalty: float | None = None
"""The repetition penalty to use for sampling."""
presence_penalty: float | None = 0.0
"""The presence penalty to use for sampling."""
max_completion_tokens: int | None = None
"""The maximum number of tokens to generate."""
# --8<-- [end:transcription-sampling-params]
# Default sampling parameters for transcription requests.
_DEFAULT_SAMPLING_PARAMS: dict = {
"repetition_penalty": 1.0,
"temperature": 1.0,
"top_p": 1.0,
"top_k": 0,
"min_p": 0.0,
}
def build_stt_params(
self,
audio: "np.ndarray",
stt_config: "SpeechToTextConfig",
model_config: "ModelConfig",
task_type: str,
) -> SpeechToTextParams:
return SpeechToTextParams(
audio=audio,
stt_config=stt_config,
model_config=model_config,
language=self.language,
task_type=task_type,
request_prompt=self.prompt,
to_language=self.to_language,
hotwords=self.hotwords,
)
def to_beam_search_params(
self,
default_max_tokens: int,
default_sampling_params: dict | None = None,
) -> BeamSearchParams:
if default_sampling_params is None:
default_sampling_params = {}
max_tokens = default_max_tokens
n = self.n if self.n is not None else 1
# NOTE: Temp 0 is a different fallback than completions
if (temperature := self.temperature) is None:
temperature = default_sampling_params.get("temperature", 0)
return BeamSearchParams(
beam_width=n,
max_tokens=max_tokens,
temperature=temperature,
length_penalty=self.length_penalty,
include_stop_str_in_output=self.include_stop_str_in_output,
)
def to_sampling_params(
self, default_max_tokens: int, default_sampling_params: dict | None = None
) -> SamplingParams:
max_tokens = default_max_tokens
if default_sampling_params is None:
default_sampling_params = {}
# Default parameters
if (temperature := self.temperature) is None:
temperature = default_sampling_params.get(
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
)
if (top_p := self.top_p) is None:
top_p = default_sampling_params.get(
"top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
)
if (top_k := self.top_k) is None:
top_k = default_sampling_params.get(
"top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
)
if (min_p := self.min_p) is None:
min_p = default_sampling_params.get(
"min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
)
if (repetition_penalty := self.repetition_penalty) is None:
repetition_penalty = default_sampling_params.get(
"repetition_penalty",
self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
)
return SamplingParams.from_optional(
temperature=temperature,
max_tokens=max_tokens,
seed=self.seed,
top_p=top_p,
top_k=top_k,
min_p=min_p,
frequency_penalty=self.frequency_penalty,
repetition_penalty=repetition_penalty,
presence_penalty=self.presence_penalty,
output_kind=RequestOutputKind.DELTA
if self.stream
else RequestOutputKind.FINAL_ONLY,
extra_args=self.vllm_xargs,
skip_clone=True, # Created fresh per request, safe to skip clone
)
@model_validator(mode="before")
@classmethod
def validate_transcription_request(cls, data):
if isinstance(data.get("file"), str):
raise HTTPException(
status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
detail="Expected 'file' to be a file-like object, not 'str'.",
)
stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
stream = data.get("stream", False)
if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
# Find which specific stream option was set
invalid_param = next(
(so for so in stream_opts if data.get(so, False)),
"stream_include_usage",
)
raise VLLMValidationError(
"Stream options can only be defined when `stream=True`.",
parameter=invalid_param,
)
# Parse vllm_xargs from JSON string (form data sends it as a string)
xargs = data.get("vllm_xargs")
if isinstance(xargs, str):
try:
data["vllm_xargs"] = json.loads(xargs)
except json.JSONDecodeError as e:
raise VLLMValidationError(
f"Failed to parse vllm_xargs. Must be valid JSON: {e}",
parameter="vllm_xargs",
) from e
return data
# Transcription response objects
class TranscriptionUsageAudio(OpenAIBaseModel):
type: Literal["duration"] = "duration"
seconds: int
class TranscriptionResponse(OpenAIBaseModel):
text: str
"""The transcribed text."""
usage: TranscriptionUsageAudio
class TranscriptionWord(OpenAIBaseModel):
end: float
"""End time of the word in seconds."""
start: float
"""Start time of the word in seconds."""
word: str
"""The text content of the word."""
class TranscriptionSegment(OpenAIBaseModel):
id: int
"""Unique identifier of the segment."""
avg_logprob: float
"""Average logprob of the segment.
If the value is lower than -1, consider the logprobs failed.
"""
compression_ratio: float
"""Compression ratio of the segment.
If the value is greater than 2.4, consider the compression failed.
"""
end: float
"""End time of the segment in seconds."""
no_speech_prob: float | None = None
"""Probability of no speech in the segment.
If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
this segment silent.
"""
seek: int
"""Seek offset of the segment."""
start: float
"""Start time of the segment in seconds."""
temperature: float
"""Temperature parameter used for generating the segment."""
text: str
"""Text content of the segment."""
tokens: list[int]
"""Array of token IDs for the text content."""
class TranscriptionResponseVerbose(OpenAIBaseModel):
duration: str
"""The duration of the input audio."""
language: str
"""The language of the input audio."""
text: str
"""The transcribed text."""
segments: list[TranscriptionSegment] | None = None
"""Segments of the transcribed text and their corresponding details."""
words: list[TranscriptionWord] | None = None
"""Extracted words and their corresponding timestamps."""
TranscriptionResponseVariant: TypeAlias = (
TranscriptionResponse | TranscriptionResponseVerbose
)
@@ -0,0 +1,99 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import AsyncGenerator
from fastapi import Request
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
RequestResponseMetadata,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.logger import init_logger
from vllm.outputs import RequestOutput
from ..base.serving import OpenAISpeechToText
from .protocol import (
TranscriptionRequest,
TranscriptionResponse,
TranscriptionResponseStreamChoice,
TranscriptionResponseVerbose,
TranscriptionStreamResponse,
)
logger = init_logger(__name__)
class OpenAIServingTranscription(OpenAISpeechToText):
"""Handles transcription requests."""
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
enable_force_include_usage: bool = False,
):
super().__init__(
engine_client=engine_client,
models=models,
request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids,
task_type="transcribe",
enable_force_include_usage=enable_force_include_usage,
)
async def create_transcription(
self,
audio_data: bytes,
request: TranscriptionRequest,
raw_request: Request | None = None,
) -> (
TranscriptionResponse
| TranscriptionResponseVerbose
| AsyncGenerator[str, None]
| ErrorResponse
):
"""Transcription API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/audio/createTranscription
for the API specification. This API mimics the OpenAI transcription API.
"""
return await self._create_speech_to_text(
audio_data=audio_data,
request=request,
raw_request=raw_request,
response_class=(
TranscriptionResponseVerbose
if request.response_format == "verbose_json"
else TranscriptionResponse
),
stream_generator_method=self.transcription_stream_generator,
)
async def transcription_stream_generator(
self,
request: TranscriptionRequest,
result_generator: list[AsyncGenerator[RequestOutput, None]],
request_id: str,
request_metadata: RequestResponseMetadata,
audio_duration_s: float,
separator: str,
) -> AsyncGenerator[str, None]:
generator = self._speech_to_text_stream_generator(
request=request,
list_result_generator=result_generator,
request_id=request_id,
request_metadata=request_metadata,
audio_duration_s=audio_duration_s,
chunk_object_type="transcription.chunk",
response_stream_choice_class=TranscriptionResponseStreamChoice,
stream_response_class=TranscriptionStreamResponse,
separator=separator,
)
async for chunk in generator:
yield chunk
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from typing import Annotated
from fastapi import APIRouter, Form, Request
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.utils import (
load_aware_call,
with_cancellation,
)
from vllm.logger import init_logger
from .protocol import TranslationRequest, TranslationResponseVariant
from .serving import OpenAIServingTranslation
logger = init_logger(__name__)
router = APIRouter()
def translation(request: Request) -> OpenAIServingTranslation:
return request.app.state.openai_serving_translation
@router.post(
"/v1/audio/translations",
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.UNPROCESSABLE_ENTITY.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_translations(
request: Annotated[TranslationRequest, Form()], raw_request: Request
):
handler = translation(raw_request)
if handler is None:
raise NotImplementedError("The model does not support Translations API")
audio_data = await request.file.read()
generator = await handler.create_translation(audio_data, request, raw_request)
if isinstance(generator, ErrorResponse):
return JSONResponse(
content=generator.model_dump(), status_code=generator.error.code
)
elif isinstance(generator, TranslationResponseVariant):
return JSONResponse(content=generator.model_dump())
return StreamingResponse(content=generator, media_type="text/event-stream")
@@ -0,0 +1,308 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import TYPE_CHECKING, Literal, TypeAlias
from fastapi import UploadFile
from pydantic import (
Field,
model_validator,
)
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
OpenAIBaseModel,
UsageInfo,
)
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.sampling_params import (
BeamSearchParams,
RequestOutputKind,
SamplingParams,
)
from vllm.utils import random_uuid
from ..base.protocol import _LONG_INFO, AudioResponseFormat
if TYPE_CHECKING:
import numpy as np
from vllm.config import ModelConfig, SpeechToTextConfig
logger = init_logger(__name__)
class TranslationResponseStreamChoice(OpenAIBaseModel):
delta: DeltaMessage
finish_reason: str | None = None
stop_reason: int | str | None = None
class TranslationStreamResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
object: Literal["translation.chunk"] = "translation.chunk"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
choices: list[TranslationResponseStreamChoice]
usage: UsageInfo | None = Field(default=None)
class TranslationRequest(OpenAIBaseModel):
# Ordered by official OpenAI API documentation
# https://platform.openai.com/docs/api-reference/audio/createTranslation
file: UploadFile
"""
The audio file object (not file name) to translate, in one of these
formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
"""
model: str | None = None
"""ID of the model to use.
"""
prompt: str = Field(default="")
"""An optional text to guide the model's style or continue a previous audio
segment.
The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
should match the audio language.
"""
response_format: AudioResponseFormat = Field(default="json")
"""
The format of the output, in one of these options: `json`, `text`, `srt`,
`verbose_json`, or `vtt`.
"""
# TODO support additional sampling parameters
# --8<-- [start:translation-sampling-params]
use_beam_search: bool = False
"""Whether or not beam search should be used."""
n: int = 1
"""The number of beams to be used in beam search."""
length_penalty: float = 1.0
"""Length penalty to be used for beam search."""
include_stop_str_in_output: bool = False
"""Whether to include the stop strings in output text."""
seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
"""The seed to use for sampling."""
temperature: float = Field(default=0.0)
"""The sampling temperature, between 0 and 1.
Higher values like 0.8 will make the output more random, while lower values
like 0.2 will make it more focused / deterministic. If set to 0, the model
will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
to automatically increase the temperature until certain thresholds are hit.
"""
# --8<-- [end:translation-sampling-params]
# --8<-- [start:translation-extra-params]
language: str | None = None
"""The language of the input audio we translate from.
Supplying the input language in
[ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
will improve accuracy.
"""
hotwords: str | None = None
"""
hotwords refers to a list of important words or phrases that the model
should pay extra attention to during transcription.
"""
to_language: str | None = None
"""The language of the input audio we translate to.
Please note that this is not supported by all models, refer to the specific
model documentation for more details.
For instance, Whisper only supports `to_language=en`.
"""
stream: bool | None = False
"""Custom field not present in the original OpenAI definition. When set,
it will enable output to be streamed in a similar fashion as the Chat
Completion endpoint.
"""
# Flattened stream option to simplify form data.
stream_include_usage: bool | None = False
stream_continuous_usage_stats: bool | None = False
max_completion_tokens: int | None = None
"""The maximum number of tokens to generate."""
# --8<-- [end:translation-extra-params]
# Default sampling parameters for translation requests.
_DEFAULT_SAMPLING_PARAMS: dict = {
"temperature": 0,
}
def build_stt_params(
self,
audio: "np.ndarray",
stt_config: "SpeechToTextConfig",
model_config: "ModelConfig",
task_type: str,
) -> SpeechToTextParams:
return SpeechToTextParams(
audio=audio,
stt_config=stt_config,
model_config=model_config,
language=self.language,
task_type=task_type,
request_prompt=self.prompt,
to_language=self.to_language,
hotwords=self.hotwords,
)
def to_beam_search_params(
self,
default_max_tokens: int,
default_sampling_params: dict | None = None,
) -> BeamSearchParams:
if default_sampling_params is None:
default_sampling_params = {}
max_tokens = default_max_tokens
n = self.n if self.n is not None else 1
# NOTE: Temp 0 is a different fallback than completions
if (temperature := self.temperature) is None:
temperature = default_sampling_params.get("temperature", 0)
return BeamSearchParams(
beam_width=n,
max_tokens=max_tokens,
temperature=temperature,
length_penalty=self.length_penalty,
include_stop_str_in_output=self.include_stop_str_in_output,
)
def to_sampling_params(
self, default_max_tokens: int, default_sampling_params: dict | None = None
) -> SamplingParams:
max_tokens = default_max_tokens
if default_sampling_params is None:
default_sampling_params = {}
# Default parameters
if (temperature := self.temperature) is None:
temperature = default_sampling_params.get(
"temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
)
return SamplingParams.from_optional(
temperature=temperature,
max_tokens=max_tokens,
seed=self.seed,
output_kind=RequestOutputKind.DELTA
if self.stream
else RequestOutputKind.FINAL_ONLY,
skip_clone=True, # Created fresh per request, safe to skip clone
)
@model_validator(mode="before")
@classmethod
def validate_stream_options(cls, data):
stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
stream = data.get("stream", False)
if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
# Find which specific stream option was set
invalid_param = next(
(so for so in stream_opts if data.get(so, False)),
"stream_include_usage",
)
raise VLLMValidationError(
"Stream options can only be defined when `stream=True`.",
parameter=invalid_param,
)
return data
# Translation response objects
class TranslationResponse(OpenAIBaseModel):
text: str
"""The translated text."""
class TranslationWord(OpenAIBaseModel):
end: float
"""End time of the word in seconds."""
start: float
"""Start time of the word in seconds."""
word: str
"""The text content of the word."""
class TranslationSegment(OpenAIBaseModel):
id: int
"""Unique identifier of the segment."""
avg_logprob: float
"""Average logprob of the segment.
If the value is lower than -1, consider the logprobs failed.
"""
compression_ratio: float
"""Compression ratio of the segment.
If the value is greater than 2.4, consider the compression failed.
"""
end: float
"""End time of the segment in seconds."""
no_speech_prob: float | None = None
"""Probability of no speech in the segment.
If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
this segment silent.
"""
seek: int
"""Seek offset of the segment."""
start: float
"""Start time of the segment in seconds."""
temperature: float
"""Temperature parameter used for generating the segment."""
text: str
"""Text content of the segment."""
tokens: list[int]
"""Array of token IDs for the text content."""
class TranslationResponseVerbose(OpenAIBaseModel):
duration: str
"""The duration of the input audio."""
language: str
"""The language of the input audio."""
text: str
"""The translated text."""
segments: list[TranslationSegment] | None = None
"""Segments of the translated text and their corresponding details."""
words: list[TranslationWord] | None = None
"""Extracted words and their corresponding timestamps."""
TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose
@@ -0,0 +1,99 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import AsyncGenerator
from fastapi import Request
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
RequestResponseMetadata,
)
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.logger import init_logger
from vllm.outputs import RequestOutput
from ..base.serving import OpenAISpeechToText
from .protocol import (
TranslationRequest,
TranslationResponse,
TranslationResponseStreamChoice,
TranslationResponseVerbose,
TranslationStreamResponse,
)
logger = init_logger(__name__)
class OpenAIServingTranslation(OpenAISpeechToText):
"""Handles translation requests."""
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
return_tokens_as_token_ids: bool = False,
enable_force_include_usage: bool = False,
):
super().__init__(
engine_client=engine_client,
models=models,
request_logger=request_logger,
return_tokens_as_token_ids=return_tokens_as_token_ids,
task_type="translate",
enable_force_include_usage=enable_force_include_usage,
)
async def create_translation(
self,
audio_data: bytes,
request: TranslationRequest,
raw_request: Request | None = None,
) -> (
TranslationResponse
| TranslationResponseVerbose
| AsyncGenerator[str, None]
| ErrorResponse
):
"""Translation API similar to OpenAI's API.
See https://platform.openai.com/docs/api-reference/audio/createTranslation
for the API specification. This API mimics the OpenAI translation API.
"""
return await self._create_speech_to_text(
audio_data=audio_data,
request=request,
raw_request=raw_request,
response_class=(
TranslationResponseVerbose
if request.response_format == "verbose_json"
else TranslationResponse
),
stream_generator_method=self.translation_stream_generator,
)
async def translation_stream_generator(
self,
request: TranslationRequest,
result_generator: list[AsyncGenerator[RequestOutput, None]],
request_id: str,
request_metadata: RequestResponseMetadata,
audio_duration_s: float,
separator: str,
) -> AsyncGenerator[str, None]:
generator = self._speech_to_text_stream_generator(
request=request,
list_result_generator=result_generator,
request_id=request_id,
request_metadata=request_metadata,
audio_duration_s=audio_duration_s,
chunk_object_type="translation.chunk",
response_stream_choice_class=TranslationResponseStreamChoice,
stream_response_class=TranslationStreamResponse,
separator=separator,
)
async for chunk in generator:
yield chunk