[Audio] Bundle get_generation_prompt() params into SpeechToTextParams (#36268)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
Ekagra Ranjan
2026-04-22 12:24:18 +08:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Cyrus Leung
parent 6d09769700
commit 4eafc72928
18 changed files with 214 additions and 161 deletions
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import time
from http import HTTPStatus
from typing import Literal, TypeAlias
from typing import TYPE_CHECKING, Literal, TypeAlias
import torch
from fastapi import HTTPException, UploadFile
@@ -12,6 +13,7 @@ from pydantic import (
model_validator,
)
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
OpenAIBaseModel,
@@ -26,6 +28,11 @@ from vllm.sampling_params import (
)
from vllm.utils import random_uuid
if TYPE_CHECKING:
import numpy as np
from vllm.config import ModelConfig, SpeechToTextConfig
logger = init_logger(__name__)
_LONG_INFO = torch.iinfo(torch.long)
@@ -183,6 +190,23 @@ class TranscriptionRequest(OpenAIBaseModel):
"min_p": 0.0,
}
def build_stt_params(
self,
audio: "np.ndarray",
stt_config: "SpeechToTextConfig",
model_config: "ModelConfig",
task_type: str,
) -> SpeechToTextParams:
return SpeechToTextParams(
audio=audio,
stt_config=stt_config,
model_config=model_config,
language=self.language,
task_type=task_type,
request_prompt=self.prompt,
to_language=self.to_language,
)
def to_beam_search_params(
self,
default_max_tokens: int,
@@ -277,6 +301,17 @@ class TranscriptionRequest(OpenAIBaseModel):
parameter=invalid_param,
)
# Parse vllm_xargs from JSON string (form data sends it as a string)
xargs = data.get("vllm_xargs")
if isinstance(xargs, str):
try:
data["vllm_xargs"] = json.loads(xargs)
except json.JSONDecodeError as e:
raise VLLMValidationError(
f"Failed to parse vllm_xargs. Must be valid JSON: {e}",
parameter="vllm_xargs",
) from e
return data
@@ -472,6 +507,23 @@ class TranslationRequest(OpenAIBaseModel):
"temperature": 0,
}
def build_stt_params(
self,
audio: "np.ndarray",
stt_config: "SpeechToTextConfig",
model_config: "ModelConfig",
task_type: str,
) -> SpeechToTextParams:
return SpeechToTextParams(
audio=audio,
stt_config=stt_config,
model_config=model_config,
language=self.language,
task_type=task_type,
request_prompt=self.prompt,
to_language=self.to_language,
)
def to_beam_search_params(
self,
default_max_tokens: int,
@@ -184,9 +184,8 @@ class OpenAISpeechToText(OpenAIServing):
request_id: str,
) -> tuple[list[EngineInput], float]:
# Validate request
language = self.model_cls.validate_language(request.language)
# Skip to_language validation to avoid extra logging for Whisper.
to_language = (
request.language = self.model_cls.validate_language(request.language)
request.to_language = (
self.model_cls.validate_language(request.to_language)
if request.to_language
else None
@@ -229,28 +228,23 @@ class OpenAISpeechToText(OpenAIServing):
min_energy_window_size=self.asr_config.min_energy_split_window_size,
)
if language is None and getattr(
if request.language is None and getattr(
self.model_cls, "supports_explicit_language_detection", False
):
# Auto-detect language from the first chunk.
language = await self._detect_language(
request.language = await self._detect_language(
chunks[0], f"{request_id}-lang_detect"
)
request.language = language
parsed_prompts: list[DictPrompt] = []
for chunk in chunks:
# The model has control over the construction, as long as it
# returns a valid PromptType.
prompt = self.model_cls.get_generation_prompt(
stt_params = request.build_stt_params(
audio=chunk,
stt_config=self.asr_config,
model_config=self.model_config,
language=language,
task_type=self.task_type,
request_prompt=request.prompt,
to_language=to_language,
)
prompt = self.model_cls.get_generation_prompt(stt_params)
parsed_prompt: DictPrompt
if request.response_format == "verbose_json":