[Audio] Bundle get_generation_prompt() params into SpeechToTextParams (#36268)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Cyrus Leung
parent
6d09769700
commit
4eafc72928
@@ -1,9 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
from typing import Literal, TypeAlias
|
||||
from typing import TYPE_CHECKING, Literal, TypeAlias
|
||||
|
||||
import torch
|
||||
from fastapi import HTTPException, UploadFile
|
||||
@@ -12,6 +13,7 @@ from pydantic import (
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
OpenAIBaseModel,
|
||||
@@ -26,6 +28,11 @@ from vllm.sampling_params import (
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_LONG_INFO = torch.iinfo(torch.long)
|
||||
|
||||
@@ -183,6 +190,23 @@ class TranscriptionRequest(OpenAIBaseModel):
|
||||
"min_p": 0.0,
|
||||
}
|
||||
|
||||
def build_stt_params(
|
||||
self,
|
||||
audio: "np.ndarray",
|
||||
stt_config: "SpeechToTextConfig",
|
||||
model_config: "ModelConfig",
|
||||
task_type: str,
|
||||
) -> SpeechToTextParams:
|
||||
return SpeechToTextParams(
|
||||
audio=audio,
|
||||
stt_config=stt_config,
|
||||
model_config=model_config,
|
||||
language=self.language,
|
||||
task_type=task_type,
|
||||
request_prompt=self.prompt,
|
||||
to_language=self.to_language,
|
||||
)
|
||||
|
||||
def to_beam_search_params(
|
||||
self,
|
||||
default_max_tokens: int,
|
||||
@@ -277,6 +301,17 @@ class TranscriptionRequest(OpenAIBaseModel):
|
||||
parameter=invalid_param,
|
||||
)
|
||||
|
||||
# Parse vllm_xargs from JSON string (form data sends it as a string)
|
||||
xargs = data.get("vllm_xargs")
|
||||
if isinstance(xargs, str):
|
||||
try:
|
||||
data["vllm_xargs"] = json.loads(xargs)
|
||||
except json.JSONDecodeError as e:
|
||||
raise VLLMValidationError(
|
||||
f"Failed to parse vllm_xargs. Must be valid JSON: {e}",
|
||||
parameter="vllm_xargs",
|
||||
) from e
|
||||
|
||||
return data
|
||||
|
||||
|
||||
@@ -472,6 +507,23 @@ class TranslationRequest(OpenAIBaseModel):
|
||||
"temperature": 0,
|
||||
}
|
||||
|
||||
def build_stt_params(
|
||||
self,
|
||||
audio: "np.ndarray",
|
||||
stt_config: "SpeechToTextConfig",
|
||||
model_config: "ModelConfig",
|
||||
task_type: str,
|
||||
) -> SpeechToTextParams:
|
||||
return SpeechToTextParams(
|
||||
audio=audio,
|
||||
stt_config=stt_config,
|
||||
model_config=model_config,
|
||||
language=self.language,
|
||||
task_type=task_type,
|
||||
request_prompt=self.prompt,
|
||||
to_language=self.to_language,
|
||||
)
|
||||
|
||||
def to_beam_search_params(
|
||||
self,
|
||||
default_max_tokens: int,
|
||||
|
||||
@@ -184,9 +184,8 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
request_id: str,
|
||||
) -> tuple[list[EngineInput], float]:
|
||||
# Validate request
|
||||
language = self.model_cls.validate_language(request.language)
|
||||
# Skip to_language validation to avoid extra logging for Whisper.
|
||||
to_language = (
|
||||
request.language = self.model_cls.validate_language(request.language)
|
||||
request.to_language = (
|
||||
self.model_cls.validate_language(request.to_language)
|
||||
if request.to_language
|
||||
else None
|
||||
@@ -229,28 +228,23 @@ class OpenAISpeechToText(OpenAIServing):
|
||||
min_energy_window_size=self.asr_config.min_energy_split_window_size,
|
||||
)
|
||||
|
||||
if language is None and getattr(
|
||||
if request.language is None and getattr(
|
||||
self.model_cls, "supports_explicit_language_detection", False
|
||||
):
|
||||
# Auto-detect language from the first chunk.
|
||||
language = await self._detect_language(
|
||||
request.language = await self._detect_language(
|
||||
chunks[0], f"{request_id}-lang_detect"
|
||||
)
|
||||
request.language = language
|
||||
|
||||
parsed_prompts: list[DictPrompt] = []
|
||||
for chunk in chunks:
|
||||
# The model has control over the construction, as long as it
|
||||
# returns a valid PromptType.
|
||||
prompt = self.model_cls.get_generation_prompt(
|
||||
stt_params = request.build_stt_params(
|
||||
audio=chunk,
|
||||
stt_config=self.asr_config,
|
||||
model_config=self.model_config,
|
||||
language=language,
|
||||
task_type=self.task_type,
|
||||
request_prompt=request.prompt,
|
||||
to_language=to_language,
|
||||
)
|
||||
prompt = self.model_cls.get_generation_prompt(stt_params)
|
||||
|
||||
parsed_prompt: DictPrompt
|
||||
if request.response_format == "verbose_json":
|
||||
|
||||
Reference in New Issue
Block a user