[Audio] Bundle get_generation_prompt() params into SpeechToTextParams (#36268)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Cyrus Leung
parent
6d09769700
commit
4eafc72928
@@ -3,7 +3,6 @@
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import AutoModel, BatchFeature
|
||||
@@ -19,6 +18,7 @@ from transformers.models.siglip import SiglipImageProcessorFast
|
||||
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -769,21 +769,17 @@ class Gemma3nForConditionalGeneration(
|
||||
raise ValueError(f"Unsupported modality: {modality}")
|
||||
|
||||
@classmethod
|
||||
def get_generation_prompt(
|
||||
cls,
|
||||
audio: np.ndarray,
|
||||
stt_config: SpeechToTextConfig,
|
||||
model_config: ModelConfig,
|
||||
language: str | None,
|
||||
task_type: Literal["transcribe", "translate"],
|
||||
request_prompt: str,
|
||||
to_language: str | None,
|
||||
) -> PromptType:
|
||||
def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
|
||||
"""
|
||||
Gemma3n supports "free-form" transcription.
|
||||
We fix its prompt here to standardize transcriptions/translations
|
||||
requests.
|
||||
"""
|
||||
audio = stt_params.audio
|
||||
stt_config = stt_params.stt_config
|
||||
language = stt_params.language
|
||||
task_type = stt_params.task_type
|
||||
to_language = stt_params.to_language
|
||||
# Transcribe this audio [into <>] | for transcription
|
||||
# Translate this audio [from <> into <>] | for translation
|
||||
prompt = "<start_of_turn>user\n"
|
||||
|
||||
Reference in New Issue
Block a user