[Audio] Bundle get_generation_prompt() params into SpeechToTextParams (#36268)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
Ekagra Ranjan
2026-04-22 12:24:18 +08:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Cyrus Leung
parent 6d09769700
commit 4eafc72928
18 changed files with 214 additions and 161 deletions
+9 -13
View File
@@ -3,9 +3,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Literal
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
@@ -13,6 +11,8 @@ from transformers import PretrainedConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
from vllm.logger import init_logger
@@ -1900,7 +1900,7 @@ class CohereASRDummyInputsBuilder(BaseDummyInputsBuilder[CohereASRProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options=None,
mm_options: Mapping[str, BaseDummyOptions],
mm_processor_kwargs=None,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
@@ -2021,16 +2021,12 @@ class CohereAsrForConditionalGeneration(
return super().validate_language(language)
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig, # not needed here
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
) -> PromptType:
def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
audio = stt_params.audio
stt_config = stt_params.stt_config
language = stt_params.language
request_prompt = stt_params.request_prompt
if language is None:
raise ValueError(
"Language must be specified when creating the CohereASR prompt"
+7 -9
View File
@@ -2,9 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, cast
from typing import Annotated, cast
import numpy as np
import torch
from torch import nn
from transformers import (
@@ -14,6 +13,7 @@ from transformers import (
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
@@ -356,14 +356,12 @@ class FireRedASR2ForConditionalGeneration(
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig, # not needed here
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
stt_params: SpeechToTextParams,
) -> PromptType:
audio = stt_params.audio
stt_config = stt_params.stt_config
language = stt_params.language
if language is None:
raise ValueError(
"Language must be specified when creating the fireredasr2 prompt"
+7 -9
View File
@@ -3,9 +3,8 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, cast
from typing import Annotated, cast
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
@@ -16,6 +15,7 @@ from transformers import (
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict, PromptType
from vllm.logger import init_logger
@@ -876,14 +876,12 @@ class FunASRForConditionalGeneration(
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig, # not needed here
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
stt_params: SpeechToTextParams,
) -> PromptType:
audio = stt_params.audio
stt_config = stt_params.stt_config
language = stt_params.language
if language is None:
raise ValueError(
"Language must be specified when creating the funasr prompt"
+7 -11
View File
@@ -3,7 +3,6 @@
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal
import numpy as np
import torch
from torch import nn
from transformers import AutoModel, BatchFeature
@@ -19,6 +18,7 @@ from transformers.models.siglip import SiglipImageProcessorFast
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -769,21 +769,17 @@ class Gemma3nForConditionalGeneration(
raise ValueError(f"Unsupported modality: {modality}")
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
stt_config: SpeechToTextConfig,
model_config: ModelConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
) -> PromptType:
def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
"""
Gemma3n supports "free-form" transcription.
We fix its prompt here to standardize transcriptions/translations
requests.
"""
audio = stt_params.audio
stt_config = stt_params.stt_config
language = stt_params.language
task_type = stt_params.task_type
to_language = stt_params.to_language
# Transcribe this audio [into <>] | for transcription
# Translate this audio [from <> into <>] | for translation
prompt = "<start_of_turn>user\n"
+6 -11
View File
@@ -4,7 +4,6 @@
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, TypeAlias
import numpy as np
import torch
import torch.nn as nn
from transformers import BatchFeature
@@ -13,6 +12,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
from vllm.model_executor.layers.activation import get_act_fn
@@ -1131,17 +1131,12 @@ class GlmAsrForConditionalGeneration(
)
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig,
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
) -> PromptType:
def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
"""Get the generation prompt to be used for transcription requests."""
audio = stt_params.audio
model_config = stt_params.model_config
task_type = stt_params.task_type
to_language = stt_params.to_language
tokenizer = cached_tokenizer_from_config(model_config)
audio_token = cls._get_audio_token(model_config)
+8 -9
View File
@@ -26,9 +26,8 @@
import math
from collections.abc import Iterable, Mapping
from typing import Annotated, Literal
from typing import Annotated
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
@@ -36,6 +35,7 @@ from transformers import BatchFeature, PretrainedConfig
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -852,15 +852,14 @@ class GraniteSpeechForConditionalGeneration(
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig,
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
stt_params: SpeechToTextParams,
) -> PromptType:
"""Get the generation prompt to be used for transcription requests."""
audio = stt_params.audio
model_config = stt_params.model_config
task_type = stt_params.task_type
to_language = stt_params.to_language
# Audio placeholders don't use an index, so value doesn't matter
audio_tok = cls.get_placeholder_str("audio", 0)
+2 -8
View File
@@ -29,7 +29,7 @@ from torch import Tensor
from transformers.models.whisper.tokenization_whisper import LANGUAGES
from typing_extensions import Self, TypeIs
from vllm.config import ModelConfig, SpeechToTextConfig
from vllm.config import ModelConfig, SpeechToTextConfig, SpeechToTextParams
from vllm.inputs import PromptType, TokensPrompt
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
@@ -1119,13 +1119,7 @@ class SupportsTranscription(Protocol):
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
stt_config: SpeechToTextConfig,
model_config: ModelConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
stt_params: SpeechToTextParams,
) -> PromptType:
"""Get the prompt for the ASR model.
The model has control over the construction, as long as it
+7 -10
View File
@@ -14,6 +14,7 @@ from transformers import WhisperConfig as HFWhisperConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import PromptType, TokensPrompt
from vllm.model_executor.model_loader import DefaultModelLoader
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -626,16 +627,12 @@ class KimiAudioForConditionalGeneration(
)
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig,
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
) -> PromptType:
def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
audio = stt_params.audio
model_config = stt_params.model_config
task_type = stt_params.task_type
request_prompt = stt_params.request_prompt
tokenizer = cached_get_tokenizer(
model_config.tokenizer,
tokenizer_cls=KimiAudioTokenizer,
+7 -12
View File
@@ -23,9 +23,8 @@
"""Inference-only Qwen3-ASR model."""
from collections.abc import Iterable, Mapping, Sequence
from typing import Any, Literal
from typing import Any
import numpy as np
import torch
import torch.nn as nn
from transformers.feature_extraction_utils import BatchFeature
@@ -33,6 +32,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
@@ -549,17 +549,12 @@ class Qwen3ASRForConditionalGeneration(
)
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig,
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
) -> PromptType:
def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
"""Get the generation prompt to be used for transcription requests."""
audio = stt_params.audio
model_config = stt_params.model_config
task_type = stt_params.task_type
to_language = stt_params.to_language
tokenizer = cached_tokenizer_from_config(model_config)
audio_placeholder = cls.get_placeholder_str("audio", 0)
@@ -24,7 +24,7 @@
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from functools import partial
from typing import Any, Literal, cast
from typing import Any, cast
import numpy as np
import torch
@@ -46,6 +46,7 @@ from transformers.models.whisper import WhisperFeatureExtractor
from vllm.compilation.decorators import support_torch_compile
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.inputs import PromptType
from vllm.logger import init_logger
@@ -2201,19 +2202,17 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
)
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
stt_config: SpeechToTextConfig,
model_config: ModelConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
) -> PromptType:
def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
"""
Construct a transcription/translation prompt for Qwen3-Omni.
"""
audio = stt_params.audio
stt_config = stt_params.stt_config
model_config = stt_params.model_config
language = stt_params.language
task_type = stt_params.task_type
to_language = stt_params.to_language
request_prompt = stt_params.request_prompt
# Transcribe this audio [into <language>] | for transcription
# Translate this audio [from <language> into <to_language>] | for translation
instruction = "Transcribe" if task_type == "transcribe" else "Translate"
+8 -8
View File
@@ -4,7 +4,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from functools import partial
from typing import Literal, cast
from typing import cast
import numpy as np
import regex as re
@@ -19,6 +19,7 @@ from transformers import BatchFeature, WhisperConfig
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -446,14 +447,13 @@ class VoxtralForConditionalGeneration(
# for speech-to-text transcription
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig,
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
stt_params: SpeechToTextParams,
) -> PromptType:
audio = stt_params.audio
model_config = stt_params.model_config
stt_config = stt_params.stt_config
language = stt_params.language
tokenizer = cached_tokenizer_from_config(model_config)
audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless
req = TranscriptionRequest(
@@ -4,7 +4,6 @@
import asyncio
import math
from collections.abc import AsyncGenerator, Iterable, Iterator, Mapping
from typing import Literal
import numpy as np
import torch
@@ -18,6 +17,7 @@ from mistral_common.tokens.tokenizers.audio import AudioConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.engine.protocol import StreamingInput
from vllm.envs import VLLM_ENGINE_ITERATION_TIMEOUT_S
from vllm.inputs import PromptType, TokensPrompt
@@ -465,14 +465,13 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
# for speech-to-text transcription
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig,
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
stt_params: SpeechToTextParams,
) -> PromptType:
audio = stt_params.audio
model_config = stt_params.model_config
stt_config = stt_params.stt_config
language = stt_params.language
tokenizer = cached_tokenizer_from_config(model_config)
audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless
+9 -8
View File
@@ -5,7 +5,7 @@ import enum
import math
from collections.abc import Iterable, Mapping, Sequence
from contextlib import nullcontext
from typing import Annotated, Literal
from typing import Annotated
import numpy as np
import torch
@@ -20,6 +20,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import (
ExplicitEncoderDecoderPrompt,
@@ -830,14 +831,14 @@ class WhisperForConditionalGeneration(
@classmethod
def get_generation_prompt(
cls,
audio: np.ndarray,
model_config: ModelConfig, # not needed here
stt_config: SpeechToTextConfig,
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: str | None,
stt_params: SpeechToTextParams,
) -> PromptType:
audio = stt_params.audio
stt_config = stt_params.stt_config
language = stt_params.language
task_type = stt_params.task_type
request_prompt = stt_params.request_prompt
if language is None:
raise ValueError(
"Language must be specified when creating the Whisper prompt"