[v1] Add Whisper model support (encoder-decoder) (#21088)
Signed-off-by: Russell Bryant <rbryant@redhat.com> Co-authored-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
co-authored by
NickLucche
parent
4db4426404
commit
37e8182bfe
@@ -600,7 +600,6 @@ class VoxtralEncoderModel(nn.Module):
|
||||
self.whisper_encoder = WhisperEncoder(vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(
|
||||
prefix, "whisper_encoder"),
|
||||
is_standalone_encoder=True,
|
||||
init_in_fp32=True)
|
||||
mel_filters = mel_filter_bank(
|
||||
num_frequency_bins=1 + self.config.window_size // 2,
|
||||
|
||||
@@ -15,6 +15,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids
|
||||
|
||||
from vllm.attention import Attention, AttentionType
|
||||
from vllm.attention.layer import MultiHeadAttention
|
||||
from vllm.attention.layers.cross_attention import CrossAttention
|
||||
from vllm.config import (CacheConfig, ModelConfig, SpeechToTextConfig,
|
||||
VllmConfig)
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
@@ -43,7 +44,7 @@ from vllm.transformers_utils.processor import cached_get_processor
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import (MultiModalEmbeddings, SupportsMultiModal,
|
||||
SupportsTranscription, SupportsV0Only)
|
||||
SupportsTranscription)
|
||||
from .utils import (AutoWeightsLoader, WeightsMapper, cast_overflow_tensors,
|
||||
make_layers)
|
||||
|
||||
@@ -124,6 +125,34 @@ class WhisperAudioInputs(TensorSchema):
|
||||
TensorShape("b", "nmb", "t")]
|
||||
|
||||
|
||||
class WhisperEncoderAttention(MultiHeadAttention):
|
||||
"""Multi-headed attention for Whisper encoder with 2D tensor support."""
|
||||
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Input shape: batch_size x seq_len x hidden_size
|
||||
or seq_len x hidden_size
|
||||
"""
|
||||
is_2d = query.dim() == 2
|
||||
if is_2d:
|
||||
query = query.unsqueeze(0)
|
||||
key = key.unsqueeze(0)
|
||||
value = value.unsqueeze(0)
|
||||
|
||||
# Call the parent forward method
|
||||
out = super().forward(query, key, value)
|
||||
|
||||
if is_2d:
|
||||
out = out.squeeze(0)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
class WhisperPositionalEmbedding(nn.Embedding):
|
||||
|
||||
def __init__(self, num_positions: int, embedding_dim: int):
|
||||
@@ -144,7 +173,6 @@ class WhisperAttention(nn.Module):
|
||||
cache_config: Optional[CacheConfig] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
standalone_encoder: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.embed_dim = embed_dim
|
||||
@@ -180,14 +208,25 @@ class WhisperAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
)
|
||||
if standalone_encoder:
|
||||
self.attn = MultiHeadAttention(
|
||||
if attn_type == AttentionType.ENCODER:
|
||||
self.attn = WhisperEncoderAttention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
)
|
||||
else:
|
||||
elif self.attn_type == AttentionType.ENCODER_DECODER:
|
||||
self.attn = CrossAttention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
attn_type=self.attn_type,
|
||||
)
|
||||
else: # AttentionType.DECODER (regular decoder self-attention)
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
@@ -332,11 +371,7 @@ class WhisperMLP(nn.Module):
|
||||
|
||||
class WhisperEncoderLayer(nn.Module):
|
||||
|
||||
def __init__(self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
is_standalone_encoder: bool = False):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
@@ -350,7 +385,6 @@ class WhisperEncoderLayer(nn.Module):
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
standalone_encoder=is_standalone_encoder,
|
||||
)
|
||||
self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
|
||||
self.mlp = WhisperMLP(
|
||||
@@ -446,12 +480,10 @@ class WhisperEncoder(nn.Module):
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
is_standalone_encoder: bool = False,
|
||||
init_in_fp32: bool = False):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
embed_dim = config.d_model
|
||||
self.is_standalone_encoder = is_standalone_encoder
|
||||
self.num_mel_bins = config.num_mel_bins
|
||||
self.max_source_positions = config.max_source_positions
|
||||
self.embed_scale = (math.sqrt(embed_dim)
|
||||
@@ -469,9 +501,7 @@ class WhisperEncoder(nn.Module):
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.encoder_layers,
|
||||
lambda prefix: WhisperEncoderLayer(vllm_config=vllm_config,
|
||||
prefix=f"{prefix}.layers",
|
||||
is_standalone_encoder=
|
||||
is_standalone_encoder),
|
||||
prefix=f"{prefix}.layers"),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.layer_norm = nn.LayerNorm(config.d_model)
|
||||
@@ -752,7 +782,7 @@ class WhisperMultiModalProcessor(
|
||||
info=WhisperProcessingInfo,
|
||||
dummy_inputs=WhisperDummyInputsBuilder)
|
||||
class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,
|
||||
SupportsMultiModal, SupportsV0Only):
|
||||
SupportsMultiModal):
|
||||
packed_modules_mapping = {
|
||||
"self_attn.qkv_proj": [
|
||||
"self_attn.q_proj",
|
||||
@@ -880,19 +910,17 @@ class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,
|
||||
|
||||
def get_multimodal_embeddings(self,
|
||||
**kwargs: object) -> MultiModalEmbeddings:
|
||||
# TODO: This method does not obey the interface for SupportsMultiModal.
|
||||
# Refactor this once encoder/decoder support is implemented in V1.
|
||||
# Required as part of SupportsMultiModal interface.
|
||||
audio_input = self._parse_and_validate_audio_input(**kwargs)
|
||||
return self.model.get_encoder_outputs(audio_input["input_features"])
|
||||
return [self.model.get_encoder_outputs(audio_input["input_features"])]
|
||||
|
||||
def get_input_embeddings(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: Optional[NestedTensors] = None,
|
||||
) -> torch.Tensor:
|
||||
# TODO: This method just returns the decoder sequence embeddings since
|
||||
# Whisper does not have encoder text tokens. Refactor this once
|
||||
# encoder/decoder support is implemented in V1.
|
||||
# This method just returns the decoder sequence embeddings since
|
||||
# Whisper does not have encoder text tokens.
|
||||
return self.model.decoder.get_input_embeddings(input_ids)
|
||||
|
||||
def _parse_and_validate_audio_input(
|
||||
|
||||
Reference in New Issue
Block a user