[Model][LoRA] Add tower/connector LoRA support for Ultravox (#48215)
Signed-off-by: arthurgao2003 <arthurgao2003@gmail.com> Signed-off-by: Arthur Gao <arthurgao2003@gmail.com> Signed-off-by: 高杨懿 <15312196+gao-yangyi@user.noreply.gitee.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: 高杨懿 <15312196+gao-yangyi@user.noreply.gitee.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Fable 5
高杨懿
OpenAI Codex
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5658391af7
commit
b908a21f9a
@@ -4,27 +4,26 @@
|
||||
# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
|
||||
"""PyTorch Ultravox model."""
|
||||
|
||||
import copy
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from types import SimpleNamespace
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from transformers import BatchFeature, ProcessorMixin
|
||||
from transformers.modeling_utils import ModuleUtilsMixin
|
||||
from transformers.models.whisper import WhisperFeatureExtractor
|
||||
from transformers.models.whisper.modeling_whisper import (
|
||||
WhisperEncoder,
|
||||
WhisperEncoderLayer,
|
||||
)
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
|
||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader import DefaultModelLoader
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
@@ -59,6 +58,11 @@ from .utils import (
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .whisper import (
|
||||
WhisperEncoder,
|
||||
WhisperEncoderLayer,
|
||||
_create_fake_bias_for_k_proj,
|
||||
)
|
||||
|
||||
_AUDIO_PLACEHOLDER_OVERRIDE = "<|audio|>"
|
||||
_MAX_ENCODER_BATCH_SIZE = 16
|
||||
@@ -304,14 +308,97 @@ class StackAudioFrames(nn.Module):
|
||||
return audio_embeds
|
||||
|
||||
|
||||
def _build_chunk_attn_metadata(
|
||||
attn: MMEncoderAttention,
|
||||
feature_lens: torch.Tensor,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
device: torch.device,
|
||||
) -> dict[str, torch.Tensor | None]:
|
||||
"""Segmented varlen attention metadata for a padded batch of audio chunks.
|
||||
|
||||
Each padded row contributes up to two sequences to `cu_seqlens`: its valid
|
||||
frames and its padding tail. Attention therefore never crosses a
|
||||
valid/padding boundary (equivalent to the key-padding mask the HF
|
||||
implementation uses), while every row still flows through the
|
||||
(potentially LoRA-wrapped) linears, keeping the per-chunk token counts
|
||||
constant as required by `get_num_mm_encoder_tokens` /
|
||||
`get_num_mm_connector_tokens`. Queries at padding positions produce
|
||||
(garbage) outputs, which are trimmed by `audio_token_len` downstream.
|
||||
"""
|
||||
batch_size = feature_lens.shape[0]
|
||||
starts = np.arange(batch_size, dtype=np.int64) * seq_len
|
||||
lens_np = feature_lens.cpu().numpy().astype(np.int64)
|
||||
bounds = np.stack([starts + lens_np, starts + seq_len], axis=1).reshape(-1)
|
||||
cu_seqlens_np = np.concatenate(([0], bounds))
|
||||
# Fully-valid rows produce empty padding segments; drop the duplicates.
|
||||
cu_seqlens_np = np.unique(cu_seqlens_np).astype(np.int32)
|
||||
|
||||
attn_backend = attn.attn_backend
|
||||
sequence_lengths = MMEncoderAttention.maybe_compute_seq_lens(
|
||||
attn_backend, cu_seqlens_np, device
|
||||
)
|
||||
max_seqlen = torch.tensor(
|
||||
MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens_np),
|
||||
dtype=torch.int32,
|
||||
)
|
||||
cu_seqlens = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
||||
attn_backend,
|
||||
cu_seqlens_np,
|
||||
hidden_size,
|
||||
get_tensor_model_parallel_world_size(),
|
||||
device,
|
||||
)
|
||||
return {
|
||||
"cu_seqlens": cu_seqlens,
|
||||
"max_seqlen": max_seqlen,
|
||||
"sequence_lengths": sequence_lengths,
|
||||
}
|
||||
|
||||
|
||||
# Maps HF whisper encoder layer names to vLLM's fused qkv_proj and nested
|
||||
# mlp; `_create_fake_bias_for_k_proj` supplies the fused bias' k-slice
|
||||
# (HF whisper k_proj has no bias).
|
||||
_WHISPER_ENCODER_MAPPER = WeightsMapper(
|
||||
orig_to_new_substr={".fc1.": ".mlp.fc1.", ".fc2.": ".mlp.fc2."},
|
||||
orig_to_new_stacked={
|
||||
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
|
||||
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
|
||||
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _load_whisper_layer_weights(
|
||||
module: nn.Module, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> set[str]:
|
||||
weights = _create_fake_bias_for_k_proj(weights, ".self_attn.k_proj.weight")
|
||||
loader = AutoWeightsLoader(module)
|
||||
return loader.load_weights(weights, mapper=_WHISPER_ENCODER_MAPPER)
|
||||
|
||||
|
||||
class UltravoxFeedForwardProjector(nn.Module):
|
||||
def __init__(self, config: UltravoxConfig):
|
||||
def __init__(
|
||||
self,
|
||||
config: UltravoxConfig,
|
||||
*,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_dim = config.hidden_size
|
||||
self._pad_and_stack = StackAudioFrames(config.stack_factor)
|
||||
dim_in = config.audio_config.hidden_size * config.stack_factor
|
||||
self.ln_pre = RMSNorm(dim_in)
|
||||
self.linear_1 = nn.Linear(dim_in, self.hidden_dim, bias=False)
|
||||
# ReplicatedLinear (a vLLM-native linear) so the connector can be
|
||||
# wrapped for LoRA; a plain nn.Linear is left unwrapped by from_layer.
|
||||
self.linear_1 = ReplicatedLinear(
|
||||
dim_in,
|
||||
self.hidden_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_1",
|
||||
)
|
||||
dim_mid = self.hidden_dim
|
||||
|
||||
if config.projector_act == "swiglu":
|
||||
@@ -321,7 +408,13 @@ class UltravoxFeedForwardProjector(nn.Module):
|
||||
self.act = get_act_fn(config.projector_act)
|
||||
|
||||
dim_out = config.text_config.hidden_size
|
||||
self.linear_2 = nn.Linear(dim_mid, dim_out, bias=False)
|
||||
self.linear_2 = ReplicatedLinear(
|
||||
dim_mid,
|
||||
dim_out,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_2",
|
||||
)
|
||||
|
||||
# Ultravox v0.4.1 and below use layer_norm after the second linear layer
|
||||
# while v0.5.0 and above uses layer_norm after the first linear layer.
|
||||
@@ -337,42 +430,64 @@ class UltravoxFeedForwardProjector(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
audio_features = self._pad_and_stack(audio_features)
|
||||
audio_features = self.ln_pre(audio_features)
|
||||
hidden_states = self.linear_1(audio_features)
|
||||
hidden_states, _ = self.linear_1(audio_features)
|
||||
hidden_states = self.act(hidden_states)
|
||||
hidden_states = self.ln_mid(hidden_states)
|
||||
hidden_states = self.linear_2(hidden_states)
|
||||
hidden_states, _ = self.linear_2(hidden_states)
|
||||
hidden_states = self.ln_post(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class UltravoxTransformerProjector(nn.Module, ModuleUtilsMixin):
|
||||
def __init__(self, config: UltravoxConfig):
|
||||
class UltravoxTransformerProjector(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = SimpleNamespace(is_decoder=False)
|
||||
config: UltravoxConfig = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
audio_config = config.audio_config
|
||||
|
||||
self._pad_and_stack = StackAudioFrames(config.stack_factor)
|
||||
dim_in = config.audio_config.hidden_size * config.stack_factor
|
||||
|
||||
projector_audio_config = copy.deepcopy(config.audio_config)
|
||||
dim_in = audio_config.hidden_size * config.stack_factor
|
||||
|
||||
self.ln_pre = RMSNorm(dim_in)
|
||||
self.linear_in = nn.Linear(dim_in, projector_audio_config.d_model)
|
||||
self.linear_in = ReplicatedLinear(
|
||||
dim_in,
|
||||
audio_config.d_model,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_in",
|
||||
)
|
||||
|
||||
self.embed_positions = nn.Embedding(
|
||||
projector_audio_config.max_source_positions,
|
||||
projector_audio_config.d_model,
|
||||
audio_config.max_source_positions,
|
||||
audio_config.d_model,
|
||||
)
|
||||
|
||||
audio_vllm_config = vllm_config.with_hf_config(
|
||||
audio_config, architectures=["UltravoxModel"]
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
WhisperEncoderLayer(projector_audio_config)
|
||||
for _ in range(config.num_projector_layers)
|
||||
WhisperEncoderLayer(
|
||||
vllm_config=audio_vllm_config,
|
||||
prefix=f"{prefix}.layers.{idx}",
|
||||
)
|
||||
for idx in range(config.num_projector_layers)
|
||||
]
|
||||
)
|
||||
self._d_model = audio_config.d_model
|
||||
|
||||
self.ln_post = RMSNorm(projector_audio_config.d_model)
|
||||
self.linear_out = nn.Linear(
|
||||
projector_audio_config.d_model, config.text_config.hidden_size
|
||||
self.ln_post = RMSNorm(audio_config.d_model)
|
||||
self.linear_out = ReplicatedLinear(
|
||||
audio_config.d_model,
|
||||
config.text_config.hidden_size,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.linear_out",
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -380,100 +495,65 @@ class UltravoxTransformerProjector(nn.Module, ModuleUtilsMixin):
|
||||
) -> torch.Tensor:
|
||||
audio_features = self._pad_and_stack(audio_features)
|
||||
|
||||
max_len_stacked = audio_features.shape[1]
|
||||
attention_mask = torch.arange(max_len_stacked, device=audio_features.device)[
|
||||
None, :
|
||||
].lt(audio_token_len[:, None])
|
||||
extended_attention_mask = self.get_extended_attention_mask(
|
||||
attention_mask, attention_mask.shape, audio_features.dtype
|
||||
)
|
||||
|
||||
hidden_states = self.ln_pre(audio_features)
|
||||
hidden_states = self.linear_in(hidden_states)
|
||||
hidden_states, _ = self.linear_in(hidden_states)
|
||||
|
||||
positions = self.embed_positions(
|
||||
torch.arange(hidden_states.size(1), device=hidden_states.device)
|
||||
)
|
||||
hidden_states = hidden_states + positions
|
||||
|
||||
batch_size, seq_len, d_model = hidden_states.shape
|
||||
attn_metadata = _build_chunk_attn_metadata(
|
||||
self.layers[0].self_attn.attn,
|
||||
audio_token_len,
|
||||
seq_len,
|
||||
d_model,
|
||||
hidden_states.device,
|
||||
)
|
||||
hidden_states = hidden_states.reshape(batch_size * seq_len, d_model)
|
||||
for layer in self.layers:
|
||||
hidden_states = layer(
|
||||
hidden_states,
|
||||
attention_mask=extended_attention_mask,
|
||||
)
|
||||
# BC version that allows for the old tupled output
|
||||
if isinstance(hidden_states, tuple):
|
||||
hidden_states = hidden_states[0]
|
||||
hidden_states = layer(hidden_states, **attn_metadata)
|
||||
hidden_states = hidden_states.reshape(batch_size, seq_len, d_model)
|
||||
|
||||
hidden_states = self.ln_post(hidden_states)
|
||||
hidden_states = self.linear_out(hidden_states)
|
||||
hidden_states, _ = self.linear_out(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
return _load_whisper_layer_weights(self, weights)
|
||||
|
||||
class ModifiedWhisperEncoder(WhisperEncoder):
|
||||
|
||||
class UltravoxWhisperEncoder(WhisperEncoder):
|
||||
"""Ultravox's `ModifiedWhisperEncoder` on top of vLLM's `WhisperEncoder`.
|
||||
|
||||
Like the original (a modified HF whisper encoder,
|
||||
see https://github.com/huggingface/transformers/issues/25744), it accepts
|
||||
mel inputs shorter than 30s (positions are sliced to the input length) and
|
||||
confines attention to each chunk's valid frames based on `audio_lens`
|
||||
(via segmented `cu_seqlens` instead of a dense key-padding mask), so its
|
||||
outputs match the HF implementation for valid positions. The linears are
|
||||
vLLM-native so the tower can be wrapped for LoRA.
|
||||
"""
|
||||
Encoder portion of OpenAI's Whisper model.
|
||||
|
||||
This implementation is a slightly modified version of HF Transformers'
|
||||
Whisper Encoder, with only a few fixes:
|
||||
1. base_model_prefix updated to allow for doing `.from_pretrained`
|
||||
directly on the encoder
|
||||
2. allow less than 30 second of audio padding to be passed in:
|
||||
- relaxed ValueError check for `input_features` length to be less
|
||||
than or equal to `expected_seq_length` instead of strictly equal
|
||||
- embed_pos is now sliced to match the length of `inputs_embeds`
|
||||
|
||||
Original: https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/modeling_whisper.py
|
||||
See commentary: https://github.com/huggingface/transformers/issues/25744
|
||||
"""
|
||||
|
||||
base_model_prefix = "model.encoder"
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.config.is_decoder = False
|
||||
|
||||
@property
|
||||
def max_context_length(self):
|
||||
return (
|
||||
self.config.max_source_positions
|
||||
* self.conv1.stride[0]
|
||||
* self.conv2.stride[0]
|
||||
)
|
||||
def dtype(self) -> torch.dtype:
|
||||
return self.conv1.weight.dtype
|
||||
|
||||
def get_attention_mask_by_audio_len(
|
||||
self, audio_lens: torch.Tensor | None, hidden_states: torch.Tensor
|
||||
):
|
||||
"""
|
||||
Create attention mask based on audio lengths to mask out padding tokens
|
||||
For each sample in batch:
|
||||
- Convert raw audio length to feature length after convolutions
|
||||
- Create bool mask: True for valid positions and False for padding
|
||||
- Convert to attention mask format expected by transformer layers
|
||||
(1.0 for positions to attend to, large negative for positions to ignore)
|
||||
This masking ensures consistent behavior between training and inference
|
||||
by preventing the model from attending to padding tokens in both cases
|
||||
"""
|
||||
if audio_lens is None:
|
||||
return None
|
||||
@property
|
||||
def max_context_length(self) -> int:
|
||||
return self.max_source_positions * self.total_stride
|
||||
|
||||
audio_feature_len = self._get_feat_extract_output_lengths(audio_lens)
|
||||
max_seq_len = hidden_states.shape[1]
|
||||
attention_mask = torch.arange(max_seq_len, device=hidden_states.device)[
|
||||
None, :
|
||||
].lt(audio_feature_len.view(-1, 1))
|
||||
attention_mask = self.get_extended_attention_mask(
|
||||
attention_mask,
|
||||
None,
|
||||
dtype=hidden_states.dtype,
|
||||
)
|
||||
return attention_mask
|
||||
def _get_feat_extract_output_lengths(
|
||||
self, input_lengths: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return (input_lengths - 1) // 2 + 1
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_features: torch.Tensor,
|
||||
audio_lens: torch.Tensor | None = None,
|
||||
):
|
||||
audio_lens: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
expected_seq_length = self.max_context_length
|
||||
if input_features.shape[-1] > expected_seq_length:
|
||||
raise ValueError(
|
||||
@@ -489,25 +569,27 @@ class ModifiedWhisperEncoder(WhisperEncoder):
|
||||
inputs_embeds = inputs_embeds.permute(0, 2, 1)
|
||||
embed_pos = self.embed_positions.weight[: inputs_embeds.size(-2)]
|
||||
|
||||
hidden_states = inputs_embeds + embed_pos
|
||||
hidden_states = nn.functional.dropout(
|
||||
hidden_states, p=self.dropout, training=self.training
|
||||
hidden_states = (inputs_embeds + embed_pos).to(inputs_embeds.dtype)
|
||||
|
||||
batch_size, seq_len, d_model = hidden_states.shape
|
||||
attn_metadata = _build_chunk_attn_metadata(
|
||||
self.layers[0].self_attn.attn,
|
||||
self._get_feat_extract_output_lengths(audio_lens),
|
||||
seq_len,
|
||||
d_model,
|
||||
hidden_states.device,
|
||||
)
|
||||
|
||||
attention_mask = self.get_attention_mask_by_audio_len(audio_lens, hidden_states)
|
||||
|
||||
hidden_states = hidden_states.reshape(batch_size * seq_len, d_model)
|
||||
for encoder_layer in self.layers:
|
||||
hidden_states = encoder_layer(
|
||||
hidden_states,
|
||||
attention_mask,
|
||||
)
|
||||
# BC version that allows for the old tupled output
|
||||
if isinstance(hidden_states, tuple):
|
||||
hidden_states = hidden_states[0]
|
||||
hidden_states = encoder_layer(hidden_states, **attn_metadata)
|
||||
hidden_states = hidden_states.reshape(batch_size, seq_len, d_model)
|
||||
|
||||
hidden_states = self.layer_norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
return _load_whisper_layer_weights(self, weights)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
UltravoxMultiModalProcessor,
|
||||
@@ -521,7 +603,13 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."}
|
||||
orig_to_new_prefix={
|
||||
"audio_tower.model.encoder.": "audio_tower.",
|
||||
# A whisper checkpoint loaded via `audio_model_id` also carries
|
||||
# decoder and LM-head weights the tower does not use.
|
||||
"audio_tower.model.": None,
|
||||
"audio_tower.proj_out.": None,
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -535,10 +623,20 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
super().__init__()
|
||||
config: UltravoxConfig = vllm_config.model_config.hf_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
quant_config = vllm_config.quant_config
|
||||
lora_config = vllm_config.lora_config
|
||||
self.config = config
|
||||
self.multi_modal_config = multimodal_config
|
||||
assert self.multi_modal_config
|
||||
|
||||
# LoRA on the tower/connector requires per-item token counts that are
|
||||
# predictable from the placeholder count alone (see
|
||||
# `get_num_mm_encoder_tokens`), so pad every audio chunk's mel input
|
||||
# to the tower's full context instead of the batch's max length.
|
||||
self.pad_audio_to_max_context = bool(
|
||||
lora_config is not None and lora_config.enable_tower_connector_lora
|
||||
)
|
||||
|
||||
self.configure_mm_token_handling(
|
||||
self.config.vocab_size,
|
||||
[self.config.audio_token_index],
|
||||
@@ -569,11 +667,24 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
)
|
||||
|
||||
with self._mark_tower_model(vllm_config, "audio"):
|
||||
self.audio_tower = ModifiedWhisperEncoder(config.audio_config)
|
||||
self.audio_tower = UltravoxWhisperEncoder(
|
||||
vllm_config=vllm_config.with_hf_config(
|
||||
config.audio_config, architectures=["UltravoxModel"]
|
||||
),
|
||||
prefix=maybe_prefix(prefix, "audio_tower"),
|
||||
enable_pp=False,
|
||||
)
|
||||
if config.num_projector_layers > 0:
|
||||
self.multi_modal_projector = UltravoxTransformerProjector(config)
|
||||
self.multi_modal_projector = UltravoxTransformerProjector(
|
||||
vllm_config,
|
||||
prefix=maybe_prefix(prefix, "multi_modal_projector"),
|
||||
)
|
||||
else:
|
||||
self.multi_modal_projector = UltravoxFeedForwardProjector(config)
|
||||
self.multi_modal_projector = UltravoxFeedForwardProjector(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "multi_modal_projector"),
|
||||
)
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = init_vllm_registered_model(
|
||||
@@ -596,6 +707,32 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
tower_model="audio_tower.",
|
||||
)
|
||||
|
||||
def _get_max_tokens_per_chunk(self) -> int:
|
||||
# Audio is chunked at the tower's full context (30s -> 1500 encoder
|
||||
# frames); the projector stacks `stack_factor` frames per LM token, so
|
||||
# a full chunk yields ceil(max_source_positions / stack_factor) tokens
|
||||
# and only an audio's last chunk can yield fewer.
|
||||
return math.ceil(
|
||||
self.config.audio_config.max_source_positions / self.config.stack_factor
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_audio_tokens: int) -> int:
|
||||
# With `pad_audio_to_max_context` (enforced when tower/connector LoRA
|
||||
# is enabled), the tower's LoRA-wrapped linears always run on
|
||||
# `max_source_positions` conv-downsampled tokens per chunk, regardless
|
||||
# of the valid frame count.
|
||||
num_chunks = math.ceil(num_audio_tokens / self._get_max_tokens_per_chunk())
|
||||
return num_chunks * self.config.audio_config.max_source_positions
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_encoder_tokens: int) -> int:
|
||||
# The connector runs on the frame-stacked tower output. Stacking pads
|
||||
# each chunk to a multiple of `stack_factor`, so this is
|
||||
# ceil(max_source_positions / stack_factor) tokens per chunk (188 for
|
||||
# whisper's 1500), not `num_encoder_tokens // stack_factor` (187).
|
||||
max_source_positions = self.config.audio_config.max_source_positions
|
||||
num_chunks = num_encoder_tokens // max_source_positions
|
||||
return num_chunks * self._get_max_tokens_per_chunk()
|
||||
|
||||
def _audio_features_to_embeddings(
|
||||
self,
|
||||
input_features: torch.Tensor,
|
||||
@@ -606,9 +743,19 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
batch_size = audio_features.size(0)
|
||||
audio_embeddings = []
|
||||
|
||||
# Process audio features in batches to keep memory usage predictable
|
||||
for start in range(0, batch_size, _MAX_ENCODER_BATCH_SIZE):
|
||||
end = min(start + _MAX_ENCODER_BATCH_SIZE, batch_size)
|
||||
# Process audio features in batches to keep memory usage predictable.
|
||||
# With tower/connector LoRA, the punica kernels always map the first
|
||||
# `x.size(0)` entries of the token-LoRA mapping (set once for the whole
|
||||
# scheduled encoder batch) to the rows of each linear's input, so
|
||||
# splitting the batch would apply the wrong LoRA ids to every chunk
|
||||
# after the first sub-batch; process all chunks in a single pass.
|
||||
encoder_batch_size = (
|
||||
max(batch_size, 1)
|
||||
if self.pad_audio_to_max_context
|
||||
else _MAX_ENCODER_BATCH_SIZE
|
||||
)
|
||||
for start in range(0, batch_size, encoder_batch_size):
|
||||
end = min(start + encoder_batch_size, batch_size)
|
||||
# Process through audio tower
|
||||
batch_features = self.audio_tower(
|
||||
audio_features[start:end], audio_lens[start:end]
|
||||
@@ -662,6 +809,17 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
# [[B1, 80, M1], [B2, 80, M2]] -> [B1+B2, 80, max(M1, M2)]
|
||||
audio_features = pad_and_concat_to_dim3(audio_input["data"])
|
||||
|
||||
if self.pad_audio_to_max_context:
|
||||
# Pad every chunk to the tower's full context so the number of
|
||||
# tokens processed by the tower/connector per chunk is constant,
|
||||
# keeping the LoRA token mappings computed by
|
||||
# `get_num_mm_encoder_tokens`/`get_num_mm_connector_tokens` exact.
|
||||
# Attention to the extra padding is masked via `audio_lens`.
|
||||
# Over-long inputs are not trimmed here; the tower raises on them.
|
||||
pad_len = self.audio_tower.max_context_length - audio_features.shape[-1]
|
||||
if pad_len > 0:
|
||||
audio_features = F.pad(audio_features, (0, pad_len))
|
||||
|
||||
audio_lens = audio_input["lens"]
|
||||
audio_token_len = audio_input["token_len"]
|
||||
|
||||
@@ -755,7 +913,7 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self, ignore_unexpected_prefixes=["audio_tower."])
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.models.whisper_utils import (
|
||||
ISO639_1_SUPPORTED_LANGS,
|
||||
)
|
||||
from vllm.model_executor.offloader import get_offloader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
@@ -114,6 +115,10 @@ class WhisperEncoderAttention(MMEncoderAttention):
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None, # Only used for Flash Attention
|
||||
# Only used for FlashInfer CuDNN backend.
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Input shape: batch_size x seq_len x hidden_size
|
||||
@@ -126,7 +131,14 @@ class WhisperEncoderAttention(MMEncoderAttention):
|
||||
value = value.unsqueeze(0)
|
||||
|
||||
# Call the parent forward method
|
||||
out = super().forward(query, key, value)
|
||||
out = super().forward(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
|
||||
if is_2d:
|
||||
out = out.squeeze(0)
|
||||
@@ -240,11 +252,29 @@ class WhisperAttention(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None, # Only used for Flash Attention
|
||||
# Only used for FlashInfer CuDNN backend.
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
attn_output = self.attn(q, k, v)
|
||||
if cu_seqlens is None:
|
||||
attn_output = self.attn(q, k, v)
|
||||
else:
|
||||
assert self.attn_type == AttentionType.ENCODER, (
|
||||
"Variable-length attention metadata is only supported for "
|
||||
"encoder self-attention."
|
||||
)
|
||||
attn_output = self.attn(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
|
||||
output, _ = self.out_proj(attn_output)
|
||||
|
||||
@@ -381,10 +411,19 @@ class WhisperEncoderLayer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
max_seqlen: torch.Tensor | None = None, # Only used for Flash Attention
|
||||
# Only used for FlashInfer CuDNN backend.
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
):
|
||||
residual = hidden_states
|
||||
hidden_states = self.self_attn_layer_norm(hidden_states)
|
||||
hidden_states = self.self_attn(hidden_states=hidden_states)
|
||||
hidden_states = self.self_attn(
|
||||
hidden_states=hidden_states,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
hidden_states = residual + hidden_states
|
||||
residual = hidden_states
|
||||
hidden_states = self.final_layer_norm(hidden_states)
|
||||
@@ -457,7 +496,12 @@ class WhisperDecoderLayer(nn.Module):
|
||||
|
||||
class WhisperEncoder(nn.Module):
|
||||
def __init__(
|
||||
self, *, vllm_config: VllmConfig, prefix: str = "", init_in_fp32: bool = False
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
init_in_fp32: bool = False,
|
||||
enable_pp: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
@@ -474,13 +518,27 @@ class WhisperEncoder(nn.Module):
|
||||
self.conv2 = nn.Conv1d(embed_dim, embed_dim, stride=2, kernel_size=3, padding=1)
|
||||
|
||||
self.total_stride = self.conv1.stride[0] * self.conv2.stride[0]
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.encoder_layers,
|
||||
lambda prefix: WhisperEncoderLayer(
|
||||
|
||||
def create_layer(prefix: str) -> nn.Module:
|
||||
return WhisperEncoderLayer(
|
||||
vllm_config=vllm_config, prefix=f"{prefix}.layers"
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
)
|
||||
|
||||
if enable_pp:
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.encoder_layers,
|
||||
create_layer,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
else:
|
||||
self.start_layer = 0
|
||||
self.end_layer = config.encoder_layers
|
||||
self.layers = nn.ModuleList(
|
||||
get_offloader().wrap_modules(
|
||||
create_layer(prefix=f"{prefix}.layers.{idx}")
|
||||
for idx in range(config.encoder_layers)
|
||||
)
|
||||
)
|
||||
self.layer_norm = nn.LayerNorm(config.d_model)
|
||||
|
||||
if self.pos_embed_type not in (
|
||||
|
||||
Reference in New Issue
Block a user