# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools import logging import math from dataclasses import replace from fractions import Fraction from functools import partial from typing import ClassVar, Protocol import torch import torch.nn.functional as F from torch import nn from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.models.mistral import MistralMLP from vllm.model_executor.models.whisper import WhisperPosEmbedType from vllm.utils.math_utils import cdiv from vllm.v1.attention.backend import ( AttentionBackend, AttentionCGSupport, AttentionMetadata, AttentionType, CommonAttentionMetadata, subclass_attention_backend_with_overrides, ) from vllm.v1.attention.backends.cpu_attn import CPUAttentionBackend from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend from vllm.v1.attention.selector import get_attn_backend from vllm.v1.kv_cache_interface import AttentionSpec, SlidingWindowSpec from .utils import make_layers AiterFlashAttentionBackend: type[AttentionBackend] | None = None try: from vllm.v1.attention.backends import rocm_aiter_fa AiterFlashAttentionBackend = rocm_aiter_fa.AiterFlashAttentionBackend except ImportError: pass logger = logging.getLogger(__name__) CausalRMSNorm = partial(RMSNorm, eps=1e-5) class _BlockPoolingAttentionMetadata(Protocol): @property def slot_mapping(self) -> torch.Tensor: ... def _pad1d( x: torch.Tensor, paddings: tuple[int, int], mode: str = "constant", value: float = 0.0, ) -> torch.Tensor: """Tiny wrapper around F.pad, just to allow for reflect padding on small input. If this is the case, we insert extra 0 padding to the right before the reflection happen. """ length = x.shape[-1] padding_left, padding_right = paddings assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right) if mode == "reflect": max_pad = max(padding_left, padding_right) extra_pad = 0 if length <= max_pad: extra_pad = max_pad - length + 1 x = F.pad(x, (0, extra_pad)) padded = F.pad(x, paddings, mode, value) end = padded.shape[-1] - extra_pad return padded[..., :end] else: return F.pad(x, paddings, mode, value) class WhisperCausalConv1d(nn.Conv1d): def __init__( self, in_channels: int, out_channels: int, kernel_size: int, stride: int = 1, padding: int = 0, bias: bool = True, ) -> None: super().__init__( in_channels, out_channels, kernel_size, stride=stride, padding=padding, bias=bias, ) self._stride = self.stride[0] self._effective_kernel_size = (kernel_size - 1) * self.dilation[0] + 1 self._padding_total = self._effective_kernel_size - self._stride def forward(self, x: torch.Tensor) -> torch.Tensor: n_frames = ( x.shape[-1] - self._effective_kernel_size + self._padding_total ) / self._stride + 1 target_length = (math.ceil(n_frames) - 1) * self._stride + ( self._effective_kernel_size - self._padding_total ) extra_padding = target_length - x.shape[-1] x = _pad1d(x, (self._padding_total, extra_padding), mode="constant") return super().forward(x) @functools.lru_cache def create_whisper_attention_backend_with_block_pooling( underlying_attn_backend: type[AttentionBackend], block_pool_size: int, sliding_window: int | None = None, ) -> type[AttentionBackend]: prefix = "WhisperCausalAttentionWithBlockPooling_" underlying_builder = underlying_attn_backend.get_builder_cls() underlying_impl = underlying_attn_backend.get_impl_cls() class WhisperCausalAttentionWithBlockPoolingBuilder(underlying_builder): # type: ignore # Full cudagraphs only for uniform single-token decode: capture bakes in # tensor addresses, so `build` writes metadata into persistent buffers # (below). Prefill/mixed batches fall back to piecewise. _cudagraph_support: ClassVar[AttentionCGSupport] = ( AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE ) # Disable the update_block_table fast path: it would bypass block pooling # by splicing the raw slot_mapping/block_table into cached metadata. supports_update_block_table: bool = False def __init__( self, kv_cache_spec: AttentionSpec, layer_names: list[str], vllm_config: VllmConfig, device: torch.device, ): kv_cache_spec = replace( kv_cache_spec, block_size=kv_cache_spec.block_size * block_pool_size, tokens_per_state=1, ) if isinstance(kv_cache_spec, SlidingWindowSpec): # The manager keeps `sliding_window` in pooled units; the kernel # runs on the expanded sequence, so give it the model's window in # unpooled units (`get_kv_cache_spec` sizes the pooled window so # this window always stays resident). assert sliding_window is not None kv_cache_spec = replace(kv_cache_spec, sliding_window=sliding_window) super().__init__(kv_cache_spec, layer_names, vllm_config, device) # Override model_config-derived values with the actual # encoder values from kv_cache_spec self.num_heads_kv = kv_cache_spec.num_kv_heads self.headdim = kv_cache_spec.head_size # num_heads_q for the encoder is the same as num_kv_heads # (no GQA in whisper encoder) self.num_heads_q = kv_cache_spec.num_kv_heads # Persistent capture-safe buffers for pooled metadata; lazily sized # on first `build` to match the incoming tensors' dtype/device. self._pool_buffers_ready = False def _maybe_init_pool_buffers( self, common_attn_metadata: CommonAttentionMetadata ) -> None: if self._pool_buffers_ready: return device = common_attn_metadata.slot_mapping.device sched = self.vllm_config.scheduler_config self._qsl_buf = torch.empty( sched.max_num_seqs + 1, dtype=common_attn_metadata.query_start_loc.dtype, device=device, ) self._seq_buf = torch.empty( sched.max_num_seqs, dtype=common_attn_metadata.seq_lens.dtype, device=device, ) self._slot_buf = torch.empty( sched.max_num_batched_tokens * block_pool_size, dtype=common_attn_metadata.slot_mapping.dtype, device=device, ) self._pool_arange = torch.arange( block_pool_size, dtype=common_attn_metadata.slot_mapping.dtype, device=device, ) self._pool_buffers_ready = True def build( self, common_prefix_len: int, common_attn_metadata: CommonAttentionMetadata, fast_build: bool = False, ) -> AttentionMetadata: cm = common_attn_metadata self._maybe_init_pool_buffers(cm) # Scale query_start_loc / seq_lens into the persistent buffers. n = cm.query_start_loc.numel() assert n <= self._qsl_buf.numel(), ( f"{n=} exceeds pooled query_start_loc buffer {self._qsl_buf.numel()}" ) qsl_buf = self._qsl_buf[:n] qsl_buf.copy_(cm.query_start_loc) qsl_buf.mul_(block_pool_size) r = cm.seq_lens.numel() assert r <= self._seq_buf.numel(), ( f"{r=} exceeds pooled seq_lens buffer {self._seq_buf.numel()}" ) seq_buf = self._seq_buf[:r] seq_buf.copy_(cm.seq_lens) seq_buf.mul_(block_pool_size) # Expand each slot into block_pool_size contiguous slots. Padding # slots (-1) stay -1 after clamp so the KV write is skipped. m = cm.slot_mapping.numel() assert m * block_pool_size <= self._slot_buf.numel(), ( f"pooled slot_mapping {m * block_pool_size} exceeds buffer " f"{self._slot_buf.numel()}" ) slot_buf = self._slot_buf[: m * block_pool_size] slot_buf.view(m, block_pool_size).copy_( cm.slot_mapping.unsqueeze(1) * block_pool_size + self._pool_arange ) slot_buf.clamp_(min=-1) # CPU tensors aren't captured; fresh scaled copies avoid mutating # the runner's shared buffers. qsl_cpu = cm.query_start_loc_cpu * block_pool_size seq_lens_cpu_upper_bound = ( cm.seq_lens_cpu_upper_bound * block_pool_size if cm.seq_lens_cpu_upper_bound is not None else None ) new_common_attn_metadata = cm.replace( query_start_loc=qsl_buf, query_start_loc_cpu=qsl_cpu, seq_lens=seq_buf, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, num_actual_tokens=cm.num_actual_tokens * block_pool_size, max_query_len=cm.max_query_len * block_pool_size, max_seq_len=cm.max_seq_len * block_pool_size, slot_mapping=slot_buf, ) return super().build( common_prefix_len * block_pool_size, new_common_attn_metadata, fast_build, ) # NOTE: We need a custom impl so we can use the transformed slot_mapping # computed by `WhisperCausalAttentionWithBlockPoolingBuilder` instead of # the one from `forward_context.slot_mapping` (gpu_model_runner). # This follows the same pattern as CrossAttentionImpl. class WhisperCausalAttentionWithBlockPoolingImpl(underlying_impl): # type: ignore[valid-type,misc] def forward( self, layer: torch.nn.Module, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, kv_cache: torch.Tensor, attn_metadata: _BlockPoolingAttentionMetadata | None, output: torch.Tensor, output_scale: torch.Tensor | None = None, output_block_scale: torch.Tensor | None = None, ) -> torch.Tensor: if ( not underlying_attn_backend.forward_includes_kv_cache_update and attn_metadata is not None and layer.kv_sharing_target_layer_name is None and key is not None and value is not None ): self.do_kv_cache_update( layer, key, value, kv_cache, attn_metadata.slot_mapping ) return super().forward( layer, query, key, value, kv_cache, attn_metadata, output, output_scale, output_block_scale, ) _SUPPORTED_BACKENDS = tuple( b for b in ( AiterFlashAttentionBackend, CPUAttentionBackend, FlashAttentionBackend, RocmAttentionBackend, TritonAttentionBackend, ) if b is not None ) if not issubclass(underlying_attn_backend, _SUPPORTED_BACKENDS): raise NotImplementedError( f"{underlying_attn_backend} is not yet supported." "Contributions to support more backends are much " "appreciated." ) if not issubclass( underlying_attn_backend, (CPUAttentionBackend, FlashAttentionBackend) ): logger.info( "Using %s for Whisper causal attention with block pooling. " "This backend was recently enabled for this model. " "If you encounter any accuracy or performance issues, " "please open an issue at " "https://github.com/vllm-project/vllm/issues " "with the [ROCm] tag so it can be triaged by the " "appropriate team.", underlying_attn_backend.get_name(), ) attn_backend = subclass_attention_backend_with_overrides( name_prefix=prefix, attention_backend_cls=underlying_attn_backend, overrides={ "get_builder_cls": lambda: WhisperCausalAttentionWithBlockPoolingBuilder, "get_impl_cls": lambda: WhisperCausalAttentionWithBlockPoolingImpl, "forward_includes_kv_cache_update": True, }, ) return attn_backend class WhisperCausalAttentionWithBlockPooling(Attention): """Attention layer with block pooling.""" def __init__( self, num_heads: int, head_size: int, scale: float, num_kv_heads: int | None = None, alibi_slopes: list[float] | None = None, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, logits_soft_cap: float | None = None, per_layer_sliding_window: int | None = None, prefix: str = "", attn_type: str = AttentionType.DECODER, kv_sharing_target_layer_name: str | None = None, block_pool_size: int = 1, attn_backend: type[AttentionBackend] | None = None, **extra_impl_args, ) -> None: self.block_pool_size = block_pool_size dtype = torch.get_default_dtype() if cache_config is not None: kv_cache_dtype = cache_config.cache_dtype else: kv_cache_dtype = "auto" underlying_attn_backend = get_attn_backend( head_size, dtype, kv_cache_dtype, attn_type=attn_type, ) attn_backend = create_whisper_attention_backend_with_block_pooling( underlying_attn_backend, block_pool_size, per_layer_sliding_window ) super().__init__( num_heads=num_heads, head_size=head_size, scale=scale, num_kv_heads=num_kv_heads, alibi_slopes=alibi_slopes, cache_config=cache_config, quant_config=quant_config, logits_soft_cap=logits_soft_cap, per_layer_sliding_window=per_layer_sliding_window, prefix=prefix, attn_type=attn_type, kv_sharing_target_layer_name=kv_sharing_target_layer_name, attn_backend=attn_backend, **extra_impl_args, ) def get_kv_cache_spec(self, vllm_config: VllmConfig): kv_cache_spec = super().get_kv_cache_spec(vllm_config) assert isinstance(kv_cache_spec, AttentionSpec) kv_cache_spec = replace( kv_cache_spec, tokens_per_state=Fraction(1, self.block_pool_size), ) if isinstance(kv_cache_spec, SlidingWindowSpec): # The manager counts blocks in pooled units, so express the window in # pooled units to avoid reserving `block_pool_size`x too many blocks. # The `+1` is an eviction margin: block-aligned eviction only keeps # `pooled - 1` pooled tokens, so this guarantees the kernel's full # (unpooled) window still stays resident. pooled = cdiv(kv_cache_spec.sliding_window, self.block_pool_size) + 1 kv_cache_spec = replace(kv_cache_spec, sliding_window=pooled) return kv_cache_spec class WhisperCausalAttention(nn.Module): def __init__( self, embed_dim: int, num_heads: int, head_dim: int, max_position_embeddings: int, bias: bool = True, attn_type: AttentionType = AttentionType.DECODER, per_layer_sliding_window: int | None = None, block_pool_size: int = 1, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", ): super().__init__() self.embed_dim = embed_dim tp_size = get_tensor_model_parallel_world_size() self.total_num_heads = num_heads assert self.total_num_heads % tp_size == 0 self.num_heads = self.total_num_heads // tp_size if self.total_num_heads >= tp_size: # Number of heads is greater than TP size, so we partition # the KV heads across multiple tensor parallel GPUs. assert self.total_num_heads % tp_size == 0 else: # Number of heads is less than TP size, so we replicate # the KV heads across multiple tensor parallel GPUs. assert tp_size % self.total_num_heads == 0 self.num_kv_heads = max(1, self.total_num_heads // tp_size) self.head_dim = head_dim self.q_size = self.num_heads * self.head_dim self.kv_size = self.num_kv_heads * self.head_dim self.attn_type = attn_type self.scaling = self.head_dim**-0.5 self._init_qkv(embed_dim, bias, quant_config, prefix=prefix) self.out_proj = RowParallelLinear( input_size=self.total_num_heads * self.head_dim, output_size=embed_dim, bias=bias, quant_config=quant_config, prefix=f"{prefix}.out_proj", ) assert block_pool_size > 1, ( f"Causal attention only supports block_pool_size>1, not {block_pool_size}." ) self.attn = WhisperCausalAttentionWithBlockPooling( self.num_heads, self.head_dim, self.scaling, num_kv_heads=self.num_kv_heads, cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.attn", attn_type=AttentionType.DECODER, per_layer_sliding_window=per_layer_sliding_window, block_pool_size=block_pool_size, ) assert per_layer_sliding_window is not None, ( "rope can only used in combination with a sliding window" ) self._init_rotary_emb(max_position_embeddings) def _init_rotary_emb(self, max_position_embeddings: int) -> None: self.rotary_emb = get_rope( self.head_dim, max_position=max_position_embeddings, is_neox_style=False, rope_parameters={"rope_theta": 1e6}, ) def _init_qkv( self, embed_dim: int, bias: bool = True, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: self.qkv_proj = QKVParallelLinear( hidden_size=embed_dim, head_size=self.head_dim, total_num_heads=self.total_num_heads, total_num_kv_heads=self.total_num_heads, bias=bias, quant_config=quant_config, prefix=f"{prefix}.qkv_proj", ) def forward( self, hidden_states: torch.Tensor, positions: torch.Tensor | None = None, ): qkv, _ = self.qkv_proj(hidden_states) q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) assert positions is not None q, k = self.rotary_emb(positions, q, k) attn_output = self.attn(q, k, v) output, _ = self.out_proj(attn_output) return output class WhisperCausalEncoderLayer(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config sliding_window = getattr(config, "sliding_window", None) block_pool_size = config.block_pool_size assert block_pool_size > 1 cache_config = vllm_config.cache_config quant_config = vllm_config.quant_config self.embed_dim = config.d_model self.head_dim = self.embed_dim // config.encoder_attention_heads self.self_attn = WhisperCausalAttention( embed_dim=self.embed_dim, num_heads=config.encoder_attention_heads, head_dim=config.encoder_head_dim, max_position_embeddings=config.max_position_embeddings, block_pool_size=block_pool_size, per_layer_sliding_window=sliding_window, cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", ) self.self_attn_layer_norm = CausalRMSNorm(self.embed_dim) self.mlp = MistralMLP( hidden_size=config.d_model, intermediate_size=config.encoder_ffn_dim, hidden_act="silu", quant_config=quant_config, bias=True, gate_up_proj_bias=False, prefix=f"{prefix}.mlp", ) self.final_layer_norm = CausalRMSNorm(self.embed_dim) def forward( self, hidden_states: torch.Tensor, positions: torch.Tensor | None = None, ): residual = hidden_states hidden_states = self.self_attn_layer_norm(hidden_states) hidden_states = self.self_attn(hidden_states=hidden_states, positions=positions) hidden_states = residual + hidden_states residual = hidden_states hidden_states = self.final_layer_norm(hidden_states) hidden_states = self.mlp(hidden_states) hidden_states = residual + hidden_states return hidden_states class WhisperCausalEncoder(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config embed_dim = config.d_model assert WhisperPosEmbedType(config.pos_embed) == WhisperPosEmbedType.ROPE assert config.is_causal self.num_mel_bins = config.num_mel_bins self.max_source_positions = config.max_source_positions self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0 self.conv1 = WhisperCausalConv1d(self.num_mel_bins, embed_dim, kernel_size=3) self.conv2 = WhisperCausalConv1d(embed_dim, embed_dim, stride=2, kernel_size=3) self.total_stride = self.conv1.stride[0] * self.conv2.stride[0] self.start_layer, self.end_layer, self.layers = make_layers( config.encoder_layers, lambda prefix: WhisperCausalEncoderLayer( vllm_config=vllm_config, prefix=f"{prefix}.layers" ), prefix=f"{prefix}.layers", ) self.layer_norm = CausalRMSNorm(config.d_model) def forward_conv( self, input_features: torch.Tensor | list[torch.Tensor] ) -> torch.Tensor: hidden_states = [] for features in input_features: embeds = nn.functional.gelu(self.conv1(features)) embeds = nn.functional.gelu(self.conv2(embeds)) embeds = embeds.transpose(-1, -2).to(embeds.dtype) hidden_states.append(embeds) hidden_states = torch.cat(hidden_states) return hidden_states def forward( self, hidden_states: torch.Tensor, positions: torch.Tensor ) -> torch.Tensor: for encoder_layer in self.layers: hidden_states = encoder_layer(hidden_states, positions) hidden_states = self.layer_norm(hidden_states) return hidden_states