# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Attention layer with FlashAttention.""" import torch from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.torch_utils import ( canonicalize_singleton_dim_strides, is_quantized_kv_cache, ) from vllm.v1.attention.backend import AttentionType from vllm.v1.attention.backends.fa_utils import ( get_flash_attn_version, is_flash_attn_varlen_func_available, ) from vllm.v1.attention.ops.triton_reshape_and_cache_flash import ( triton_reshape_and_cache_flash_diffkv, ) if is_flash_attn_varlen_func_available(): from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func from .flash_attn import ( FlashAttentionBackend, FlashAttentionImpl, FlashAttentionMetadata, ) logger = init_logger(__name__) class FlashAttentionDiffKVBackend(FlashAttentionBackend): # Default to 128 for this backend head_size_v: int = 128 @classmethod def set_head_size_v(cls, head_size_v: int) -> None: cls.head_size_v = head_size_v @classmethod def is_supported_on_current_device( cls, head_size: int, head_size_v: int, has_sinks: bool, ) -> bool: """Check whether FA3/4 with this DiffKV config is usable here. DiffKV (hdim_qk != hdim_v) requires FA3 or FA4 """ if not is_flash_attn_varlen_func_available(): return False try: version = get_flash_attn_version( requires_alibi=False, head_size=head_size, head_size_v=head_size_v, has_sinks=has_sinks, ) except Exception: return False return version in (3, 4) @staticmethod def get_name() -> str: return "FLASH_ATTN_DIFFKV" @staticmethod def get_impl_cls() -> type["FlashAttentionImpl"]: return FlashAttentionDiffKVImpl class FlashAttentionDiffKVImpl(FlashAttentionImpl): vllm_flash_attn_version: int | None def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) # Re-derive the FA version with diff-kv context so that # get_flash_attn_version can apply the FA3 -> FA4 upgrade rule # for sinks + hdim != hdim_v. self.vllm_flash_attn_version = get_flash_attn_version( requires_alibi=self.alibi_slopes is not None, head_size=self.head_size, head_size_v=FlashAttentionDiffKVBackend.head_size_v, has_sinks=self.sinks is not None, ) self.fa4_hd256 = False def do_kv_cache_update( self, layer: torch.nn.Module, key: torch.Tensor, value: torch.Tensor, kv_cache: torch.Tensor, slot_mapping: torch.Tensor, ) -> None: if self.attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER): # For encoder attention, # we use direct Q, K, V tensors without caching return # DiffKV packs K and V into a single tensor along the last dim: # kv_cache shape: [num_blocks, block_size, num_kv_heads, # head_size_k + head_size_v] # (B, H, N, C) -> (B, N, H, C) for kernel compatibility. triton_reshape_and_cache_flash_diffkv( key, value, kv_cache.transpose(1, 2), slot_mapping, self.kv_cache_dtype, layer._k_scale, layer._v_scale, ) def forward( self, layer: torch.nn.Module, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, kv_cache: torch.Tensor, attn_metadata: FlashAttentionMetadata, output: torch.Tensor, output_scale: torch.Tensor | None = None, output_block_scale: torch.Tensor | None = None, ) -> torch.Tensor: """Forward pass with FlashAttention. Args: layer: The attention layer, providing the q/k/v quantization scales. query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size_v] kv_cache: shape = [num_blocks, block_size, num_kv_heads, head_size + head_size_v] attn_metadata: Metadata for attention. output: Tensor that the attention result is written into. output_scale: Scale for fused output quantization; not supported by this backend. output_block_scale: Block scale for fused output quantization; not supported by this backend. Returns: shape = [num_tokens, num_heads * head_size_v] NOTE: FP8 quantization, flash-attn expect the size of {q,k,v}_descale to be (num_sequences, num_kv_heads). We use torch's .expand() to avoid duplicating values """ assert self.vllm_flash_attn_version is not None, ( "FlashAttention version not detected." ) if output_scale is not None or output_block_scale is not None: raise NotImplementedError( "fused output quantization is not yet supported for FlashAttentionImpl" ) if attn_metadata is None: # Profiling run. return output.fill_(0) attn_type = self.attn_type # IMPORTANT! # NOTE(woosuk): With piece-wise CUDA graphs, this method is executed in # eager-mode PyTorch. Thus, we need to be careful about any CPU overhead # in this method. For example, `view` and `slice` (or `[:n]`) operations # are surprisingly slow even in the case they do not invoke any GPU ops. # Minimize the PyTorch ops in this method as much as possible. # Whenever making a change in this method, please benchmark the # performance to make sure it does not introduce any overhead. num_actual_tokens = attn_metadata.num_actual_tokens # Handle encoder attention differently - no KV cache needed if attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER): # For encoder attention, # we use direct Q, K, V tensors without caching return self._forward_encoder_attention( query[:num_actual_tokens], key[:num_actual_tokens], value[:num_actual_tokens], output[:num_actual_tokens], attn_metadata, layer, ) # (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs)) key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1) # Fix degenerate strides on size-1 dims (e.g. num_kv_heads=1 with TP). # FA3/4 on H100+ uses TMA, which requires ≥16-byte stride alignment. # See vllm.utils.torch_utils.canonicalize_singleton_dim_strides. fixed_k = canonicalize_singleton_dim_strides(key_cache) fixed_v = canonicalize_singleton_dim_strides(value_cache) if fixed_k is not key_cache or fixed_v is not value_cache: logger.debug( "Canonicalized degenerate KV cache strides (FlashAttentionDiffKV): " "shape=%s, key strides before=%s after=%s, " "value strides before=%s after=%s", key_cache.shape, key_cache.stride(), fixed_k.stride(), value_cache.stride(), fixed_v.stride(), ) key_cache, value_cache = fixed_k, fixed_v if is_quantized_kv_cache(self.kv_cache_dtype): # queries are quantized in the attention layer key_cache = key_cache.view(current_platform.fp8_dtype()) value_cache = value_cache.view(current_platform.fp8_dtype()) cu_seqlens_q = attn_metadata.query_start_loc seqused_k = attn_metadata.seq_lens max_seqlen_q = attn_metadata.max_query_len max_seqlen_k = attn_metadata.max_seq_len block_table = attn_metadata.block_table scheduler_metadata = attn_metadata.scheduler_metadata descale_shape = (cu_seqlens_q.shape[0] - 1, self.num_kv_heads) if self.dcp_world_size > 1: self._forward_with_dcp( query[:num_actual_tokens], key[:num_actual_tokens], value[:num_actual_tokens], key_cache, value_cache, output[:num_actual_tokens], attn_metadata, q_descale=layer._q_scale.expand(descale_shape), k_descale=layer._k_scale.expand(descale_shape), v_descale=layer._v_scale.expand(descale_shape), ) return output else: sliding_window_size = ( list(self.sliding_window) if self.sliding_window is not None else None ) flash_attn_varlen_func( q=query[:num_actual_tokens], k=key_cache, v=value_cache, out=output[:num_actual_tokens], cu_seqlens_q=cu_seqlens_q, max_seqlen_q=max_seqlen_q, seqused_k=seqused_k, max_seqlen_k=max_seqlen_k, softmax_scale=self.scale, causal=attn_metadata.causal, alibi_slopes=self.alibi_slopes, window_size=sliding_window_size, block_table=block_table, softcap=self.logits_soft_cap, scheduler_metadata=scheduler_metadata, fa_version=self.vllm_flash_attn_version, q_descale=layer._q_scale.expand(descale_shape), k_descale=layer._k_scale.expand(descale_shape), v_descale=layer._v_scale.expand(descale_shape), num_splits=attn_metadata.max_num_splits, s_aux=self.sinks, ) return output