Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
268 lines
9.9 KiB
Python
268 lines
9.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Attention layer with FlashAttention."""
|
|
|
|
import torch
|
|
|
|
from vllm.logger import init_logger
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.torch_utils import (
|
|
canonicalize_singleton_dim_strides,
|
|
is_quantized_kv_cache,
|
|
)
|
|
from vllm.v1.attention.backend import AttentionType
|
|
from vllm.v1.attention.backends.fa_utils import (
|
|
get_flash_attn_version,
|
|
is_flash_attn_varlen_func_available,
|
|
)
|
|
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
|
triton_reshape_and_cache_flash_diffkv,
|
|
)
|
|
|
|
if is_flash_attn_varlen_func_available():
|
|
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
|
|
|
|
from .flash_attn import (
|
|
FlashAttentionBackend,
|
|
FlashAttentionImpl,
|
|
FlashAttentionMetadata,
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class FlashAttentionDiffKVBackend(FlashAttentionBackend):
|
|
# Default to 128 for this backend
|
|
head_size_v: int = 128
|
|
|
|
@classmethod
|
|
def set_head_size_v(cls, head_size_v: int) -> None:
|
|
cls.head_size_v = head_size_v
|
|
|
|
@classmethod
|
|
def is_supported_on_current_device(
|
|
cls,
|
|
head_size: int,
|
|
head_size_v: int,
|
|
has_sinks: bool,
|
|
) -> bool:
|
|
"""Check whether FA3/4 with this DiffKV config is usable here.
|
|
|
|
DiffKV (hdim_qk != hdim_v) requires FA3 or FA4
|
|
"""
|
|
if not is_flash_attn_varlen_func_available():
|
|
return False
|
|
try:
|
|
version = get_flash_attn_version(
|
|
requires_alibi=False,
|
|
head_size=head_size,
|
|
head_size_v=head_size_v,
|
|
has_sinks=has_sinks,
|
|
)
|
|
except Exception:
|
|
return False
|
|
return version in (3, 4)
|
|
|
|
@staticmethod
|
|
def get_name() -> str:
|
|
return "FLASH_ATTN_DIFFKV"
|
|
|
|
@staticmethod
|
|
def get_impl_cls() -> type["FlashAttentionImpl"]:
|
|
return FlashAttentionDiffKVImpl
|
|
|
|
|
|
class FlashAttentionDiffKVImpl(FlashAttentionImpl):
|
|
vllm_flash_attn_version: int | None
|
|
|
|
def __init__(self, *args, **kwargs) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
# Re-derive the FA version with diff-kv context so that
|
|
# get_flash_attn_version can apply the FA3 -> FA4 upgrade rule
|
|
# for sinks + hdim != hdim_v.
|
|
self.vllm_flash_attn_version = get_flash_attn_version(
|
|
requires_alibi=self.alibi_slopes is not None,
|
|
head_size=self.head_size,
|
|
head_size_v=FlashAttentionDiffKVBackend.head_size_v,
|
|
has_sinks=self.sinks is not None,
|
|
)
|
|
self.fa4_hd256 = False
|
|
|
|
def do_kv_cache_update(
|
|
self,
|
|
layer: torch.nn.Module,
|
|
key: torch.Tensor,
|
|
value: torch.Tensor,
|
|
kv_cache: torch.Tensor,
|
|
slot_mapping: torch.Tensor,
|
|
) -> None:
|
|
if self.attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER):
|
|
# For encoder attention,
|
|
# we use direct Q, K, V tensors without caching
|
|
return
|
|
|
|
# DiffKV packs K and V into a single tensor along the last dim:
|
|
# kv_cache shape: [num_blocks, block_size, num_kv_heads,
|
|
# head_size_k + head_size_v]
|
|
# (B, H, N, C) -> (B, N, H, C) for kernel compatibility.
|
|
triton_reshape_and_cache_flash_diffkv(
|
|
key,
|
|
value,
|
|
kv_cache.transpose(1, 2),
|
|
slot_mapping,
|
|
self.kv_cache_dtype,
|
|
layer._k_scale,
|
|
layer._v_scale,
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
layer: torch.nn.Module,
|
|
query: torch.Tensor,
|
|
key: torch.Tensor,
|
|
value: torch.Tensor,
|
|
kv_cache: torch.Tensor,
|
|
attn_metadata: FlashAttentionMetadata,
|
|
output: torch.Tensor,
|
|
output_scale: torch.Tensor | None = None,
|
|
output_block_scale: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
"""Forward pass with FlashAttention.
|
|
|
|
Args:
|
|
layer: The attention layer, providing the q/k/v quantization scales.
|
|
query: shape = [num_tokens, num_heads, head_size]
|
|
key: shape = [num_tokens, num_kv_heads, head_size]
|
|
value: shape = [num_tokens, num_kv_heads, head_size_v]
|
|
kv_cache: shape =
|
|
[num_blocks, block_size, num_kv_heads, head_size + head_size_v]
|
|
attn_metadata: Metadata for attention.
|
|
output: Tensor that the attention result is written into.
|
|
output_scale: Scale for fused output quantization; not supported
|
|
by this backend.
|
|
output_block_scale: Block scale for fused output quantization;
|
|
not supported by this backend.
|
|
|
|
Returns:
|
|
shape = [num_tokens, num_heads * head_size_v]
|
|
NOTE: FP8 quantization, flash-attn expect the size of
|
|
{q,k,v}_descale to be (num_sequences, num_kv_heads).
|
|
We use torch's .expand() to avoid duplicating values
|
|
|
|
"""
|
|
assert self.vllm_flash_attn_version is not None, (
|
|
"FlashAttention version not detected."
|
|
)
|
|
|
|
if output_scale is not None or output_block_scale is not None:
|
|
raise NotImplementedError(
|
|
"fused output quantization is not yet supported for FlashAttentionImpl"
|
|
)
|
|
|
|
if attn_metadata is None:
|
|
# Profiling run.
|
|
return output.fill_(0)
|
|
|
|
attn_type = self.attn_type
|
|
|
|
# IMPORTANT!
|
|
# NOTE(woosuk): With piece-wise CUDA graphs, this method is executed in
|
|
# eager-mode PyTorch. Thus, we need to be careful about any CPU overhead
|
|
# in this method. For example, `view` and `slice` (or `[:n]`) operations
|
|
# are surprisingly slow even in the case they do not invoke any GPU ops.
|
|
# Minimize the PyTorch ops in this method as much as possible.
|
|
# Whenever making a change in this method, please benchmark the
|
|
# performance to make sure it does not introduce any overhead.
|
|
|
|
num_actual_tokens = attn_metadata.num_actual_tokens
|
|
|
|
# Handle encoder attention differently - no KV cache needed
|
|
if attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER):
|
|
# For encoder attention,
|
|
# we use direct Q, K, V tensors without caching
|
|
return self._forward_encoder_attention(
|
|
query[:num_actual_tokens],
|
|
key[:num_actual_tokens],
|
|
value[:num_actual_tokens],
|
|
output[:num_actual_tokens],
|
|
attn_metadata,
|
|
layer,
|
|
)
|
|
|
|
# (B, H, N, 2*hs) -> ((B, N, H, hs), (B, N, H, hs))
|
|
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
|
# Fix degenerate strides on size-1 dims (e.g. num_kv_heads=1 with TP).
|
|
# FA3/4 on H100+ uses TMA, which requires ≥16-byte stride alignment.
|
|
# See vllm.utils.torch_utils.canonicalize_singleton_dim_strides.
|
|
fixed_k = canonicalize_singleton_dim_strides(key_cache)
|
|
fixed_v = canonicalize_singleton_dim_strides(value_cache)
|
|
if fixed_k is not key_cache or fixed_v is not value_cache:
|
|
logger.debug(
|
|
"Canonicalized degenerate KV cache strides (FlashAttentionDiffKV): "
|
|
"shape=%s, key strides before=%s after=%s, "
|
|
"value strides before=%s after=%s",
|
|
key_cache.shape,
|
|
key_cache.stride(),
|
|
fixed_k.stride(),
|
|
value_cache.stride(),
|
|
fixed_v.stride(),
|
|
)
|
|
key_cache, value_cache = fixed_k, fixed_v
|
|
|
|
if is_quantized_kv_cache(self.kv_cache_dtype):
|
|
# queries are quantized in the attention layer
|
|
key_cache = key_cache.view(current_platform.fp8_dtype())
|
|
value_cache = value_cache.view(current_platform.fp8_dtype())
|
|
|
|
cu_seqlens_q = attn_metadata.query_start_loc
|
|
seqused_k = attn_metadata.seq_lens
|
|
max_seqlen_q = attn_metadata.max_query_len
|
|
max_seqlen_k = attn_metadata.max_seq_len
|
|
block_table = attn_metadata.block_table
|
|
scheduler_metadata = attn_metadata.scheduler_metadata
|
|
|
|
descale_shape = (cu_seqlens_q.shape[0] - 1, self.num_kv_heads)
|
|
|
|
if self.dcp_world_size > 1:
|
|
self._forward_with_dcp(
|
|
query[:num_actual_tokens],
|
|
key[:num_actual_tokens],
|
|
value[:num_actual_tokens],
|
|
key_cache,
|
|
value_cache,
|
|
output[:num_actual_tokens],
|
|
attn_metadata,
|
|
q_descale=layer._q_scale.expand(descale_shape),
|
|
k_descale=layer._k_scale.expand(descale_shape),
|
|
v_descale=layer._v_scale.expand(descale_shape),
|
|
)
|
|
return output
|
|
else:
|
|
sliding_window_size = (
|
|
list(self.sliding_window) if self.sliding_window is not None else None
|
|
)
|
|
flash_attn_varlen_func(
|
|
q=query[:num_actual_tokens],
|
|
k=key_cache,
|
|
v=value_cache,
|
|
out=output[:num_actual_tokens],
|
|
cu_seqlens_q=cu_seqlens_q,
|
|
max_seqlen_q=max_seqlen_q,
|
|
seqused_k=seqused_k,
|
|
max_seqlen_k=max_seqlen_k,
|
|
softmax_scale=self.scale,
|
|
causal=attn_metadata.causal,
|
|
alibi_slopes=self.alibi_slopes,
|
|
window_size=sliding_window_size,
|
|
block_table=block_table,
|
|
softcap=self.logits_soft_cap,
|
|
scheduler_metadata=scheduler_metadata,
|
|
fa_version=self.vllm_flash_attn_version,
|
|
q_descale=layer._q_scale.expand(descale_shape),
|
|
k_descale=layer._k_scale.expand(descale_shape),
|
|
v_descale=layer._v_scale.expand(descale_shape),
|
|
num_splits=attn_metadata.max_num_splits,
|
|
s_aux=self.sinks,
|
|
)
|
|
return output
|