[Attention] add triton diff-kv backend for mimo (#41797)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
@@ -75,6 +75,19 @@ steps:
|
||||
- pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
parallelism: 2
|
||||
|
||||
- label: Kernels Attention DiffKV Test (H100)
|
||||
key: kernels-attention-diffkv-test-h100
|
||||
timeout_in_minutes: 20
|
||||
device: h100
|
||||
num_devices: 1
|
||||
source_file_dependencies:
|
||||
- vllm/v1/attention/ops/triton_unified_attention_diffkv.py
|
||||
- vllm/v1/attention/backends/triton_attn_diffkv.py
|
||||
- vllm/v1/attention/backends/flash_attn_diffkv.py
|
||||
- tests/kernels/attention/test_triton_unified_attention_diffkv.py
|
||||
commands:
|
||||
- pytest -v -s kernels/attention/test_triton_unified_attention_diffkv.py
|
||||
|
||||
- label: Kernels Quantization Test %N
|
||||
key: kernels-quantization-test
|
||||
timeout_in_minutes: 90
|
||||
|
||||
@@ -181,6 +181,7 @@ Priority is **1 = highest** (tried first).
|
||||
| `ROCM_AITER_UNIFIED_ATTN` | | fp16, bf16 | `auto` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | N/A |
|
||||
| `ROCM_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 128, 160, 192, 224, 256 | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder, Encoder Only | N/A |
|
||||
| `TRITON_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `int8_per_token_head`, `fp8_per_token_head` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | Any |
|
||||
| `TRITON_ATTN_DIFFKV` | | fp16, bf16 | `auto`, `bfloat16` | Any | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
|
||||
| `TURBOQUANT` | | fp16, bf16 | `turboquant_k8v4`, `turboquant_4bit_nc`, `turboquant_k3v4_nc`, `turboquant_3bit_nc` | 16, 32, 64, 128 | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
|
||||
|
||||
> **†** FlashInfer uses TRTLLM attention on Blackwell (SM100), which supports sinks. Disable via `--attention-config.use_trtllm_attention=0`.
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for the Triton DiffKV unified-attention kernel.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import (
|
||||
canonicalize_singleton_dim_strides,
|
||||
set_random_seed,
|
||||
)
|
||||
from vllm.v1.attention.backends.fa_utils import (
|
||||
get_flash_attn_version,
|
||||
is_flash_attn_varlen_func_available,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_unified_attention_diffkv import (
|
||||
unified_attention_diffkv,
|
||||
)
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
# (num_query_heads, num_kv_heads): MHA, GQA, and the num_kv_heads==1
|
||||
# (degenerate-stride) case.
|
||||
NUM_HEADS = [(4, 4), (8, 2), (5, 1)]
|
||||
# (head_size_qk, head_size_v). (192, 128) is the canonical asymmetric
|
||||
# DiffKV shape; FA4 on Blackwell only supports head_size>128 when it is
|
||||
# 192, and FA3 on Hopper supports it too -- so this pair is runnable on
|
||||
# both. (128, 128) keeps the equal-dim path covered through the DiffKV
|
||||
# kernel.
|
||||
HEAD_SIZES = [(128, 128), (192, 128)]
|
||||
BLOCK_SIZES = [16]
|
||||
DTYPES = [torch.bfloat16]
|
||||
|
||||
NUM_BLOCKS = 2048
|
||||
|
||||
# 0: 2D decode kernel; 8: 3D (split-KV) decode kernel.
|
||||
SEQ_THRESHOLD_3D_VALUES = [0, 8]
|
||||
|
||||
NUM_PAR_SOFTMAX_SEGMENTS = 16
|
||||
|
||||
|
||||
def _alloc_segm_buffers(seq_threshold_3D: int, num_query_heads: int, head_size_v: int):
|
||||
"""Allocate the split-KV softmax scratch (last dim == head_size_v)."""
|
||||
head_size_v_padded = next_power_of_2(head_size_v)
|
||||
segm_output = torch.empty(
|
||||
(
|
||||
seq_threshold_3D,
|
||||
num_query_heads,
|
||||
NUM_PAR_SOFTMAX_SEGMENTS,
|
||||
head_size_v_padded,
|
||||
),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
segm_max = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, NUM_PAR_SOFTMAX_SEGMENTS),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
segm_expsum = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, NUM_PAR_SOFTMAX_SEGMENTS),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
return segm_output, segm_max, segm_expsum
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens",
|
||||
[
|
||||
[(1, 1328), (5, 18), (129, 463)], # mixed prefill + decode
|
||||
[(1, 523), (1, 37), (1, 2011)], # decode-only (exercises 3D path)
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_sizes", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("sliding_window", [None, 128])
|
||||
@pytest.mark.parametrize("soft_cap", [None, 50.0])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seq_threshold_3D", SEQ_THRESHOLD_3D_VALUES)
|
||||
@torch.inference_mode()
|
||||
def test_triton_unified_attn_diffkv_vs_fa(
|
||||
seq_lens: list[tuple[int, int]],
|
||||
num_heads: tuple[int, int],
|
||||
head_sizes: tuple[int, int],
|
||||
sliding_window: int | None,
|
||||
soft_cap: float | None,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
seq_threshold_3D: int,
|
||||
) -> None:
|
||||
head_size_qk, head_size_v = head_sizes
|
||||
|
||||
# DiffKV requires FA3 (Hopper) / FA4 (Blackwell) as the reference.
|
||||
fa_version = get_flash_attn_version(head_size=head_size_qk, head_size_v=head_size_v)
|
||||
if not is_flash_attn_varlen_func_available() or fa_version not in (3, 4):
|
||||
pytest.skip(f"FA DiffKV needs FA3/FA4 (got version {fa_version}).")
|
||||
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
|
||||
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
set_random_seed(0)
|
||||
|
||||
num_seqs = len(seq_lens)
|
||||
query_lens = [x[0] for x in seq_lens]
|
||||
kv_lens = [x[1] for x in seq_lens]
|
||||
num_query_heads, num_kv_heads = num_heads
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
max_query_len = max(query_lens)
|
||||
max_kv_len = max(kv_lens)
|
||||
window_size = (sliding_window - 1, 0) if sliding_window is not None else (-1, -1)
|
||||
scale = head_size_qk**-0.5
|
||||
|
||||
query = torch.randn(sum(query_lens), num_query_heads, head_size_qk, dtype=dtype)
|
||||
# Packed KV cache: [num_blocks, block_size, num_kv_heads, hqk + hv].
|
||||
kv_cache = torch.randn(
|
||||
NUM_BLOCKS,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size_qk + head_size_v,
|
||||
dtype=dtype,
|
||||
)
|
||||
key_cache = kv_cache[..., :head_size_qk]
|
||||
value_cache = kv_cache[..., head_size_qk:]
|
||||
|
||||
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
|
||||
dim=0, dtype=torch.int32
|
||||
)
|
||||
kv_lens_t = torch.tensor(kv_lens, dtype=torch.int32)
|
||||
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
# ---- FlashAttention DiffKV (ground truth) ---------------------------
|
||||
# Mirror the backend: fix degenerate strides on size-1 dims so FA's
|
||||
# TMA path sees ≥16-byte-aligned strides (matters for num_kv_heads==1).
|
||||
fa_k = canonicalize_singleton_dim_strides(key_cache)
|
||||
fa_v = canonicalize_singleton_dim_strides(value_cache)
|
||||
fa_out = torch.empty(sum(query_lens), num_query_heads, head_size_v, dtype=dtype)
|
||||
flash_attn_varlen_func(
|
||||
q=query,
|
||||
k=fa_k,
|
||||
v=fa_v,
|
||||
out=fa_out,
|
||||
cu_seqlens_q=cu_query_lens,
|
||||
max_seqlen_q=max_query_len,
|
||||
seqused_k=kv_lens_t,
|
||||
max_seqlen_k=max_kv_len,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=list(window_size),
|
||||
block_table=block_tables,
|
||||
softcap=soft_cap if soft_cap is not None else 0,
|
||||
fa_version=fa_version,
|
||||
)
|
||||
|
||||
# ---- Triton DiffKV --------------------------------------------------
|
||||
segm_output, segm_max, segm_expsum = _alloc_segm_buffers(
|
||||
seq_threshold_3D, num_query_heads, head_size_v
|
||||
)
|
||||
triton_out = torch.empty(sum(query_lens), num_query_heads, head_size_v, dtype=dtype)
|
||||
unified_attention_diffkv(
|
||||
q=query,
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
out=triton_out,
|
||||
cu_seqlens_q=cu_query_lens,
|
||||
seqused_k=kv_lens_t,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
block_table=block_tables,
|
||||
softcap=soft_cap if soft_cap is not None else 0,
|
||||
max_seqlen_q=max_query_len,
|
||||
seq_threshold_3D=seq_threshold_3D,
|
||||
num_par_softmax_segments=NUM_PAR_SOFTMAX_SEGMENTS,
|
||||
softmax_segm_output=segm_output,
|
||||
softmax_segm_max=segm_max,
|
||||
softmax_segm_expsum=segm_expsum,
|
||||
)
|
||||
|
||||
(
|
||||
torch.testing.assert_close(triton_out, fa_out, atol=2e-2, rtol=2e-2),
|
||||
f"triton vs FA max abs diff: {torch.max(torch.abs(triton_out - fa_out))}",
|
||||
)
|
||||
@@ -47,9 +47,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.flash_attn_diffkv import (
|
||||
FlashAttentionDiffKVBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interfaces import MixtureOfExperts, SupportsPP
|
||||
from .utils import (
|
||||
@@ -292,11 +290,27 @@ class MiMoV2Attention(nn.Module):
|
||||
|
||||
sliding_window = sliding_window_size if sliding_window_size > -1 else None
|
||||
|
||||
# Use DiffKV backend when V has a different head dim than K
|
||||
# Use DiffKV backend when V has a different head dim than K.
|
||||
# Auto-pick FA-DiffKV when FA3/4 is usable on this device, else fall
|
||||
# back to TRITON_ATTN_DIFFKV. Users can force a choice via
|
||||
# `--attention-backend <FLASH_ATTN_DIFFKV|TRITON_ATTN_DIFFKV>`.
|
||||
if self.v_head_dim != self.head_dim:
|
||||
FlashAttentionDiffKVBackend.set_head_size_v(self.v_head_dim)
|
||||
attn_backend = FlashAttentionDiffKVBackend
|
||||
logger.info_once("Using FlashAttentionDiffKVBackend for attention.")
|
||||
requested = get_current_vllm_config().attention_config.backend
|
||||
if requested is not None and requested.name.endswith("_DIFFKV"):
|
||||
backend_enum = requested
|
||||
else:
|
||||
fa_backend = AttentionBackendEnum.FLASH_ATTN_DIFFKV.get_class()
|
||||
if fa_backend.is_supported_on_current_device(
|
||||
head_size=self.head_dim,
|
||||
head_size_v=self.v_head_dim,
|
||||
has_sinks=self.attention_sink_bias is not None,
|
||||
):
|
||||
backend_enum = AttentionBackendEnum.FLASH_ATTN_DIFFKV
|
||||
else:
|
||||
backend_enum = AttentionBackendEnum.TRITON_ATTN_DIFFKV
|
||||
attn_backend = backend_enum.get_class()
|
||||
attn_backend.set_head_size_v(self.v_head_dim)
|
||||
logger.info_once("Using %s for attention.", attn_backend.get_name())
|
||||
else:
|
||||
attn_backend = None
|
||||
|
||||
|
||||
@@ -41,6 +41,30 @@ class FlashAttentionDiffKVBackend(FlashAttentionBackend):
|
||||
def set_head_size_v(cls, head_size_v: int) -> None:
|
||||
cls.head_size_v = head_size_v
|
||||
|
||||
@classmethod
|
||||
def is_supported_on_current_device(
|
||||
cls,
|
||||
head_size: int,
|
||||
head_size_v: int,
|
||||
has_sinks: bool,
|
||||
) -> bool:
|
||||
"""Check whether FA3/4 with this DiffKV config is usable here.
|
||||
|
||||
DiffKV (hdim_qk != hdim_v) requires FA3 or FA4
|
||||
"""
|
||||
if not is_flash_attn_varlen_func_available():
|
||||
return False
|
||||
try:
|
||||
version = get_flash_attn_version(
|
||||
requires_alibi=False,
|
||||
head_size=head_size,
|
||||
head_size_v=head_size_v,
|
||||
has_sinks=has_sinks,
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
return version in (3, 4)
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASH_ATTN_DIFFKV"
|
||||
@@ -49,8 +73,6 @@ class FlashAttentionDiffKVBackend(FlashAttentionBackend):
|
||||
def get_impl_cls() -> type["FlashAttentionImpl"]:
|
||||
return FlashAttentionDiffKVImpl
|
||||
|
||||
# Do not modify the interface of get_kv_cache_shape,
|
||||
# but consider head_size_v when returning result.
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
|
||||
@@ -46,6 +46,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
|
||||
"vllm.v1.attention.backends.flash_attn_diffkv.FlashAttentionDiffKVBackend"
|
||||
)
|
||||
TRITON_ATTN = "vllm.v1.attention.backends.triton_attn.TritonAttentionBackend"
|
||||
TRITON_ATTN_DIFFKV = (
|
||||
"vllm.v1.attention.backends.triton_attn_diffkv.TritonAttentionDiffKVBackend"
|
||||
)
|
||||
ROCM_ATTN = "vllm.v1.attention.backends.rocm_attn.RocmAttentionBackend"
|
||||
ROCM_AITER_MLA = "vllm.v1.attention.backends.mla.rocm_aiter_mla.AiterMLABackend"
|
||||
ROCM_AITER_TRITON_MLA = (
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton attention backend with different K/V head dimensions (DiffKV).
|
||||
|
||||
The KV cache layout is identical to ``FlashAttentionDiffKVBackend`` — K
|
||||
and V are packed along the last dim:
|
||||
|
||||
[num_blocks, block_size, num_kv_heads, head_size_qk + head_size_v]
|
||||
|
||||
so existing helpers (``triton_reshape_and_cache_flash_diffkv``) are reused.
|
||||
"""
|
||||
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.attention.backend import AttentionLayer, AttentionType
|
||||
from vllm.v1.attention.backends.triton_attn import (
|
||||
TritonAttentionBackend,
|
||||
TritonAttentionImpl,
|
||||
TritonAttentionMetadata,
|
||||
TritonAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
||||
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
|
||||
triton_reshape_and_cache_flash_diffkv,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_unified_attention_diffkv import (
|
||||
unified_attention_diffkv,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TritonAttentionDiffKVMetadataBuilder(TritonAttentionMetadataBuilder):
|
||||
"""Override the parent's softmax buffer last-dim to head_size_v.
|
||||
|
||||
The parent allocates ``softmax_segm_output`` with last-dim sized to
|
||||
``next_power_of_2(head_size)`` (== Q/K head size). For DiffKV the
|
||||
accumulator and per-segment partial outputs are V-shaped, so we
|
||||
re-allocate with ``next_power_of_2(head_size_v)`` instead.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
layer_names: list[str],
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
):
|
||||
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
|
||||
|
||||
head_size_v = TritonAttentionDiffKVBackend.head_size_v
|
||||
head_size_v_padded = next_power_of_2(head_size_v)
|
||||
self.softmax_segm_output = torch.empty(
|
||||
(
|
||||
self.seq_threshold_3D,
|
||||
self.num_heads_q,
|
||||
self.num_par_softmax_segments,
|
||||
head_size_v_padded,
|
||||
),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
class TritonAttentionDiffKVBackend(TritonAttentionBackend):
|
||||
# V head dim — set per layer via ``set_head_size_v`` before instantiation.
|
||||
head_size_v: int = 128
|
||||
|
||||
# No FP8 / int8 KV cache for the DiffKV path yet; require fp16/bf16/fp32.
|
||||
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
|
||||
"auto",
|
||||
"bfloat16",
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def set_head_size_v(cls, head_size_v: int) -> None:
|
||||
cls.head_size_v = head_size_v
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "TRITON_ATTN_DIFFKV"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["TritonAttentionDiffKVImpl"]:
|
||||
return TritonAttentionDiffKVImpl
|
||||
|
||||
@staticmethod
|
||||
def get_builder_cls() -> type["TritonAttentionDiffKVMetadataBuilder"]:
|
||||
return TritonAttentionDiffKVMetadataBuilder
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
if block_size % 16 != 0:
|
||||
raise ValueError("Block size must be a multiple of 16.")
|
||||
return (
|
||||
num_blocks,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
head_size + TritonAttentionDiffKVBackend.head_size_v,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
cache_layout = get_kv_cache_layout()
|
||||
if cache_layout == "NHD" and include_num_layers_dimension:
|
||||
# (num_blocks, num_layers, block_size,
|
||||
# num_kv_heads, head_size + head_size_v)
|
||||
return (1, 0, 2, 3, 4)
|
||||
elif cache_layout == "NHD":
|
||||
return (0, 1, 2, 3)
|
||||
elif cache_layout == "HND" and include_num_layers_dimension:
|
||||
# (num_blocks, num_kv_heads, num_layers,
|
||||
# block_size, head_size + head_size_v)
|
||||
return (1, 3, 0, 2, 4)
|
||||
elif cache_layout == "HND":
|
||||
return (0, 2, 1, 3)
|
||||
else:
|
||||
raise ValueError(f"Unknown cache layout format {cache_layout}.")
|
||||
|
||||
@classmethod
|
||||
def supports_head_size(cls, head_size: int) -> bool:
|
||||
# DiffKV K head sizes (e.g. 192 for MiMo-V2.5) need to be allowed.
|
||||
return head_size >= 32
|
||||
|
||||
@classmethod
|
||||
def supports_attn_type(cls, attn_type: str) -> bool:
|
||||
# DiffKV only implements decoder self-attention. Unlike the parent
|
||||
# TritonAttentionBackend (which advertises all types), encoder
|
||||
# attention is not supported, so gate it here at backend selection.
|
||||
return attn_type == AttentionType.DECODER
|
||||
|
||||
|
||||
class TritonAttentionDiffKVImpl(TritonAttentionImpl):
|
||||
"""Triton attention impl for the DiffKV packed KV cache layout."""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
raise NotImplementedError(
|
||||
"TritonAttentionDiffKVBackend does not yet support quantized "
|
||||
f"KV cache (got kv_cache_dtype={self.kv_cache_dtype!r})."
|
||||
)
|
||||
if self._is_per_token_head_quant:
|
||||
raise NotImplementedError(
|
||||
"TritonAttentionDiffKVBackend does not support per-token-head "
|
||||
"quantization."
|
||||
)
|
||||
if self.chunk_lookback > -1:
|
||||
raise NotImplementedError(
|
||||
"TritonAttentionDiffKVBackend does not support chunked "
|
||||
"attention with lookback."
|
||||
)
|
||||
|
||||
def do_kv_cache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> None:
|
||||
# Cache is packed [..., head_size_qk + head_size_v]; the diffkv
|
||||
# reshape kernel writes K to [..., :head_size_qk] and V to
|
||||
# [..., head_size_qk:hqk+hv].
|
||||
triton_reshape_and_cache_flash_diffkv(
|
||||
key,
|
||||
value,
|
||||
kv_cache,
|
||||
slot_mapping,
|
||||
self.kv_cache_dtype,
|
||||
layer._k_scale,
|
||||
layer._v_scale,
|
||||
)
|
||||
|
||||
def fused_rope_kvcache_supported(self):
|
||||
# The fused rope+cache path assumes the standard 2-tensor layout.
|
||||
return False
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
attn_metadata: TritonAttentionMetadata,
|
||||
output: torch.Tensor,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Forward pass.
|
||||
|
||||
Shapes:
|
||||
query: [num_tokens, num_heads, head_size_qk]
|
||||
key: [num_tokens, num_kv_heads, head_size_qk]
|
||||
value: [num_tokens, num_kv_heads, head_size_v]
|
||||
kv_cache: [num_blocks, block_size, num_kv_heads,
|
||||
head_size_qk + head_size_v]
|
||||
output: [num_tokens, num_heads, head_size_v]
|
||||
"""
|
||||
if output_scale is not None or output_block_scale is not None:
|
||||
raise NotImplementedError(
|
||||
"fused output quantization is not supported for "
|
||||
"TritonAttentionDiffKVImpl"
|
||||
)
|
||||
|
||||
if attn_metadata is None:
|
||||
return output.fill_(0)
|
||||
|
||||
assert attn_metadata.use_cascade is False, (
|
||||
"Cascade attention not supported for TritonAttentionDiffKVImpl"
|
||||
)
|
||||
|
||||
num_actual_tokens = attn_metadata.num_actual_tokens
|
||||
head_size_qk = self.head_size
|
||||
head_size_v = TritonAttentionDiffKVBackend.head_size_v
|
||||
|
||||
# Slice the packed cache into K / V views. Strides on dims 0/1/2
|
||||
# match the original cache; dim 3 stays contiguous (stride 1).
|
||||
key_cache = kv_cache[..., :head_size_qk]
|
||||
value_cache = kv_cache[..., head_size_qk : head_size_qk + head_size_v]
|
||||
|
||||
unified_attention_diffkv(
|
||||
q=query[:num_actual_tokens],
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
out=output[:num_actual_tokens],
|
||||
cu_seqlens_q=attn_metadata.query_start_loc,
|
||||
seqused_k=attn_metadata.seq_lens,
|
||||
softmax_scale=self.scale,
|
||||
causal=True,
|
||||
alibi_slopes=self.alibi_slopes,
|
||||
use_alibi_sqrt=self.use_alibi_sqrt,
|
||||
window_size=self.sliding_window,
|
||||
block_table=attn_metadata.block_table,
|
||||
softcap=self.logits_soft_cap,
|
||||
sinks=self.sinks,
|
||||
max_seqlen_q=attn_metadata.max_query_len,
|
||||
seq_threshold_3D=attn_metadata.seq_threshold_3D,
|
||||
num_par_softmax_segments=attn_metadata.num_par_softmax_segments,
|
||||
softmax_segm_output=attn_metadata.softmax_segm_output,
|
||||
softmax_segm_max=attn_metadata.softmax_segm_max,
|
||||
softmax_segm_expsum=attn_metadata.softmax_segm_expsum,
|
||||
)
|
||||
return output
|
||||
@@ -0,0 +1,529 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton unified attention with different K/V head dimensions (DiffKV).
|
||||
|
||||
This is a slimmed fork of ``triton_unified_attention.py`` for models like
|
||||
MiMo-V2.5 where the V tensor's head dimension differs from K's. The KV cache
|
||||
is the same packed layout used by ``FlashAttentionDiffKVBackend``:
|
||||
|
||||
kv_cache: [num_blocks, block_size, num_kv_heads, head_size_qk + head_size_v]
|
||||
|
||||
We slice ``key_cache = kv_cache[..., :head_size_qk]`` and
|
||||
``value_cache = kv_cache[..., head_size_qk:]`` on the host, so the kernel
|
||||
takes two cache pointers but with two distinct head sizes.
|
||||
|
||||
Both 2D and 3D launches are supported:
|
||||
- 2D: one program per (q-block, kv-head); tile-loop walks the full KV
|
||||
sequence; final output written directly. Used for prefill and large
|
||||
decode batches.
|
||||
- 3D: one program per (q-block, kv-head, segm); each program covers a
|
||||
KV slice and writes per-segment partials (max/expsum/output). A
|
||||
follow-up ``kernel_reduce_segments_diffkv`` combines them. Selected
|
||||
for decode-only batches whose 2D grid would under-fill the GPU.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.triton_attention_helpers import (
|
||||
apply_alibi_to_score,
|
||||
apply_softcap,
|
||||
cdiv_fn,
|
||||
compute_kv_seq_mask,
|
||||
compute_tile_loop_bounds,
|
||||
find_seq_idx,
|
||||
init_softmax_M,
|
||||
resolve_seq_and_query_len,
|
||||
softmax_step,
|
||||
store_segm_reduce_scalars,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
is_batch_invariant = envs.VLLM_BATCH_INVARIANT
|
||||
|
||||
|
||||
@triton.jit
|
||||
def kernel_unified_attention_diffkv(
|
||||
# Output destinations. In 2D mode we write the final result into
|
||||
# ``output_ptr``; in 3D mode we write per-segment partials into
|
||||
# ``segm_*`` and ``output_ptr`` is unused (callers may pass any
|
||||
# non-null pointer).
|
||||
output_ptr,
|
||||
segm_output_ptr,
|
||||
segm_max_ptr,
|
||||
segm_expsum_ptr,
|
||||
query_ptr,
|
||||
key_cache_ptr, # view of packed cache: [..., :head_size_qk]
|
||||
value_cache_ptr, # view of packed cache: [..., head_size_qk:hqk+hv]
|
||||
sink_ptr,
|
||||
block_tables_ptr,
|
||||
seq_lens_ptr,
|
||||
alibi_slopes_ptr,
|
||||
scale,
|
||||
softcap,
|
||||
num_query_heads: tl.constexpr,
|
||||
num_queries_per_kv: tl.constexpr,
|
||||
block_table_stride: tl.int64,
|
||||
query_stride_0: tl.int64,
|
||||
query_stride_1: tl.int64, # == HEAD_SIZE_QK
|
||||
output_stride_0: tl.int64,
|
||||
output_stride_1: tl.int64, # == HEAD_SIZE_V
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
TILE_SIZE: tl.constexpr,
|
||||
HEAD_SIZE_QK: tl.constexpr,
|
||||
HEAD_SIZE_QK_PADDED: tl.constexpr,
|
||||
HEAD_SIZE_V: tl.constexpr,
|
||||
HEAD_SIZE_V_PADDED: tl.constexpr,
|
||||
USE_ALIBI_SLOPES: tl.constexpr,
|
||||
USE_ALIBI_SQRT: tl.constexpr,
|
||||
USE_SOFTCAP: tl.constexpr,
|
||||
USE_SINKS: tl.constexpr,
|
||||
SLIDING_WINDOW: tl.constexpr,
|
||||
# Strides for both cache views (they share the same packed buffer, so
|
||||
# dims 0/1/2 strides match; only the per-head extent differs).
|
||||
stride_k_cache_0: tl.int64,
|
||||
stride_k_cache_1: tl.int64,
|
||||
stride_k_cache_2: tl.int64,
|
||||
stride_k_cache_3: tl.constexpr,
|
||||
stride_v_cache_0: tl.int64,
|
||||
stride_v_cache_1: tl.int64,
|
||||
stride_v_cache_2: tl.int64,
|
||||
stride_v_cache_3: tl.constexpr,
|
||||
query_start_len_ptr,
|
||||
BLOCK_Q: tl.constexpr,
|
||||
num_seqs: tl.int32,
|
||||
BLOCK_M: tl.constexpr,
|
||||
NUM_SEGMENTS_PER_SEQ: tl.constexpr,
|
||||
# ``IS_3D`` toggles between 2D layout (one program walks the full KV
|
||||
# sequence) and 3D layout (split-KV / FlashDecoding-style: per-segm
|
||||
# programs write partials, finalized by ``kernel_reduce_segments_diffkv``).
|
||||
IS_3D: tl.constexpr,
|
||||
):
|
||||
q_block_global_idx = tl.program_id(0)
|
||||
kv_head_idx = tl.program_id(1)
|
||||
segm_idx = tl.program_id(2) if IS_3D else 0
|
||||
|
||||
(
|
||||
seq_idx,
|
||||
q_block_local_idx,
|
||||
cur_batch_in_all_start_index,
|
||||
cur_batch_query_len,
|
||||
seq_len,
|
||||
) = resolve_seq_and_query_len(
|
||||
query_start_len_ptr, seq_lens_ptr, q_block_global_idx, num_seqs, BLOCK_Q
|
||||
)
|
||||
|
||||
if q_block_local_idx * BLOCK_Q >= cur_batch_query_len:
|
||||
return
|
||||
|
||||
if IS_3D:
|
||||
tiles_per_segment = cdiv_fn(seq_len, NUM_SEGMENTS_PER_SEQ * TILE_SIZE)
|
||||
if segm_idx * tiles_per_segment * TILE_SIZE >= seq_len:
|
||||
return
|
||||
else:
|
||||
tiles_per_segment = 0
|
||||
|
||||
offs_m = tl.arange(0, BLOCK_M)
|
||||
offs_d_qk = tl.arange(0, HEAD_SIZE_QK_PADDED)
|
||||
offs_d_v = tl.arange(0, HEAD_SIZE_V_PADDED)
|
||||
offs_t = tl.arange(0, TILE_SIZE)
|
||||
query_pos = q_block_local_idx * BLOCK_Q + offs_m // num_queries_per_kv
|
||||
|
||||
query_offset_0 = cur_batch_in_all_start_index + query_pos
|
||||
query_offset_1 = kv_head_idx * num_queries_per_kv + offs_m % num_queries_per_kv
|
||||
query_offset = (
|
||||
query_offset_0[:, None] * query_stride_0
|
||||
+ query_offset_1[:, None] * query_stride_1
|
||||
+ offs_d_qk[None, :]
|
||||
)
|
||||
|
||||
dim_mask_qk = tl.where(offs_d_qk < HEAD_SIZE_QK, 1, 0).to(tl.int1)
|
||||
dim_mask_v = tl.where(offs_d_v < HEAD_SIZE_V, 1, 0).to(tl.int1)
|
||||
query_mask_0 = tl.where(query_pos < cur_batch_query_len, 1, 0).to(tl.int1)
|
||||
query_mask_1 = tl.where(query_offset_1 < num_query_heads, 1, 0).to(tl.int1)
|
||||
|
||||
# Q : (BLOCK_M, HEAD_SIZE_QK_PADDED)
|
||||
Q = tl.load(
|
||||
query_ptr + query_offset,
|
||||
mask=dim_mask_qk[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
block_table_offset = seq_idx * block_table_stride
|
||||
|
||||
M = init_softmax_M(
|
||||
sink_ptr, query_offset_1, query_mask_1, segm_idx, BLOCK_M, USE_SINKS, IS_3D
|
||||
)
|
||||
L = tl.full([BLOCK_M], 1.0, dtype=tl.float32)
|
||||
# acc : (BLOCK_M, HEAD_SIZE_V_PADDED)
|
||||
acc = tl.zeros([BLOCK_M, HEAD_SIZE_V_PADDED], dtype=tl.float32)
|
||||
|
||||
context_len = seq_len - cur_batch_query_len
|
||||
|
||||
if USE_ALIBI_SLOPES:
|
||||
alibi_slope = tl.load(
|
||||
alibi_slopes_ptr + query_offset_1, mask=query_mask_1, other=0.0
|
||||
)
|
||||
|
||||
loop_lo, loop_hi, max_seq_prefix_len = compute_tile_loop_bounds(
|
||||
context_len,
|
||||
seq_len,
|
||||
cur_batch_query_len,
|
||||
q_block_local_idx,
|
||||
segm_idx,
|
||||
tiles_per_segment,
|
||||
TILE_SIZE,
|
||||
BLOCK_M,
|
||||
BLOCK_Q,
|
||||
num_queries_per_kv,
|
||||
SLIDING_WINDOW,
|
||||
False, # USE_MM_PREFIX
|
||||
IS_3D,
|
||||
)
|
||||
|
||||
for j in range(loop_lo, loop_hi):
|
||||
seq_offset = j * TILE_SIZE + offs_t
|
||||
tile_mask = seq_offset < max_seq_prefix_len
|
||||
|
||||
physical_block_idx = tl.load(
|
||||
block_tables_ptr + block_table_offset + seq_offset // BLOCK_SIZE
|
||||
).to(tl.int64)
|
||||
|
||||
v_offset = (
|
||||
physical_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_2
|
||||
+ offs_d_v[None, :] * stride_v_cache_3
|
||||
+ (seq_offset % BLOCK_SIZE)[:, None] * stride_v_cache_1
|
||||
)
|
||||
k_offset = (
|
||||
physical_block_idx[None, :] * stride_k_cache_0
|
||||
+ kv_head_idx * stride_k_cache_2
|
||||
+ offs_d_qk[:, None] * stride_k_cache_3
|
||||
+ (seq_offset % BLOCK_SIZE)[None, :] * stride_k_cache_1
|
||||
)
|
||||
# K : (HEAD_SIZE_QK_PADDED, TILE_SIZE)
|
||||
K_load = tl.load(
|
||||
key_cache_ptr + k_offset,
|
||||
mask=dim_mask_qk[:, None] & tile_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
K = K_load.to(Q.dtype)
|
||||
# V : (TILE_SIZE, HEAD_SIZE_V_PADDED)
|
||||
V_load = tl.load(
|
||||
value_cache_ptr + v_offset,
|
||||
mask=dim_mask_v[None, :] & tile_mask[:, None],
|
||||
other=0.0,
|
||||
)
|
||||
V = V_load.to(Q.dtype)
|
||||
|
||||
query_abs_pos = context_len + query_pos[:, None]
|
||||
seq_mask = compute_kv_seq_mask(
|
||||
query_abs_pos,
|
||||
seq_offset,
|
||||
seq_idx,
|
||||
None, # mm_prefix_range_ptr
|
||||
SLIDING_WINDOW,
|
||||
False, # USE_MM_PREFIX
|
||||
0, # MAX_MM_RANGES
|
||||
)
|
||||
|
||||
# S : (BLOCK_M, TILE_SIZE)
|
||||
S = tl.zeros(shape=(BLOCK_M, TILE_SIZE), dtype=tl.float32)
|
||||
S += scale * tl.dot(Q, K)
|
||||
|
||||
if USE_SOFTCAP:
|
||||
S = apply_softcap(S, softcap)
|
||||
|
||||
S = tl.where(
|
||||
query_mask_1[:, None] & query_mask_0[:, None] & seq_mask, S, float("-inf")
|
||||
)
|
||||
|
||||
if USE_ALIBI_SLOPES:
|
||||
S = apply_alibi_to_score(
|
||||
S, alibi_slope, seq_offset, context_len, query_pos, USE_ALIBI_SQRT
|
||||
)
|
||||
|
||||
M, L, P, alpha = softmax_step(S, M, L)
|
||||
acc = acc * alpha[:, None]
|
||||
|
||||
if SLIDING_WINDOW:
|
||||
qpos_lo = q_block_local_idx * BLOCK_Q
|
||||
V = tl.where(
|
||||
(context_len + qpos_lo - seq_offset[:, None]) < SLIDING_WINDOW,
|
||||
V,
|
||||
0.0,
|
||||
)
|
||||
acc += tl.dot(P.to(V.dtype), V)
|
||||
|
||||
# ---- Epilogue --------------------------------------------------------
|
||||
if IS_3D:
|
||||
# Store per-segment partials; finalized by reduce_segments_diffkv.
|
||||
segm_output_offset = (
|
||||
query_offset_0[:, None].to(tl.int64)
|
||||
* (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_V_PADDED)
|
||||
+ query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_V_PADDED)
|
||||
+ segm_idx * HEAD_SIZE_V_PADDED
|
||||
+ tl.arange(0, HEAD_SIZE_V_PADDED)[None, :]
|
||||
)
|
||||
tl.store(
|
||||
segm_output_ptr + segm_output_offset,
|
||||
acc,
|
||||
mask=dim_mask_v[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
|
||||
)
|
||||
store_segm_reduce_scalars(
|
||||
segm_max_ptr,
|
||||
segm_expsum_ptr,
|
||||
query_offset_0,
|
||||
query_offset_1,
|
||||
segm_idx,
|
||||
M,
|
||||
L,
|
||||
query_mask_0,
|
||||
query_mask_1,
|
||||
num_query_heads,
|
||||
NUM_SEGMENTS_PER_SEQ,
|
||||
)
|
||||
else:
|
||||
acc = acc / L[:, None]
|
||||
output_offset = (
|
||||
query_offset_0[:, None] * output_stride_0
|
||||
+ query_offset_1[:, None] * output_stride_1
|
||||
+ offs_d_v[None, :]
|
||||
)
|
||||
tl.store(
|
||||
output_ptr + output_offset,
|
||||
acc,
|
||||
mask=dim_mask_v[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def kernel_reduce_segments_diffkv(
|
||||
output_ptr, # [num_tokens, num_query_heads, head_size_v]
|
||||
segm_output_ptr,
|
||||
# [num_tokens, num_query_heads, max_num_segments, head_size_v]
|
||||
segm_max_ptr, # [num_tokens, num_query_heads, max_num_segments]
|
||||
segm_expsum_ptr, # [num_tokens, num_query_heads, max_num_segments]
|
||||
seq_lens_ptr, # [num_seqs]
|
||||
num_seqs,
|
||||
num_query_heads: tl.constexpr,
|
||||
output_stride_0: tl.int64,
|
||||
output_stride_1: tl.int64, # == HEAD_SIZE_V
|
||||
TILE_SIZE: tl.constexpr,
|
||||
HEAD_SIZE_V: tl.constexpr,
|
||||
HEAD_SIZE_V_PADDED: tl.constexpr,
|
||||
query_start_len_ptr, # [num_seqs+1]
|
||||
BLOCK_Q: tl.constexpr,
|
||||
NUM_SEGMENTS_PER_SEQ: tl.constexpr,
|
||||
):
|
||||
"""Combine per-segment partials into the final softmax output.
|
||||
|
||||
Mirrors ``reduce_segments`` from triton_unified_attention.py but
|
||||
indexes V's head size (``HEAD_SIZE_V``) instead of the shared one.
|
||||
"""
|
||||
query_token_idx = tl.program_id(0)
|
||||
query_head_idx = tl.program_id(1)
|
||||
|
||||
seq_idx = find_seq_idx(
|
||||
query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, False
|
||||
)
|
||||
seq_len = tl.load(seq_lens_ptr + seq_idx)
|
||||
|
||||
tiles_per_segment = cdiv_fn(seq_len, NUM_SEGMENTS_PER_SEQ * TILE_SIZE)
|
||||
act_num_segments = cdiv_fn(seq_len, tiles_per_segment * TILE_SIZE)
|
||||
segm_mask = tl.arange(0, NUM_SEGMENTS_PER_SEQ) < tl.full(
|
||||
[NUM_SEGMENTS_PER_SEQ], act_num_segments, dtype=tl.int32
|
||||
)
|
||||
dim_mask = tl.where(tl.arange(0, HEAD_SIZE_V_PADDED) < HEAD_SIZE_V, 1, 0).to(
|
||||
tl.int1
|
||||
)
|
||||
|
||||
segm_offset = (
|
||||
query_token_idx.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ)
|
||||
+ query_head_idx * NUM_SEGMENTS_PER_SEQ
|
||||
+ tl.arange(0, NUM_SEGMENTS_PER_SEQ)
|
||||
)
|
||||
segm_max = tl.load(segm_max_ptr + segm_offset, mask=segm_mask, other=float("-inf"))
|
||||
overall_max = tl.max(segm_max)
|
||||
|
||||
segm_expsum = tl.load(segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0)
|
||||
segm_expsum = segm_expsum * tl.exp(segm_max - overall_max)
|
||||
overall_expsum = tl.sum(segm_expsum)
|
||||
|
||||
segm_output_offset = (
|
||||
query_token_idx.to(tl.int64)
|
||||
* (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_V_PADDED)
|
||||
+ query_head_idx * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_V_PADDED)
|
||||
+ tl.arange(0, NUM_SEGMENTS_PER_SEQ)[:, None] * HEAD_SIZE_V_PADDED
|
||||
+ tl.arange(0, HEAD_SIZE_V_PADDED)[None, :]
|
||||
)
|
||||
segm_output = tl.load(
|
||||
segm_output_ptr + segm_output_offset,
|
||||
mask=segm_mask[:, None] & dim_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
segm_output *= tl.exp(segm_max - overall_max)[:, None]
|
||||
acc_sum = tl.sum(segm_output, axis=0)
|
||||
acc = tl.where(overall_expsum == 0.0, 0.0, acc_sum / overall_expsum)
|
||||
|
||||
output_offset = (
|
||||
query_token_idx * output_stride_0
|
||||
+ query_head_idx * output_stride_1
|
||||
+ tl.arange(0, HEAD_SIZE_V_PADDED)
|
||||
)
|
||||
tl.store(output_ptr + output_offset, acc, mask=dim_mask)
|
||||
|
||||
|
||||
def unified_attention_diffkv(
|
||||
q, # [num_tokens, num_query_heads, head_size_qk]
|
||||
k, # view: [num_blocks, block_size, num_kv_heads, head_size_qk]
|
||||
v, # view: [num_blocks, block_size, num_kv_heads, head_size_v]
|
||||
out, # [num_tokens, num_query_heads, head_size_v]
|
||||
cu_seqlens_q,
|
||||
seqused_k,
|
||||
softmax_scale,
|
||||
causal,
|
||||
window_size,
|
||||
block_table,
|
||||
softcap,
|
||||
max_seqlen_q: int = 1,
|
||||
alibi_slopes=None,
|
||||
sinks=None,
|
||||
use_alibi_sqrt=False,
|
||||
# 3D / split-KV softmax buffers. When all four are provided and the
|
||||
# batch is decode-only with few sequences, the 3D path is taken.
|
||||
seq_threshold_3D: int | None = None,
|
||||
num_par_softmax_segments: int | None = None,
|
||||
softmax_segm_output: torch.Tensor | None = None,
|
||||
softmax_segm_max: torch.Tensor | None = None,
|
||||
softmax_segm_expsum: torch.Tensor | None = None,
|
||||
):
|
||||
assert causal, "Only causal attention is supported"
|
||||
|
||||
if sinks is not None:
|
||||
assert sinks.shape[0] == q.shape[1], "Sinks must be num_query_heads size"
|
||||
|
||||
use_alibi_slopes = alibi_slopes is not None
|
||||
|
||||
block_size = v.shape[1]
|
||||
num_seqs = len(seqused_k)
|
||||
num_query_heads = q.shape[1]
|
||||
num_kv_heads = k.shape[2]
|
||||
num_queries_per_kv = num_query_heads // num_kv_heads
|
||||
head_size_qk = q.shape[2]
|
||||
head_size_v = v.shape[3]
|
||||
|
||||
BLOCK_M = (
|
||||
16 if num_queries_per_kv <= 16 else triton.next_power_of_2(num_queries_per_kv)
|
||||
)
|
||||
BLOCK_Q = BLOCK_M // num_queries_per_kv
|
||||
|
||||
total_num_q_blocks = q.shape[0] // BLOCK_Q + num_seqs
|
||||
|
||||
sliding_window_val = 1 + window_size[0] if window_size[0] >= 0 else 0
|
||||
|
||||
# Decide between 2D and 3D launch. Mirrors the standard launcher:
|
||||
# 3D requires preallocated softmax buffers, decode-only batches, and
|
||||
# a small number of sequences (otherwise 2D already saturates the SM).
|
||||
use_3d = not (
|
||||
seq_threshold_3D is None
|
||||
or num_par_softmax_segments is None
|
||||
or softmax_segm_output is None
|
||||
or softmax_segm_max is None
|
||||
or softmax_segm_expsum is None
|
||||
or max_seqlen_q > 1
|
||||
or num_seqs > seq_threshold_3D
|
||||
or is_batch_invariant
|
||||
)
|
||||
|
||||
# Tile size: 32 for prefill-class kernels. Decode (small Q) prefers
|
||||
# smaller tiles to expose more parallelism along the KV dim.
|
||||
tile_size = 32 if not use_3d else (16 if q.element_size() >= 2 else 32)
|
||||
|
||||
grid: tuple[Any, ...]
|
||||
if use_3d:
|
||||
grid = (total_num_q_blocks, num_kv_heads, num_par_softmax_segments)
|
||||
segm_output_ptr = softmax_segm_output
|
||||
segm_max_ptr = softmax_segm_max
|
||||
segm_expsum_ptr = softmax_segm_expsum
|
||||
num_segments = num_par_softmax_segments
|
||||
else:
|
||||
grid = (total_num_q_blocks, num_kv_heads)
|
||||
# 2D never touches the segm tensors but Triton wants a non-null
|
||||
# pointer; reuse ``out``.
|
||||
segm_output_ptr = out
|
||||
segm_max_ptr = out
|
||||
segm_expsum_ptr = out
|
||||
num_segments = 1
|
||||
|
||||
kernel_unified_attention_diffkv[grid](
|
||||
output_ptr=out,
|
||||
segm_output_ptr=segm_output_ptr,
|
||||
segm_max_ptr=segm_max_ptr,
|
||||
segm_expsum_ptr=segm_expsum_ptr,
|
||||
query_ptr=q,
|
||||
key_cache_ptr=k,
|
||||
value_cache_ptr=v,
|
||||
sink_ptr=sinks,
|
||||
block_tables_ptr=block_table,
|
||||
seq_lens_ptr=seqused_k,
|
||||
alibi_slopes_ptr=alibi_slopes,
|
||||
scale=softmax_scale,
|
||||
softcap=softcap,
|
||||
num_query_heads=num_query_heads,
|
||||
num_queries_per_kv=num_queries_per_kv,
|
||||
block_table_stride=block_table.stride(0),
|
||||
query_stride_0=q.stride(0),
|
||||
query_stride_1=q.stride(1),
|
||||
output_stride_0=out.stride(0),
|
||||
output_stride_1=out.stride(1),
|
||||
BLOCK_SIZE=block_size,
|
||||
TILE_SIZE=tile_size,
|
||||
HEAD_SIZE_QK=head_size_qk,
|
||||
HEAD_SIZE_QK_PADDED=triton.next_power_of_2(head_size_qk),
|
||||
HEAD_SIZE_V=head_size_v,
|
||||
HEAD_SIZE_V_PADDED=triton.next_power_of_2(head_size_v),
|
||||
USE_ALIBI_SLOPES=use_alibi_slopes,
|
||||
USE_ALIBI_SQRT=use_alibi_sqrt,
|
||||
USE_SOFTCAP=(softcap > 0),
|
||||
USE_SINKS=(sinks is not None),
|
||||
SLIDING_WINDOW=sliding_window_val,
|
||||
stride_k_cache_0=k.stride(0),
|
||||
stride_k_cache_1=k.stride(1),
|
||||
stride_k_cache_2=k.stride(2),
|
||||
stride_k_cache_3=k.stride(3),
|
||||
stride_v_cache_0=v.stride(0),
|
||||
stride_v_cache_1=v.stride(1),
|
||||
stride_v_cache_2=v.stride(2),
|
||||
stride_v_cache_3=v.stride(3),
|
||||
query_start_len_ptr=cu_seqlens_q,
|
||||
BLOCK_Q=BLOCK_Q,
|
||||
num_seqs=num_seqs,
|
||||
BLOCK_M=BLOCK_M,
|
||||
NUM_SEGMENTS_PER_SEQ=num_segments,
|
||||
IS_3D=use_3d,
|
||||
)
|
||||
|
||||
if use_3d:
|
||||
kernel_reduce_segments_diffkv[(q.shape[0], num_query_heads)](
|
||||
output_ptr=out,
|
||||
segm_output_ptr=softmax_segm_output,
|
||||
segm_max_ptr=softmax_segm_max,
|
||||
segm_expsum_ptr=softmax_segm_expsum,
|
||||
seq_lens_ptr=seqused_k,
|
||||
num_seqs=num_seqs,
|
||||
num_query_heads=num_query_heads,
|
||||
output_stride_0=out.stride(0),
|
||||
output_stride_1=out.stride(1),
|
||||
TILE_SIZE=tile_size,
|
||||
HEAD_SIZE_V=head_size_v,
|
||||
HEAD_SIZE_V_PADDED=triton.next_power_of_2(head_size_v),
|
||||
query_start_len_ptr=cu_seqlens_q,
|
||||
BLOCK_Q=BLOCK_Q,
|
||||
NUM_SEGMENTS_PER_SEQ=num_par_softmax_segments,
|
||||
)
|
||||
Reference in New Issue
Block a user