[Attention] add triton diff-kv backend for mimo (#41797)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Jiangyun Zhu
2026-06-11 11:36:31 -04:00
committed by GitHub
parent 4085ff7cb4
commit f81daf8880
8 changed files with 1041 additions and 9 deletions
+13
View File
@@ -75,6 +75,19 @@ steps:
- pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 2
- label: Kernels Attention DiffKV Test (H100)
key: kernels-attention-diffkv-test-h100
timeout_in_minutes: 20
device: h100
num_devices: 1
source_file_dependencies:
- vllm/v1/attention/ops/triton_unified_attention_diffkv.py
- vllm/v1/attention/backends/triton_attn_diffkv.py
- vllm/v1/attention/backends/flash_attn_diffkv.py
- tests/kernels/attention/test_triton_unified_attention_diffkv.py
commands:
- pytest -v -s kernels/attention/test_triton_unified_attention_diffkv.py
- label: Kernels Quantization Test %N
key: kernels-quantization-test
timeout_in_minutes: 90
+1
View File
@@ -181,6 +181,7 @@ Priority is **1 = highest** (tried first).
| `ROCM_AITER_UNIFIED_ATTN` | | fp16, bf16 | `auto` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | N/A |
| `ROCM_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 128, 160, 192, 224, 256 | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder, Encoder Only | N/A |
| `TRITON_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `int8_per_token_head`, `fp8_per_token_head` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | Any |
| `TRITON_ATTN_DIFFKV` | | fp16, bf16 | `auto`, `bfloat16` | Any | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
| `TURBOQUANT` | | fp16, bf16 | `turboquant_k8v4`, `turboquant_4bit_nc`, `turboquant_k3v4_nc`, `turboquant_3bit_nc` | 16, 32, 64, 128 | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
> **†** FlashInfer uses TRTLLM attention on Blackwell (SM100), which supports sinks. Disable via `--attention-config.use_trtllm_attention=0`.
@@ -0,0 +1,189 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for the Triton DiffKV unified-attention kernel.
"""
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import (
canonicalize_singleton_dim_strides,
set_random_seed,
)
from vllm.v1.attention.backends.fa_utils import (
get_flash_attn_version,
is_flash_attn_varlen_func_available,
)
from vllm.v1.attention.ops.triton_unified_attention_diffkv import (
unified_attention_diffkv,
)
DEVICE_TYPE = current_platform.device_type
# (num_query_heads, num_kv_heads): MHA, GQA, and the num_kv_heads==1
# (degenerate-stride) case.
NUM_HEADS = [(4, 4), (8, 2), (5, 1)]
# (head_size_qk, head_size_v). (192, 128) is the canonical asymmetric
# DiffKV shape; FA4 on Blackwell only supports head_size>128 when it is
# 192, and FA3 on Hopper supports it too -- so this pair is runnable on
# both. (128, 128) keeps the equal-dim path covered through the DiffKV
# kernel.
HEAD_SIZES = [(128, 128), (192, 128)]
BLOCK_SIZES = [16]
DTYPES = [torch.bfloat16]
NUM_BLOCKS = 2048
# 0: 2D decode kernel; 8: 3D (split-KV) decode kernel.
SEQ_THRESHOLD_3D_VALUES = [0, 8]
NUM_PAR_SOFTMAX_SEGMENTS = 16
def _alloc_segm_buffers(seq_threshold_3D: int, num_query_heads: int, head_size_v: int):
"""Allocate the split-KV softmax scratch (last dim == head_size_v)."""
head_size_v_padded = next_power_of_2(head_size_v)
segm_output = torch.empty(
(
seq_threshold_3D,
num_query_heads,
NUM_PAR_SOFTMAX_SEGMENTS,
head_size_v_padded,
),
dtype=torch.float32,
)
segm_max = torch.empty(
(seq_threshold_3D, num_query_heads, NUM_PAR_SOFTMAX_SEGMENTS),
dtype=torch.float32,
)
segm_expsum = torch.empty(
(seq_threshold_3D, num_query_heads, NUM_PAR_SOFTMAX_SEGMENTS),
dtype=torch.float32,
)
return segm_output, segm_max, segm_expsum
@pytest.mark.parametrize(
"seq_lens",
[
[(1, 1328), (5, 18), (129, 463)], # mixed prefill + decode
[(1, 523), (1, 37), (1, 2011)], # decode-only (exercises 3D path)
],
)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_sizes", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("sliding_window", [None, 128])
@pytest.mark.parametrize("soft_cap", [None, 50.0])
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seq_threshold_3D", SEQ_THRESHOLD_3D_VALUES)
@torch.inference_mode()
def test_triton_unified_attn_diffkv_vs_fa(
seq_lens: list[tuple[int, int]],
num_heads: tuple[int, int],
head_sizes: tuple[int, int],
sliding_window: int | None,
soft_cap: float | None,
dtype: torch.dtype,
block_size: int,
seq_threshold_3D: int,
) -> None:
head_size_qk, head_size_v = head_sizes
# DiffKV requires FA3 (Hopper) / FA4 (Blackwell) as the reference.
fa_version = get_flash_attn_version(head_size=head_size_qk, head_size_v=head_size_v)
if not is_flash_attn_varlen_func_available() or fa_version not in (3, 4):
pytest.skip(f"FA DiffKV needs FA3/FA4 (got version {fa_version}).")
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
torch.set_default_device(DEVICE_TYPE)
set_random_seed(0)
num_seqs = len(seq_lens)
query_lens = [x[0] for x in seq_lens]
kv_lens = [x[1] for x in seq_lens]
num_query_heads, num_kv_heads = num_heads
assert num_query_heads % num_kv_heads == 0
max_query_len = max(query_lens)
max_kv_len = max(kv_lens)
window_size = (sliding_window - 1, 0) if sliding_window is not None else (-1, -1)
scale = head_size_qk**-0.5
query = torch.randn(sum(query_lens), num_query_heads, head_size_qk, dtype=dtype)
# Packed KV cache: [num_blocks, block_size, num_kv_heads, hqk + hv].
kv_cache = torch.randn(
NUM_BLOCKS,
block_size,
num_kv_heads,
head_size_qk + head_size_v,
dtype=dtype,
)
key_cache = kv_cache[..., :head_size_qk]
value_cache = kv_cache[..., head_size_qk:]
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
dim=0, dtype=torch.int32
)
kv_lens_t = torch.tensor(kv_lens, dtype=torch.int32)
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
# ---- FlashAttention DiffKV (ground truth) ---------------------------
# Mirror the backend: fix degenerate strides on size-1 dims so FA's
# TMA path sees ≥16-byte-aligned strides (matters for num_kv_heads==1).
fa_k = canonicalize_singleton_dim_strides(key_cache)
fa_v = canonicalize_singleton_dim_strides(value_cache)
fa_out = torch.empty(sum(query_lens), num_query_heads, head_size_v, dtype=dtype)
flash_attn_varlen_func(
q=query,
k=fa_k,
v=fa_v,
out=fa_out,
cu_seqlens_q=cu_query_lens,
max_seqlen_q=max_query_len,
seqused_k=kv_lens_t,
max_seqlen_k=max_kv_len,
softmax_scale=scale,
causal=True,
window_size=list(window_size),
block_table=block_tables,
softcap=soft_cap if soft_cap is not None else 0,
fa_version=fa_version,
)
# ---- Triton DiffKV --------------------------------------------------
segm_output, segm_max, segm_expsum = _alloc_segm_buffers(
seq_threshold_3D, num_query_heads, head_size_v
)
triton_out = torch.empty(sum(query_lens), num_query_heads, head_size_v, dtype=dtype)
unified_attention_diffkv(
q=query,
k=key_cache,
v=value_cache,
out=triton_out,
cu_seqlens_q=cu_query_lens,
seqused_k=kv_lens_t,
softmax_scale=scale,
causal=True,
window_size=window_size,
block_table=block_tables,
softcap=soft_cap if soft_cap is not None else 0,
max_seqlen_q=max_query_len,
seq_threshold_3D=seq_threshold_3D,
num_par_softmax_segments=NUM_PAR_SOFTMAX_SEGMENTS,
softmax_segm_output=segm_output,
softmax_segm_max=segm_max,
softmax_segm_expsum=segm_expsum,
)
(
torch.testing.assert_close(triton_out, fa_out, atol=2e-2, rtol=2e-2),
f"triton vs FA max abs diff: {torch.max(torch.abs(triton_out - fa_out))}",
)
+21 -7
View File
@@ -47,9 +47,7 @@ from vllm.model_executor.model_loader.weight_utils import (
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.flash_attn_diffkv import (
FlashAttentionDiffKVBackend,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interfaces import MixtureOfExperts, SupportsPP
from .utils import (
@@ -292,11 +290,27 @@ class MiMoV2Attention(nn.Module):
sliding_window = sliding_window_size if sliding_window_size > -1 else None
# Use DiffKV backend when V has a different head dim than K
# Use DiffKV backend when V has a different head dim than K.
# Auto-pick FA-DiffKV when FA3/4 is usable on this device, else fall
# back to TRITON_ATTN_DIFFKV. Users can force a choice via
# `--attention-backend <FLASH_ATTN_DIFFKV|TRITON_ATTN_DIFFKV>`.
if self.v_head_dim != self.head_dim:
FlashAttentionDiffKVBackend.set_head_size_v(self.v_head_dim)
attn_backend = FlashAttentionDiffKVBackend
logger.info_once("Using FlashAttentionDiffKVBackend for attention.")
requested = get_current_vllm_config().attention_config.backend
if requested is not None and requested.name.endswith("_DIFFKV"):
backend_enum = requested
else:
fa_backend = AttentionBackendEnum.FLASH_ATTN_DIFFKV.get_class()
if fa_backend.is_supported_on_current_device(
head_size=self.head_dim,
head_size_v=self.v_head_dim,
has_sinks=self.attention_sink_bias is not None,
):
backend_enum = AttentionBackendEnum.FLASH_ATTN_DIFFKV
else:
backend_enum = AttentionBackendEnum.TRITON_ATTN_DIFFKV
attn_backend = backend_enum.get_class()
attn_backend.set_head_size_v(self.v_head_dim)
logger.info_once("Using %s for attention.", attn_backend.get_name())
else:
attn_backend = None
@@ -41,6 +41,30 @@ class FlashAttentionDiffKVBackend(FlashAttentionBackend):
def set_head_size_v(cls, head_size_v: int) -> None:
cls.head_size_v = head_size_v
@classmethod
def is_supported_on_current_device(
cls,
head_size: int,
head_size_v: int,
has_sinks: bool,
) -> bool:
"""Check whether FA3/4 with this DiffKV config is usable here.
DiffKV (hdim_qk != hdim_v) requires FA3 or FA4
"""
if not is_flash_attn_varlen_func_available():
return False
try:
version = get_flash_attn_version(
requires_alibi=False,
head_size=head_size,
head_size_v=head_size_v,
has_sinks=has_sinks,
)
except Exception:
return False
return version in (3, 4)
@staticmethod
def get_name() -> str:
return "FLASH_ATTN_DIFFKV"
@@ -49,8 +73,6 @@ class FlashAttentionDiffKVBackend(FlashAttentionBackend):
def get_impl_cls() -> type["FlashAttentionImpl"]:
return FlashAttentionDiffKVImpl
# Do not modify the interface of get_kv_cache_shape,
# but consider head_size_v when returning result.
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
+3
View File
@@ -46,6 +46,9 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
"vllm.v1.attention.backends.flash_attn_diffkv.FlashAttentionDiffKVBackend"
)
TRITON_ATTN = "vllm.v1.attention.backends.triton_attn.TritonAttentionBackend"
TRITON_ATTN_DIFFKV = (
"vllm.v1.attention.backends.triton_attn_diffkv.TritonAttentionDiffKVBackend"
)
ROCM_ATTN = "vllm.v1.attention.backends.rocm_attn.RocmAttentionBackend"
ROCM_AITER_MLA = "vllm.v1.attention.backends.mla.rocm_aiter_mla.AiterMLABackend"
ROCM_AITER_TRITON_MLA = (
@@ -0,0 +1,261 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton attention backend with different K/V head dimensions (DiffKV).
The KV cache layout is identical to ``FlashAttentionDiffKVBackend`` — K
and V are packed along the last dim:
[num_blocks, block_size, num_kv_heads, head_size_qk + head_size_v]
so existing helpers (``triton_reshape_and_cache_flash_diffkv``) are reused.
"""
from typing import ClassVar
import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.attention.backend import AttentionLayer, AttentionType
from vllm.v1.attention.backends.triton_attn import (
TritonAttentionBackend,
TritonAttentionImpl,
TritonAttentionMetadata,
TritonAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import get_kv_cache_layout
from vllm.v1.attention.ops.triton_reshape_and_cache_flash import (
triton_reshape_and_cache_flash_diffkv,
)
from vllm.v1.attention.ops.triton_unified_attention_diffkv import (
unified_attention_diffkv,
)
from vllm.v1.kv_cache_interface import AttentionSpec
logger = init_logger(__name__)
class TritonAttentionDiffKVMetadataBuilder(TritonAttentionMetadataBuilder):
"""Override the parent's softmax buffer last-dim to head_size_v.
The parent allocates ``softmax_segm_output`` with last-dim sized to
``next_power_of_2(head_size)`` (== Q/K head size). For DiffKV the
accumulator and per-segment partial outputs are V-shaped, so we
re-allocate with ``next_power_of_2(head_size_v)`` instead.
"""
def __init__(
self,
kv_cache_spec: AttentionSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
):
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
head_size_v = TritonAttentionDiffKVBackend.head_size_v
head_size_v_padded = next_power_of_2(head_size_v)
self.softmax_segm_output = torch.empty(
(
self.seq_threshold_3D,
self.num_heads_q,
self.num_par_softmax_segments,
head_size_v_padded,
),
dtype=torch.float32,
device=device,
)
class TritonAttentionDiffKVBackend(TritonAttentionBackend):
# V head dim — set per layer via ``set_head_size_v`` before instantiation.
head_size_v: int = 128
# No FP8 / int8 KV cache for the DiffKV path yet; require fp16/bf16/fp32.
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
"auto",
"bfloat16",
]
@classmethod
def set_head_size_v(cls, head_size_v: int) -> None:
cls.head_size_v = head_size_v
@staticmethod
def get_name() -> str:
return "TRITON_ATTN_DIFFKV"
@staticmethod
def get_impl_cls() -> type["TritonAttentionDiffKVImpl"]:
return TritonAttentionDiffKVImpl
@staticmethod
def get_builder_cls() -> type["TritonAttentionDiffKVMetadataBuilder"]:
return TritonAttentionDiffKVMetadataBuilder
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
if block_size % 16 != 0:
raise ValueError("Block size must be a multiple of 16.")
return (
num_blocks,
block_size,
num_kv_heads,
head_size + TritonAttentionDiffKVBackend.head_size_v,
)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
cache_layout = get_kv_cache_layout()
if cache_layout == "NHD" and include_num_layers_dimension:
# (num_blocks, num_layers, block_size,
# num_kv_heads, head_size + head_size_v)
return (1, 0, 2, 3, 4)
elif cache_layout == "NHD":
return (0, 1, 2, 3)
elif cache_layout == "HND" and include_num_layers_dimension:
# (num_blocks, num_kv_heads, num_layers,
# block_size, head_size + head_size_v)
return (1, 3, 0, 2, 4)
elif cache_layout == "HND":
return (0, 2, 1, 3)
else:
raise ValueError(f"Unknown cache layout format {cache_layout}.")
@classmethod
def supports_head_size(cls, head_size: int) -> bool:
# DiffKV K head sizes (e.g. 192 for MiMo-V2.5) need to be allowed.
return head_size >= 32
@classmethod
def supports_attn_type(cls, attn_type: str) -> bool:
# DiffKV only implements decoder self-attention. Unlike the parent
# TritonAttentionBackend (which advertises all types), encoder
# attention is not supported, so gate it here at backend selection.
return attn_type == AttentionType.DECODER
class TritonAttentionDiffKVImpl(TritonAttentionImpl):
"""Triton attention impl for the DiffKV packed KV cache layout."""
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
if is_quantized_kv_cache(self.kv_cache_dtype):
raise NotImplementedError(
"TritonAttentionDiffKVBackend does not yet support quantized "
f"KV cache (got kv_cache_dtype={self.kv_cache_dtype!r})."
)
if self._is_per_token_head_quant:
raise NotImplementedError(
"TritonAttentionDiffKVBackend does not support per-token-head "
"quantization."
)
if self.chunk_lookback > -1:
raise NotImplementedError(
"TritonAttentionDiffKVBackend does not support chunked "
"attention with lookback."
)
def do_kv_cache_update(
self,
layer: AttentionLayer,
key: torch.Tensor,
value: torch.Tensor,
kv_cache: torch.Tensor,
slot_mapping: torch.Tensor,
) -> None:
# Cache is packed [..., head_size_qk + head_size_v]; the diffkv
# reshape kernel writes K to [..., :head_size_qk] and V to
# [..., head_size_qk:hqk+hv].
triton_reshape_and_cache_flash_diffkv(
key,
value,
kv_cache,
slot_mapping,
self.kv_cache_dtype,
layer._k_scale,
layer._v_scale,
)
def fused_rope_kvcache_supported(self):
# The fused rope+cache path assumes the standard 2-tensor layout.
return False
def forward(
self,
layer: torch.nn.Module,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
kv_cache: torch.Tensor,
attn_metadata: TritonAttentionMetadata,
output: torch.Tensor,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> torch.Tensor:
"""Forward pass.
Shapes:
query: [num_tokens, num_heads, head_size_qk]
key: [num_tokens, num_kv_heads, head_size_qk]
value: [num_tokens, num_kv_heads, head_size_v]
kv_cache: [num_blocks, block_size, num_kv_heads,
head_size_qk + head_size_v]
output: [num_tokens, num_heads, head_size_v]
"""
if output_scale is not None or output_block_scale is not None:
raise NotImplementedError(
"fused output quantization is not supported for "
"TritonAttentionDiffKVImpl"
)
if attn_metadata is None:
return output.fill_(0)
assert attn_metadata.use_cascade is False, (
"Cascade attention not supported for TritonAttentionDiffKVImpl"
)
num_actual_tokens = attn_metadata.num_actual_tokens
head_size_qk = self.head_size
head_size_v = TritonAttentionDiffKVBackend.head_size_v
# Slice the packed cache into K / V views. Strides on dims 0/1/2
# match the original cache; dim 3 stays contiguous (stride 1).
key_cache = kv_cache[..., :head_size_qk]
value_cache = kv_cache[..., head_size_qk : head_size_qk + head_size_v]
unified_attention_diffkv(
q=query[:num_actual_tokens],
k=key_cache,
v=value_cache,
out=output[:num_actual_tokens],
cu_seqlens_q=attn_metadata.query_start_loc,
seqused_k=attn_metadata.seq_lens,
softmax_scale=self.scale,
causal=True,
alibi_slopes=self.alibi_slopes,
use_alibi_sqrt=self.use_alibi_sqrt,
window_size=self.sliding_window,
block_table=attn_metadata.block_table,
softcap=self.logits_soft_cap,
sinks=self.sinks,
max_seqlen_q=attn_metadata.max_query_len,
seq_threshold_3D=attn_metadata.seq_threshold_3D,
num_par_softmax_segments=attn_metadata.num_par_softmax_segments,
softmax_segm_output=attn_metadata.softmax_segm_output,
softmax_segm_max=attn_metadata.softmax_segm_max,
softmax_segm_expsum=attn_metadata.softmax_segm_expsum,
)
return output
@@ -0,0 +1,529 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton unified attention with different K/V head dimensions (DiffKV).
This is a slimmed fork of ``triton_unified_attention.py`` for models like
MiMo-V2.5 where the V tensor's head dimension differs from K's. The KV cache
is the same packed layout used by ``FlashAttentionDiffKVBackend``:
kv_cache: [num_blocks, block_size, num_kv_heads, head_size_qk + head_size_v]
We slice ``key_cache = kv_cache[..., :head_size_qk]`` and
``value_cache = kv_cache[..., head_size_qk:]`` on the host, so the kernel
takes two cache pointers but with two distinct head sizes.
Both 2D and 3D launches are supported:
- 2D: one program per (q-block, kv-head); tile-loop walks the full KV
sequence; final output written directly. Used for prefill and large
decode batches.
- 3D: one program per (q-block, kv-head, segm); each program covers a
KV slice and writes per-segment partials (max/expsum/output). A
follow-up ``kernel_reduce_segments_diffkv`` combines them. Selected
for decode-only batches whose 2D grid would under-fill the GPU.
"""
from typing import Any
import torch
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.attention.ops.triton_attention_helpers import (
apply_alibi_to_score,
apply_softcap,
cdiv_fn,
compute_kv_seq_mask,
compute_tile_loop_bounds,
find_seq_idx,
init_softmax_M,
resolve_seq_and_query_len,
softmax_step,
store_segm_reduce_scalars,
)
logger = init_logger(__name__)
is_batch_invariant = envs.VLLM_BATCH_INVARIANT
@triton.jit
def kernel_unified_attention_diffkv(
# Output destinations. In 2D mode we write the final result into
# ``output_ptr``; in 3D mode we write per-segment partials into
# ``segm_*`` and ``output_ptr`` is unused (callers may pass any
# non-null pointer).
output_ptr,
segm_output_ptr,
segm_max_ptr,
segm_expsum_ptr,
query_ptr,
key_cache_ptr, # view of packed cache: [..., :head_size_qk]
value_cache_ptr, # view of packed cache: [..., head_size_qk:hqk+hv]
sink_ptr,
block_tables_ptr,
seq_lens_ptr,
alibi_slopes_ptr,
scale,
softcap,
num_query_heads: tl.constexpr,
num_queries_per_kv: tl.constexpr,
block_table_stride: tl.int64,
query_stride_0: tl.int64,
query_stride_1: tl.int64, # == HEAD_SIZE_QK
output_stride_0: tl.int64,
output_stride_1: tl.int64, # == HEAD_SIZE_V
BLOCK_SIZE: tl.constexpr,
TILE_SIZE: tl.constexpr,
HEAD_SIZE_QK: tl.constexpr,
HEAD_SIZE_QK_PADDED: tl.constexpr,
HEAD_SIZE_V: tl.constexpr,
HEAD_SIZE_V_PADDED: tl.constexpr,
USE_ALIBI_SLOPES: tl.constexpr,
USE_ALIBI_SQRT: tl.constexpr,
USE_SOFTCAP: tl.constexpr,
USE_SINKS: tl.constexpr,
SLIDING_WINDOW: tl.constexpr,
# Strides for both cache views (they share the same packed buffer, so
# dims 0/1/2 strides match; only the per-head extent differs).
stride_k_cache_0: tl.int64,
stride_k_cache_1: tl.int64,
stride_k_cache_2: tl.int64,
stride_k_cache_3: tl.constexpr,
stride_v_cache_0: tl.int64,
stride_v_cache_1: tl.int64,
stride_v_cache_2: tl.int64,
stride_v_cache_3: tl.constexpr,
query_start_len_ptr,
BLOCK_Q: tl.constexpr,
num_seqs: tl.int32,
BLOCK_M: tl.constexpr,
NUM_SEGMENTS_PER_SEQ: tl.constexpr,
# ``IS_3D`` toggles between 2D layout (one program walks the full KV
# sequence) and 3D layout (split-KV / FlashDecoding-style: per-segm
# programs write partials, finalized by ``kernel_reduce_segments_diffkv``).
IS_3D: tl.constexpr,
):
q_block_global_idx = tl.program_id(0)
kv_head_idx = tl.program_id(1)
segm_idx = tl.program_id(2) if IS_3D else 0
(
seq_idx,
q_block_local_idx,
cur_batch_in_all_start_index,
cur_batch_query_len,
seq_len,
) = resolve_seq_and_query_len(
query_start_len_ptr, seq_lens_ptr, q_block_global_idx, num_seqs, BLOCK_Q
)
if q_block_local_idx * BLOCK_Q >= cur_batch_query_len:
return
if IS_3D:
tiles_per_segment = cdiv_fn(seq_len, NUM_SEGMENTS_PER_SEQ * TILE_SIZE)
if segm_idx * tiles_per_segment * TILE_SIZE >= seq_len:
return
else:
tiles_per_segment = 0
offs_m = tl.arange(0, BLOCK_M)
offs_d_qk = tl.arange(0, HEAD_SIZE_QK_PADDED)
offs_d_v = tl.arange(0, HEAD_SIZE_V_PADDED)
offs_t = tl.arange(0, TILE_SIZE)
query_pos = q_block_local_idx * BLOCK_Q + offs_m // num_queries_per_kv
query_offset_0 = cur_batch_in_all_start_index + query_pos
query_offset_1 = kv_head_idx * num_queries_per_kv + offs_m % num_queries_per_kv
query_offset = (
query_offset_0[:, None] * query_stride_0
+ query_offset_1[:, None] * query_stride_1
+ offs_d_qk[None, :]
)
dim_mask_qk = tl.where(offs_d_qk < HEAD_SIZE_QK, 1, 0).to(tl.int1)
dim_mask_v = tl.where(offs_d_v < HEAD_SIZE_V, 1, 0).to(tl.int1)
query_mask_0 = tl.where(query_pos < cur_batch_query_len, 1, 0).to(tl.int1)
query_mask_1 = tl.where(query_offset_1 < num_query_heads, 1, 0).to(tl.int1)
# Q : (BLOCK_M, HEAD_SIZE_QK_PADDED)
Q = tl.load(
query_ptr + query_offset,
mask=dim_mask_qk[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
other=0.0,
)
block_table_offset = seq_idx * block_table_stride
M = init_softmax_M(
sink_ptr, query_offset_1, query_mask_1, segm_idx, BLOCK_M, USE_SINKS, IS_3D
)
L = tl.full([BLOCK_M], 1.0, dtype=tl.float32)
# acc : (BLOCK_M, HEAD_SIZE_V_PADDED)
acc = tl.zeros([BLOCK_M, HEAD_SIZE_V_PADDED], dtype=tl.float32)
context_len = seq_len - cur_batch_query_len
if USE_ALIBI_SLOPES:
alibi_slope = tl.load(
alibi_slopes_ptr + query_offset_1, mask=query_mask_1, other=0.0
)
loop_lo, loop_hi, max_seq_prefix_len = compute_tile_loop_bounds(
context_len,
seq_len,
cur_batch_query_len,
q_block_local_idx,
segm_idx,
tiles_per_segment,
TILE_SIZE,
BLOCK_M,
BLOCK_Q,
num_queries_per_kv,
SLIDING_WINDOW,
False, # USE_MM_PREFIX
IS_3D,
)
for j in range(loop_lo, loop_hi):
seq_offset = j * TILE_SIZE + offs_t
tile_mask = seq_offset < max_seq_prefix_len
physical_block_idx = tl.load(
block_tables_ptr + block_table_offset + seq_offset // BLOCK_SIZE
).to(tl.int64)
v_offset = (
physical_block_idx[:, None] * stride_v_cache_0
+ kv_head_idx * stride_v_cache_2
+ offs_d_v[None, :] * stride_v_cache_3
+ (seq_offset % BLOCK_SIZE)[:, None] * stride_v_cache_1
)
k_offset = (
physical_block_idx[None, :] * stride_k_cache_0
+ kv_head_idx * stride_k_cache_2
+ offs_d_qk[:, None] * stride_k_cache_3
+ (seq_offset % BLOCK_SIZE)[None, :] * stride_k_cache_1
)
# K : (HEAD_SIZE_QK_PADDED, TILE_SIZE)
K_load = tl.load(
key_cache_ptr + k_offset,
mask=dim_mask_qk[:, None] & tile_mask[None, :],
other=0.0,
)
K = K_load.to(Q.dtype)
# V : (TILE_SIZE, HEAD_SIZE_V_PADDED)
V_load = tl.load(
value_cache_ptr + v_offset,
mask=dim_mask_v[None, :] & tile_mask[:, None],
other=0.0,
)
V = V_load.to(Q.dtype)
query_abs_pos = context_len + query_pos[:, None]
seq_mask = compute_kv_seq_mask(
query_abs_pos,
seq_offset,
seq_idx,
None, # mm_prefix_range_ptr
SLIDING_WINDOW,
False, # USE_MM_PREFIX
0, # MAX_MM_RANGES
)
# S : (BLOCK_M, TILE_SIZE)
S = tl.zeros(shape=(BLOCK_M, TILE_SIZE), dtype=tl.float32)
S += scale * tl.dot(Q, K)
if USE_SOFTCAP:
S = apply_softcap(S, softcap)
S = tl.where(
query_mask_1[:, None] & query_mask_0[:, None] & seq_mask, S, float("-inf")
)
if USE_ALIBI_SLOPES:
S = apply_alibi_to_score(
S, alibi_slope, seq_offset, context_len, query_pos, USE_ALIBI_SQRT
)
M, L, P, alpha = softmax_step(S, M, L)
acc = acc * alpha[:, None]
if SLIDING_WINDOW:
qpos_lo = q_block_local_idx * BLOCK_Q
V = tl.where(
(context_len + qpos_lo - seq_offset[:, None]) < SLIDING_WINDOW,
V,
0.0,
)
acc += tl.dot(P.to(V.dtype), V)
# ---- Epilogue --------------------------------------------------------
if IS_3D:
# Store per-segment partials; finalized by reduce_segments_diffkv.
segm_output_offset = (
query_offset_0[:, None].to(tl.int64)
* (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_V_PADDED)
+ query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_V_PADDED)
+ segm_idx * HEAD_SIZE_V_PADDED
+ tl.arange(0, HEAD_SIZE_V_PADDED)[None, :]
)
tl.store(
segm_output_ptr + segm_output_offset,
acc,
mask=dim_mask_v[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
)
store_segm_reduce_scalars(
segm_max_ptr,
segm_expsum_ptr,
query_offset_0,
query_offset_1,
segm_idx,
M,
L,
query_mask_0,
query_mask_1,
num_query_heads,
NUM_SEGMENTS_PER_SEQ,
)
else:
acc = acc / L[:, None]
output_offset = (
query_offset_0[:, None] * output_stride_0
+ query_offset_1[:, None] * output_stride_1
+ offs_d_v[None, :]
)
tl.store(
output_ptr + output_offset,
acc,
mask=dim_mask_v[None, :] & query_mask_0[:, None] & query_mask_1[:, None],
)
@triton.jit
def kernel_reduce_segments_diffkv(
output_ptr, # [num_tokens, num_query_heads, head_size_v]
segm_output_ptr,
# [num_tokens, num_query_heads, max_num_segments, head_size_v]
segm_max_ptr, # [num_tokens, num_query_heads, max_num_segments]
segm_expsum_ptr, # [num_tokens, num_query_heads, max_num_segments]
seq_lens_ptr, # [num_seqs]
num_seqs,
num_query_heads: tl.constexpr,
output_stride_0: tl.int64,
output_stride_1: tl.int64, # == HEAD_SIZE_V
TILE_SIZE: tl.constexpr,
HEAD_SIZE_V: tl.constexpr,
HEAD_SIZE_V_PADDED: tl.constexpr,
query_start_len_ptr, # [num_seqs+1]
BLOCK_Q: tl.constexpr,
NUM_SEGMENTS_PER_SEQ: tl.constexpr,
):
"""Combine per-segment partials into the final softmax output.
Mirrors ``reduce_segments`` from triton_unified_attention.py but
indexes V's head size (``HEAD_SIZE_V``) instead of the shared one.
"""
query_token_idx = tl.program_id(0)
query_head_idx = tl.program_id(1)
seq_idx = find_seq_idx(
query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, False
)
seq_len = tl.load(seq_lens_ptr + seq_idx)
tiles_per_segment = cdiv_fn(seq_len, NUM_SEGMENTS_PER_SEQ * TILE_SIZE)
act_num_segments = cdiv_fn(seq_len, tiles_per_segment * TILE_SIZE)
segm_mask = tl.arange(0, NUM_SEGMENTS_PER_SEQ) < tl.full(
[NUM_SEGMENTS_PER_SEQ], act_num_segments, dtype=tl.int32
)
dim_mask = tl.where(tl.arange(0, HEAD_SIZE_V_PADDED) < HEAD_SIZE_V, 1, 0).to(
tl.int1
)
segm_offset = (
query_token_idx.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ)
+ query_head_idx * NUM_SEGMENTS_PER_SEQ
+ tl.arange(0, NUM_SEGMENTS_PER_SEQ)
)
segm_max = tl.load(segm_max_ptr + segm_offset, mask=segm_mask, other=float("-inf"))
overall_max = tl.max(segm_max)
segm_expsum = tl.load(segm_expsum_ptr + segm_offset, mask=segm_mask, other=0.0)
segm_expsum = segm_expsum * tl.exp(segm_max - overall_max)
overall_expsum = tl.sum(segm_expsum)
segm_output_offset = (
query_token_idx.to(tl.int64)
* (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_V_PADDED)
+ query_head_idx * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_V_PADDED)
+ tl.arange(0, NUM_SEGMENTS_PER_SEQ)[:, None] * HEAD_SIZE_V_PADDED
+ tl.arange(0, HEAD_SIZE_V_PADDED)[None, :]
)
segm_output = tl.load(
segm_output_ptr + segm_output_offset,
mask=segm_mask[:, None] & dim_mask[None, :],
other=0.0,
)
segm_output *= tl.exp(segm_max - overall_max)[:, None]
acc_sum = tl.sum(segm_output, axis=0)
acc = tl.where(overall_expsum == 0.0, 0.0, acc_sum / overall_expsum)
output_offset = (
query_token_idx * output_stride_0
+ query_head_idx * output_stride_1
+ tl.arange(0, HEAD_SIZE_V_PADDED)
)
tl.store(output_ptr + output_offset, acc, mask=dim_mask)
def unified_attention_diffkv(
q, # [num_tokens, num_query_heads, head_size_qk]
k, # view: [num_blocks, block_size, num_kv_heads, head_size_qk]
v, # view: [num_blocks, block_size, num_kv_heads, head_size_v]
out, # [num_tokens, num_query_heads, head_size_v]
cu_seqlens_q,
seqused_k,
softmax_scale,
causal,
window_size,
block_table,
softcap,
max_seqlen_q: int = 1,
alibi_slopes=None,
sinks=None,
use_alibi_sqrt=False,
# 3D / split-KV softmax buffers. When all four are provided and the
# batch is decode-only with few sequences, the 3D path is taken.
seq_threshold_3D: int | None = None,
num_par_softmax_segments: int | None = None,
softmax_segm_output: torch.Tensor | None = None,
softmax_segm_max: torch.Tensor | None = None,
softmax_segm_expsum: torch.Tensor | None = None,
):
assert causal, "Only causal attention is supported"
if sinks is not None:
assert sinks.shape[0] == q.shape[1], "Sinks must be num_query_heads size"
use_alibi_slopes = alibi_slopes is not None
block_size = v.shape[1]
num_seqs = len(seqused_k)
num_query_heads = q.shape[1]
num_kv_heads = k.shape[2]
num_queries_per_kv = num_query_heads // num_kv_heads
head_size_qk = q.shape[2]
head_size_v = v.shape[3]
BLOCK_M = (
16 if num_queries_per_kv <= 16 else triton.next_power_of_2(num_queries_per_kv)
)
BLOCK_Q = BLOCK_M // num_queries_per_kv
total_num_q_blocks = q.shape[0] // BLOCK_Q + num_seqs
sliding_window_val = 1 + window_size[0] if window_size[0] >= 0 else 0
# Decide between 2D and 3D launch. Mirrors the standard launcher:
# 3D requires preallocated softmax buffers, decode-only batches, and
# a small number of sequences (otherwise 2D already saturates the SM).
use_3d = not (
seq_threshold_3D is None
or num_par_softmax_segments is None
or softmax_segm_output is None
or softmax_segm_max is None
or softmax_segm_expsum is None
or max_seqlen_q > 1
or num_seqs > seq_threshold_3D
or is_batch_invariant
)
# Tile size: 32 for prefill-class kernels. Decode (small Q) prefers
# smaller tiles to expose more parallelism along the KV dim.
tile_size = 32 if not use_3d else (16 if q.element_size() >= 2 else 32)
grid: tuple[Any, ...]
if use_3d:
grid = (total_num_q_blocks, num_kv_heads, num_par_softmax_segments)
segm_output_ptr = softmax_segm_output
segm_max_ptr = softmax_segm_max
segm_expsum_ptr = softmax_segm_expsum
num_segments = num_par_softmax_segments
else:
grid = (total_num_q_blocks, num_kv_heads)
# 2D never touches the segm tensors but Triton wants a non-null
# pointer; reuse ``out``.
segm_output_ptr = out
segm_max_ptr = out
segm_expsum_ptr = out
num_segments = 1
kernel_unified_attention_diffkv[grid](
output_ptr=out,
segm_output_ptr=segm_output_ptr,
segm_max_ptr=segm_max_ptr,
segm_expsum_ptr=segm_expsum_ptr,
query_ptr=q,
key_cache_ptr=k,
value_cache_ptr=v,
sink_ptr=sinks,
block_tables_ptr=block_table,
seq_lens_ptr=seqused_k,
alibi_slopes_ptr=alibi_slopes,
scale=softmax_scale,
softcap=softcap,
num_query_heads=num_query_heads,
num_queries_per_kv=num_queries_per_kv,
block_table_stride=block_table.stride(0),
query_stride_0=q.stride(0),
query_stride_1=q.stride(1),
output_stride_0=out.stride(0),
output_stride_1=out.stride(1),
BLOCK_SIZE=block_size,
TILE_SIZE=tile_size,
HEAD_SIZE_QK=head_size_qk,
HEAD_SIZE_QK_PADDED=triton.next_power_of_2(head_size_qk),
HEAD_SIZE_V=head_size_v,
HEAD_SIZE_V_PADDED=triton.next_power_of_2(head_size_v),
USE_ALIBI_SLOPES=use_alibi_slopes,
USE_ALIBI_SQRT=use_alibi_sqrt,
USE_SOFTCAP=(softcap > 0),
USE_SINKS=(sinks is not None),
SLIDING_WINDOW=sliding_window_val,
stride_k_cache_0=k.stride(0),
stride_k_cache_1=k.stride(1),
stride_k_cache_2=k.stride(2),
stride_k_cache_3=k.stride(3),
stride_v_cache_0=v.stride(0),
stride_v_cache_1=v.stride(1),
stride_v_cache_2=v.stride(2),
stride_v_cache_3=v.stride(3),
query_start_len_ptr=cu_seqlens_q,
BLOCK_Q=BLOCK_Q,
num_seqs=num_seqs,
BLOCK_M=BLOCK_M,
NUM_SEGMENTS_PER_SEQ=num_segments,
IS_3D=use_3d,
)
if use_3d:
kernel_reduce_segments_diffkv[(q.shape[0], num_query_heads)](
output_ptr=out,
segm_output_ptr=softmax_segm_output,
segm_max_ptr=softmax_segm_max,
segm_expsum_ptr=softmax_segm_expsum,
seq_lens_ptr=seqused_k,
num_seqs=num_seqs,
num_query_heads=num_query_heads,
output_stride_0=out.stride(0),
output_stride_1=out.stride(1),
TILE_SIZE=tile_size,
HEAD_SIZE_V=head_size_v,
HEAD_SIZE_V_PADDED=triton.next_power_of_2(head_size_v),
query_start_len_ptr=cu_seqlens_q,
BLOCK_Q=BLOCK_Q,
NUM_SEGMENTS_PER_SEQ=num_par_softmax_segments,
)