From 87d9996abe7ae0173fa8c1b6ccc2411041bcac81 Mon Sep 17 00:00:00 2001 From: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:37:51 +0200 Subject: [PATCH] [ROCm][Perf][GLM-5.3-Flash] BF16 splitk kernel for sparse MLA decode (#58584) Signed-off-by: simondanielsson Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> --- .../benchmark_sparse_mla_decode_bf16.py | 130 +++++++ .../attention/test_rocm_aiter_mla_sink.py | 6 + .../attention/test_rocm_triton_attn_dsv4.py | 96 +++++ .../backends/mla/rocm_aiter_mla_sparse.py | 53 ++- .../v1/attention/ops/rocm_aiter_mla_sparse.py | 355 ++++++++++++++++++ 5 files changed, 627 insertions(+), 13 deletions(-) create mode 100644 benchmarks/kernels/benchmark_sparse_mla_decode_bf16.py diff --git a/benchmarks/kernels/benchmark_sparse_mla_decode_bf16.py b/benchmarks/kernels/benchmark_sparse_mla_decode_bf16.py new file mode 100644 index 0000000000..c32c8519aa --- /dev/null +++ b/benchmarks/kernels/benchmark_sparse_mla_decode_bf16.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Benchmark the ROCm bf16 sparse-MLA decode split-K kernel in isolation. + +Example usage: +python3 benchmarks/kernels/benchmark_sparse_mla_decode_bf16.py \ + --rows 1 4 16 64 --lens 2048 --splits auto 4 16 32 +""" + +import argparse +import itertools +import json + +import torch + +from vllm.triton_utils import triton +from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _rocm_sparse_attn_decode_ragged_bf16_triton, + _rocm_sparse_attn_prefill_ragged_triton, + rocm_sparse_decode_bf16_num_splits, +) + +BLOCK_H = 16 + + +def make_inputs(rows: int, live: int, args): + torch.manual_seed(args.seed) + kv = ( + torch.randn(args.capacity, args.head_dim, device="cuda", dtype=torch.bfloat16) + * 0.125 + ) + q = ( + torch.randn( + rows, args.heads, args.head_dim, device="cuda", dtype=torch.bfloat16 + ) + * 0.125 + ) + slots = torch.stack( + [torch.randperm(args.capacity, device="cuda")[:live] for _ in range(rows)] + ) + indices = slots.reshape(-1).to(torch.int32) + indptr = torch.arange(rows + 1, device="cuda", dtype=torch.int32) * live + sink = ( + torch.randn(args.heads, device="cuda", dtype=torch.float32) + if args.sink + else None + ) + return q, kv, indices, indptr, sink + + +def resolve_splits(spec: str, rows: int, live: int, args) -> int: + if spec != "auto": + return int(spec) + return rocm_sparse_decode_bf16_num_splits(rows, args.heads, live) + + +def benchmark(backend: str, rows: int, live: int, spec: str, args) -> dict: + q, kv, indices, indptr, sink = make_inputs(rows, live, args) + nope, rope = args.head_dim, 0 + splits = resolve_splits(spec, rows, live, args) + + def ragged(): + return _rocm_sparse_attn_prefill_ragged_triton( + q, kv, indices, indptr, args.scale, sink, nope, rope + ) + + def splitk(): + return _rocm_sparse_attn_decode_ragged_bf16_triton( + q, kv, indices, indptr, args.scale, sink, nope, rope, splits + ) + + run = ragged if backend == "ragged" else splitk + torch.testing.assert_close(run(), ragged(), atol=2e-2, rtol=2e-2) + ms = triton.testing.do_bench(run, warmup=args.warmup, rep=args.rep) + return dict( + backend=backend, + rows=rows, + live=live, + heads=args.heads, + head_dim=args.head_dim, + splits=splits if backend == "splitk" else 1, + workgroups=rows * triton.cdiv(args.heads, BLOCK_H) * splits, + us=ms * 1000, + kv_gbps=rows * live * args.head_dim * 2 / (ms * 1e6), + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, nargs="+", default=[1, 2, 4, 8, 16, 64]) + parser.add_argument( + "--lens", + type=int, + nargs="+", + default=[2048], + help="Selected KV slots per row; GLM-5.3-Flash uses index_topk=2048", + ) + parser.add_argument( + "--splits", + nargs="+", + default=["auto"], + help="Split counts to sweep, or auto for the runtime heuristic", + ) + parser.add_argument( + "--backends", + nargs="+", + choices=["ragged", "splitk"], + default=["ragged", "splitk"], + ) + parser.add_argument( + "--heads", type=int, default=8, help="Heads per rank; 64 at TP8 gives 8" + ) + parser.add_argument("--head-dim", type=int, default=512) + parser.add_argument("--capacity", type=int, default=65536) + parser.add_argument("--scale", type=float, default=0.044194173824159216) + parser.add_argument("--sink", action="store_true") + parser.add_argument("--warmup", type=int, default=25) + parser.add_argument("--rep", type=int, default=200) + parser.add_argument("--seed", type=int, default=123) + args = parser.parse_args() + + print(json.dumps(dict(gpu=torch.cuda.get_device_name(0), args=vars(args)))) + for backend, rows, live in itertools.product(args.backends, args.rows, args.lens): + specs = ["1"] if backend == "ragged" else args.splits + for spec in specs: + print(json.dumps(benchmark(backend, rows, live, spec, args)), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tests/kernels/attention/test_rocm_aiter_mla_sink.py b/tests/kernels/attention/test_rocm_aiter_mla_sink.py index 40b888c362..0b007dbc87 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_sink.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_sink.py @@ -156,6 +156,8 @@ def test_sparse_mla_sink_matches_ragged_reference( num_decodes=batch_size, num_decode_tokens=batch_size, max_query_len=1, + max_seq_len=max(seq_lens), + topk_tokens=max(seq_lens), ) sinks = torch.linspace(-2.0, 6.0, real_heads, device=device) @@ -367,12 +369,15 @@ def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache for page, offset in rows ] lengths = torch.tensor([0, *(len(rows) for rows in selected_rows)]) + longest_row = max(len(rows) for rows in selected_rows) metadata = SimpleNamespace( block_size=block_size, num_prefills=0, num_decodes=len(selected_rows), num_decode_tokens=len(selected_rows), max_query_len=1, + max_seq_len=longest_row, + topk_tokens=longest_row, qo_indptr=torch.arange( len(selected_rows) + 1, dtype=torch.int32, device="cuda" ), @@ -479,6 +484,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype): num_decodes=num_tokens, num_decode_tokens=num_tokens, max_query_len=1, + max_seq_len=kv.shape[0], block_size=block_size, topk_tokens=impl.topk_indices_buffer.shape[1], req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device="cuda"), diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index 3b200be514..ca2cf2eb70 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -596,6 +596,87 @@ def test_sparse_attn_prefill_ragged_kernel() -> None: torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) +NOPE_ONLY_HEAD_DIM = 512 + + +def _nope_only_ragged_inputs( + kv_lens: list[int], num_heads: int, device: torch.device +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Build a GLM-like 512+0 NoPE decode batch with ragged KV segments.""" + num_kv = 4096 + kv = ( + torch.randn(num_kv, NOPE_ONLY_HEAD_DIM, dtype=torch.bfloat16, device=device) + * 0.125 + ) + q = ( + torch.randn( + len(kv_lens), + num_heads, + NOPE_ONLY_HEAD_DIM, + dtype=torch.bfloat16, + device=device, + ) + * 0.125 + ) + rows = [torch.randperm(num_kv)[:length].tolist() for length in kv_lens] + indices, indptr = _ragged_from_rows(rows, device) + return q, kv, indices, indptr + + +# Lengths below the split count leave splits empty; non-multiples force a tail. +_NOPE_ONLY_KV_LENS = [0, 1, 3, 37, 129, 2048] + + +@requires_split_decode_arch +@pytest.mark.parametrize("num_splits", [2, 32]) +@pytest.mark.parametrize("with_sink", [True, False]) +@torch.inference_mode() +def test_sparse_attn_decode_bf16_split_k_matches_ragged( + num_splits: int, with_sink: bool +) -> None: + """Split-K decode must reproduce the single-workgroup ragged kernel.""" + from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _rocm_sparse_attn_decode_ragged_bf16_triton, + _rocm_sparse_attn_prefill_ragged_triton, + ) + + device = torch.device("cuda") + set_random_seed(0) + num_heads = 16 + q, kv, indices, indptr = _nope_only_ragged_inputs( + _NOPE_ONLY_KV_LENS, num_heads, device + ) + attn_sink = ( + torch.randn(num_heads, dtype=torch.float32, device=device) + if with_sink + else None + ) + scale = NOPE_ONLY_HEAD_DIM**-0.5 + + expected = _rocm_sparse_attn_prefill_ragged_triton( + q=q, + kv=kv, + indices=indices, + indptr=indptr, + scale=scale, + attn_sink=attn_sink, + nope_head_dim=NOPE_ONLY_HEAD_DIM, + rope_head_dim=0, + ) + actual = _rocm_sparse_attn_decode_ragged_bf16_triton( + q=q, + kv=kv, + indices=indices, + indptr=indptr, + scale=scale, + attn_sink=attn_sink, + nope_head_dim=NOPE_ONLY_HEAD_DIM, + rope_head_dim=0, + num_splits=num_splits, + ) + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) + + def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict: device = torch.device("cuda") set_random_seed(7) @@ -1105,6 +1186,21 @@ def test_decode_num_splits_gfx950(monkeypatch) -> None: assert mod._decode_gfx950_num_splits(1, 1, 128, 8192) == 32 assert mod._decode_gfx950_num_splits(17, 1, 128, 32) == 4 assert mod._decode_gfx950_num_splits(512, 1, 128, 7812) == 1 + assert mod._decode_gfx950_num_splits(4, 1, 2048.0, 0.0, 32) == 32 + + +def test_sparse_decode_bf16_num_splits_floor_and_tile_clamp(monkeypatch) -> None: + """Short rows skip splitting, and splits never exceed the tiles they walk.""" + from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod + + monkeypatch.setattr(mod, "_decode_num_splits", lambda *args: 32) + monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 32) + + floor = mod._SPARSE_DECODE_BF16_MIN_SPLIT_LEN + block_k = mod._SPARSE_DECODE_BF16_BLOCK_K + assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor - 1) == 1 + assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor) == floor // block_k + assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, 2048) == 32 @requires_split_decode_arch diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index e473f85948..e3d982e882 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -37,7 +37,9 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( ) from vllm.v1.attention.backends.utils import split_decodes_and_prefills from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + rocm_sparse_attn_decode_bf16, rocm_sparse_attn_prefill, + rocm_sparse_decode_bf16_num_splits, ) from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout from vllm.v1.worker.workspace import current_workspace_manager @@ -904,20 +906,45 @@ class ROCMAiterMLASparseImpl( self.sinks.reshape(1, self.num_heads, 1), q.shape[1], ).reshape(-1) - rocm_sparse_attn_prefill( - q=q, - kv=kv_c_and_k_pe_cache.view(-1, 1, q.shape[-1]), - indices=None, - topk_length=None, - scale=self.scale, - head_dim=q.shape[-1], - nope_head_dim=self.kv_lora_rank, - rope_head_dim=q.shape[-1] - self.kv_lora_rank, - attn_sink=triton_sinks, - output=output, - ragged_indices=attn_metadata.paged_kv_indices, - ragged_indptr=attn_metadata.paged_kv_indptr, + kv = kv_c_and_k_pe_cache.view(-1, 1, q.shape[-1]) + decode_num_splits = ( + rocm_sparse_decode_bf16_num_splits( + num_tokens, + q.shape[1], + min(attn_metadata.max_seq_len, attn_metadata.topk_tokens), + ) + if attn_metadata.num_decode_tokens == num_tokens + else 1 ) + if decode_num_splits > 1: + rocm_sparse_attn_decode_bf16( + q=q, + kv=kv, + scale=self.scale, + head_dim=q.shape[-1], + nope_head_dim=self.kv_lora_rank, + rope_head_dim=q.shape[-1] - self.kv_lora_rank, + attn_sink=triton_sinks, + output=output, + ragged_indices=attn_metadata.paged_kv_indices, + ragged_indptr=attn_metadata.paged_kv_indptr, + num_splits=decode_num_splits, + ) + else: + rocm_sparse_attn_prefill( + q=q, + kv=kv, + indices=None, + topk_length=None, + scale=self.scale, + head_dim=q.shape[-1], + nope_head_dim=self.kv_lora_rank, + rope_head_dim=q.shape[-1] - self.kv_lora_rank, + attn_sink=triton_sinks, + output=output, + ragged_indices=attn_metadata.paged_kv_indices, + ragged_indptr=attn_metadata.paged_kv_indptr, + ) output = AiterMLAHelper.get_mla_unpadded_o(self.num_heads, output) return output, None diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index a3412aa835..90a73c101c 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -2208,6 +2208,119 @@ def _sparse_attn_prefill_ragged_kernel( ) +@triton.jit +def _sparse_attn_decode_ragged_bf16_partial_kernel( + q_ptr, + kv_ptr, + kv_indices_ptr, + kv_indptr_ptr, + part_m_ptr, + part_l_ptr, + part_acc_ptr, + q_stride_t, + q_stride_h, + q_stride_d, + kv_stride_n, + kv_stride_d, + pm_stride0, + pm_stride_s, + pa_stride0, + pa_stride_s, + pa_stride_h, + num_heads, + head_dim, + num_kv, + scale, + BLOCK_H: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_K: tl.constexpr, + NUM_SPLITS: tl.constexpr, +): + query_idx = tl.program_id(0) + split_id = tl.program_id(1) + pid_h = tl.program_id(2) + + head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) + dim_offsets = tl.arange(0, BLOCK_D) + head_mask = head_offsets < num_heads + dim_mask = dim_offsets < head_dim + + q = tl.load( + q_ptr + + query_idx * q_stride_t + + head_offsets[:, None] * q_stride_h + + dim_offsets[None, :] * q_stride_d, + mask=head_mask[:, None] & dim_mask[None, :], + other=0.0, + ) + + neg_large = -3.4028234663852886e38 + m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32) + l_i = tl.zeros((BLOCK_H,), dtype=tl.float32) + acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32) + + kv_start = tl.load(kv_indptr_ptr + query_idx) + kv_end = tl.load(kv_indptr_ptr + query_idx + 1) + kv_len = kv_end - kv_start + chunk = (kv_len + NUM_SPLITS - 1) // NUM_SPLITS + kv_lo = split_id * chunk + kv_hi = tl.minimum(kv_lo + chunk, kv_len) + + k_offsets = tl.arange(0, BLOCK_K) + slot = tl.load( + kv_indices_ptr + kv_start + kv_lo + k_offsets, + mask=kv_lo + k_offsets < kv_hi, + other=-1, + ) + for k_start in tl.range(kv_lo, kv_hi, BLOCK_K): + k_pos = k_start + k_offsets + in_range = k_pos < kv_hi + valid = in_range & (slot >= 0) & (slot < num_kv) + safe_slot = tl.where(valid, slot, 0) + + kv = tl.load( + kv_ptr + + _sparse_kv_row_offset(safe_slot[:, None], kv_stride_n) + + dim_offsets[None, :] * kv_stride_d, + mask=valid[:, None] & dim_mask[None, :], + other=0.0, + ) + + next_k_pos = k_start + BLOCK_K + k_offsets + slot = tl.load( + kv_indices_ptr + kv_start + next_k_pos, mask=next_k_pos < kv_hi, other=-1 + ) + + scores = tl.dot(q, tl.trans(kv)) * scale + scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large) + + m_block = tl.max(scores, axis=1) + m_new = tl.maximum(m_i, m_block) + alpha = tl.exp(m_i - m_new) + p = tl.exp(scores - m_new[:, None]) + p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0) + l_new = l_i * alpha + tl.sum(p, axis=1) + + acc = acc * alpha[:, None] + tl.dot(p.to(kv.dtype), kv) + m_i = m_new + l_i = l_new + + # Every launched slot is written; empty splits store the (neg_large, 0.0) + # sentinel the reduce combines to nothing. + pm_base = query_idx * pm_stride0 + split_id * pm_stride_s + head_offsets + tl.store(part_m_ptr + pm_base, m_i, mask=head_mask) + tl.store(part_l_ptr + pm_base, l_i, mask=head_mask) + tl.store( + part_acc_ptr + + query_idx * pa_stride0 + + split_id * pa_stride_s + + head_offsets[:, None] * pa_stride_h + + dim_offsets[None, :], + acc, + mask=head_mask[:, None] & dim_mask[None, :], + ) + + @triton.jit def _decode_e8m0_scales_triton(encoded_scales): scale_bits = encoded_scales.to(tl.int32) << 23 @@ -3664,6 +3777,182 @@ def _decode_gfx950_num_splits( return num_splits +# Empirically single-pass bf16 decode is faster than split-K below this number of +# selected slots per query, so we do not split. +_SPARSE_DECODE_BF16_MIN_SPLIT_LEN = 512 +_SPARSE_DECODE_BF16_BLOCK_H = 16 +_SPARSE_DECODE_BF16_BLOCK_K = 32 + + +def rocm_sparse_decode_bf16_num_splits( + num_queries: int, num_heads: int, sparse_len: int +) -> int: + """Number or kv splits in splitK for the sparse bf16 decode, or 1 for single-pass. + + Args: + num_queries: Decode rows in the batch. + num_heads: Query heads per row. + sparse_len: Longest selected KV run any decode row can walk. + + Returns: + The split count, or 1 when the caller should use the single-pass kernel. + + """ + if sparse_len < _SPARSE_DECODE_BF16_MIN_SPLIT_LEN: + return 1 + block_k = _SPARSE_DECODE_BF16_BLOCK_K + heads_blocks = triton.cdiv(num_heads, _SPARSE_DECODE_BF16_BLOCK_H) + select = _decode_gfx950_num_splits if _ON_GFX950 else _decode_num_splits + num_splits = select(num_queries, heads_blocks, sparse_len, 0.0, block_k) + # Number of splits cannot exceed the available k tiles + return max(1, min(num_splits, math.ceil(sparse_len / block_k))) + + +def _rocm_sparse_attn_decode_ragged_bf16_triton( + q: torch.Tensor, + kv: torch.Tensor, + indices: torch.Tensor, + indptr: torch.Tensor, + scale: float, + attn_sink: torch.Tensor | None, + nope_head_dim: int, + rope_head_dim: int, + num_splits: int, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Split-K decode over an bf16 ragged KV cache. + + Partitions each query's selected tokens across different workgroups and combines + the partials through reduction. + + Args: + q: Queries laid out as ``[sq, h, d]``. + kv: Unquantized KV rows laid out as ``[skv, d]``. + indices: Flattened per-query KV slots. + indptr: Segment offsets into ``indices``, ``[sq + 1]``. + scale: Softmax scale. + attn_sink: Optional per-head sink logits. + nope_head_dim: NoPE width of ``d``. + rope_head_dim: RoPE width of ``d``. + num_splits: Number of KV splits per query. + out: Optional destination with ``d`` trailing elements. + + Returns: + The attention output, ``out`` when provided. + + """ + assert q.ndim == 3, f"expected q=[sq,h,d], got {q.shape}" + assert kv.ndim == 2, f"expected kv=[skv,d], got {kv.shape}" + assert indices.ndim == 1, f"expected indices=[nnz], got {indices.shape}" + assert indptr.ndim == 1, f"expected indptr=[sq+1], got {indptr.shape}" + assert not q.is_cpu and not kv.is_cpu and not indices.is_cpu and not indptr.is_cpu + + indices = _as_int32_contiguous_1d(indices) + indptr = _as_int32_contiguous_1d(indptr) + has_attn_sink = attn_sink is not None + if attn_sink is None: + attn_sink = torch.empty(1, device=q.device, dtype=torch.float32) + else: + attn_sink = attn_sink.contiguous() + + num_queries, num_heads, head_dim = q.shape + assert indptr.numel() == num_queries + 1, ( + f"expected indptr shape [{num_queries + 1}], got {indptr.shape}" + ) + _validate_sparse_dims( + head_dim, + nope_head_dim, + rope_head_dim, + "_rocm_sparse_attn_decode_ragged_bf16_triton", + ) + if out is None: + out = torch.empty_like(q) + assert out.shape[-1] == head_dim, ( + f"expected out trailing dim {head_dim}, got {out.shape[-1]}" + ) + + block_h = _SPARSE_DECODE_BF16_BLOCK_H + block_d = triton.next_power_of_2(head_dim) + block_k = _SPARSE_DECODE_BF16_BLOCK_K + heads_blocks = triton.cdiv(num_heads, block_h) + comb_dim = nope_head_dim + rope_head_dim + + part_m = torch.empty( + (num_queries, num_splits, num_heads), dtype=torch.float32, device=q.device + ) + part_l = torch.empty_like(part_m) + part_acc = torch.empty( + (num_queries, num_splits, num_heads, comb_dim), + dtype=torch.float32, + device=q.device, + ) + + _sparse_attn_decode_ragged_bf16_partial_kernel[ + (num_queries, num_splits, heads_blocks) + ]( + q, + kv, + indices, + indptr, + part_m, + part_l, + part_acc, + q.stride(0), + q.stride(1), + q.stride(2), + kv.stride(0), + kv.stride(1), + part_m.stride(0), + part_m.stride(1), + part_acc.stride(0), + part_acc.stride(1), + part_acc.stride(2), + num_heads, + head_dim, + kv.shape[0], + float(scale), + BLOCK_H=block_h, + BLOCK_D=block_d, + BLOCK_K=block_k, + NUM_SPLITS=num_splits, + num_warps=4, + ) + + _sparse_attn_decode_reduce_kernel[(num_queries, num_heads)]( + part_m, + part_l, + part_acc, + attn_sink, + out, + None, + None, + None, + out.stride(0), + out.stride(1), + 0, + head_dim // 32, + part_m.stride(0), + part_m.stride(1), + part_acc.stride(0), + part_acc.stride(1), + part_acc.stride(2), + 0, + num_heads, + HAS_ATTN_SINK=has_attn_sink, + ADAPTIVE_SPLITS=False, + COMB_DIM=comb_dim, + BLOCK_H=1, + NUM_SPLITS=num_splits, + SPLITS_PAD=triton.next_power_of_2(num_splits), + FUSE_INV_ROPE=False, + NOPE=nope_head_dim, + HALF=rope_head_dim // 2, + QUANT_OUT=False, + num_warps=4, + ) + return out + + def _rocm_sparse_attn_decode_ragged_triton( q: torch.Tensor, main_cache: torch.Tensor, @@ -4143,6 +4432,72 @@ def rocm_sparse_attn_prefill( ) +def rocm_sparse_attn_decode_bf16( + q: torch.Tensor, + kv: torch.Tensor, + scale: float, + head_dim: int, + nope_head_dim: int, + rope_head_dim: int, + attn_sink: torch.Tensor | None, + output: torch.Tensor, + ragged_indices: torch.Tensor, + ragged_indptr: torch.Tensor, + num_splits: int, +) -> None: + """Run split-K sparse attention over decode rows using an unquantized KV cache. + + Args: + q: Decode queries laid out as ``[sq, h, d]``. + kv: KV cache laid out as ``[skv, 1, d]``. + scale: Softmax scale. + head_dim: Post-absorption head width. + nope_head_dim: NoPE width of ``head_dim``. + rope_head_dim: RoPE width of ``head_dim``. + attn_sink: Optional per-head sink logits. + output: Destination, written in place. + ragged_indices: Flattened per-query KV slots. + ragged_indptr: Segment offsets into ``ragged_indices``, ``[sq + 1]``. + num_splits: KV splits per query, from + :func:`rocm_sparse_decode_bf16_num_splits`. + + """ + assert kv.ndim == 3 and kv.shape[1] == 1, ( + f"ROCm Triton sparse decode expects kv=[skv,1,d], got {kv.shape}" + ) + _validate_sparse_dims( + head_dim, + nope_head_dim, + rope_head_dim, + "rocm_sparse_attn_decode_bf16", + ) + num_queries, num_heads = q.shape[0], q.shape[1] + direct = output.shape[-1] == head_dim + out = ( + output + if direct + else torch.empty( + (num_queries, num_heads, head_dim), + dtype=output.dtype, + device=output.device, + ) + ) + _rocm_sparse_attn_decode_ragged_bf16_triton( + q=q, + kv=kv.squeeze(1), + indices=ragged_indices, + indptr=ragged_indptr, + scale=scale, + attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]], + nope_head_dim=nope_head_dim, + rope_head_dim=rope_head_dim, + num_splits=num_splits, + out=out, + ) + if not direct: + output.copy_(out[..., : output.shape[-1]]) + + def rocm_sparse_attn_decode( q: torch.Tensor, kv_cache: torch.Tensor | None,