[ROCm][Perf][GLM-5.3-Flash] BF16 splitk kernel for sparse MLA decode (#58584)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Simon Danielsson
2026-10-07 21:37:51 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent 3722645539
commit 87d9996abe
5 changed files with 627 additions and 13 deletions
@@ -0,0 +1,130 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark the ROCm bf16 sparse-MLA decode split-K kernel in isolation.
Example usage:
python3 benchmarks/kernels/benchmark_sparse_mla_decode_bf16.py \
--rows 1 4 16 64 --lens 2048 --splits auto 4 16 32
"""
import argparse
import itertools
import json
import torch
from vllm.triton_utils import triton
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_decode_ragged_bf16_triton,
_rocm_sparse_attn_prefill_ragged_triton,
rocm_sparse_decode_bf16_num_splits,
)
BLOCK_H = 16
def make_inputs(rows: int, live: int, args):
torch.manual_seed(args.seed)
kv = (
torch.randn(args.capacity, args.head_dim, device="cuda", dtype=torch.bfloat16)
* 0.125
)
q = (
torch.randn(
rows, args.heads, args.head_dim, device="cuda", dtype=torch.bfloat16
)
* 0.125
)
slots = torch.stack(
[torch.randperm(args.capacity, device="cuda")[:live] for _ in range(rows)]
)
indices = slots.reshape(-1).to(torch.int32)
indptr = torch.arange(rows + 1, device="cuda", dtype=torch.int32) * live
sink = (
torch.randn(args.heads, device="cuda", dtype=torch.float32)
if args.sink
else None
)
return q, kv, indices, indptr, sink
def resolve_splits(spec: str, rows: int, live: int, args) -> int:
if spec != "auto":
return int(spec)
return rocm_sparse_decode_bf16_num_splits(rows, args.heads, live)
def benchmark(backend: str, rows: int, live: int, spec: str, args) -> dict:
q, kv, indices, indptr, sink = make_inputs(rows, live, args)
nope, rope = args.head_dim, 0
splits = resolve_splits(spec, rows, live, args)
def ragged():
return _rocm_sparse_attn_prefill_ragged_triton(
q, kv, indices, indptr, args.scale, sink, nope, rope
)
def splitk():
return _rocm_sparse_attn_decode_ragged_bf16_triton(
q, kv, indices, indptr, args.scale, sink, nope, rope, splits
)
run = ragged if backend == "ragged" else splitk
torch.testing.assert_close(run(), ragged(), atol=2e-2, rtol=2e-2)
ms = triton.testing.do_bench(run, warmup=args.warmup, rep=args.rep)
return dict(
backend=backend,
rows=rows,
live=live,
heads=args.heads,
head_dim=args.head_dim,
splits=splits if backend == "splitk" else 1,
workgroups=rows * triton.cdiv(args.heads, BLOCK_H) * splits,
us=ms * 1000,
kv_gbps=rows * live * args.head_dim * 2 / (ms * 1e6),
)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--rows", type=int, nargs="+", default=[1, 2, 4, 8, 16, 64])
parser.add_argument(
"--lens",
type=int,
nargs="+",
default=[2048],
help="Selected KV slots per row; GLM-5.3-Flash uses index_topk=2048",
)
parser.add_argument(
"--splits",
nargs="+",
default=["auto"],
help="Split counts to sweep, or auto for the runtime heuristic",
)
parser.add_argument(
"--backends",
nargs="+",
choices=["ragged", "splitk"],
default=["ragged", "splitk"],
)
parser.add_argument(
"--heads", type=int, default=8, help="Heads per rank; 64 at TP8 gives 8"
)
parser.add_argument("--head-dim", type=int, default=512)
parser.add_argument("--capacity", type=int, default=65536)
parser.add_argument("--scale", type=float, default=0.044194173824159216)
parser.add_argument("--sink", action="store_true")
parser.add_argument("--warmup", type=int, default=25)
parser.add_argument("--rep", type=int, default=200)
parser.add_argument("--seed", type=int, default=123)
args = parser.parse_args()
print(json.dumps(dict(gpu=torch.cuda.get_device_name(0), args=vars(args))))
for backend, rows, live in itertools.product(args.backends, args.rows, args.lens):
specs = ["1"] if backend == "ragged" else args.splits
for spec in specs:
print(json.dumps(benchmark(backend, rows, live, spec, args)), flush=True)
if __name__ == "__main__":
main()
@@ -156,6 +156,8 @@ def test_sparse_mla_sink_matches_ragged_reference(
num_decodes=batch_size,
num_decode_tokens=batch_size,
max_query_len=1,
max_seq_len=max(seq_lens),
topk_tokens=max(seq_lens),
)
sinks = torch.linspace(-2.0, 6.0, real_heads, device=device)
@@ -367,12 +369,15 @@ def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache
for page, offset in rows
]
lengths = torch.tensor([0, *(len(rows) for rows in selected_rows)])
longest_row = max(len(rows) for rows in selected_rows)
metadata = SimpleNamespace(
block_size=block_size,
num_prefills=0,
num_decodes=len(selected_rows),
num_decode_tokens=len(selected_rows),
max_query_len=1,
max_seq_len=longest_row,
topk_tokens=longest_row,
qo_indptr=torch.arange(
len(selected_rows) + 1, dtype=torch.int32, device="cuda"
),
@@ -479,6 +484,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
num_decodes=num_tokens,
num_decode_tokens=num_tokens,
max_query_len=1,
max_seq_len=kv.shape[0],
block_size=block_size,
topk_tokens=impl.topk_indices_buffer.shape[1],
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device="cuda"),
@@ -596,6 +596,87 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
NOPE_ONLY_HEAD_DIM = 512
def _nope_only_ragged_inputs(
kv_lens: list[int], num_heads: int, device: torch.device
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build a GLM-like 512+0 NoPE decode batch with ragged KV segments."""
num_kv = 4096
kv = (
torch.randn(num_kv, NOPE_ONLY_HEAD_DIM, dtype=torch.bfloat16, device=device)
* 0.125
)
q = (
torch.randn(
len(kv_lens),
num_heads,
NOPE_ONLY_HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125
)
rows = [torch.randperm(num_kv)[:length].tolist() for length in kv_lens]
indices, indptr = _ragged_from_rows(rows, device)
return q, kv, indices, indptr
# Lengths below the split count leave splits empty; non-multiples force a tail.
_NOPE_ONLY_KV_LENS = [0, 1, 3, 37, 129, 2048]
@requires_split_decode_arch
@pytest.mark.parametrize("num_splits", [2, 32])
@pytest.mark.parametrize("with_sink", [True, False])
@torch.inference_mode()
def test_sparse_attn_decode_bf16_split_k_matches_ragged(
num_splits: int, with_sink: bool
) -> None:
"""Split-K decode must reproduce the single-workgroup ragged kernel."""
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_decode_ragged_bf16_triton,
_rocm_sparse_attn_prefill_ragged_triton,
)
device = torch.device("cuda")
set_random_seed(0)
num_heads = 16
q, kv, indices, indptr = _nope_only_ragged_inputs(
_NOPE_ONLY_KV_LENS, num_heads, device
)
attn_sink = (
torch.randn(num_heads, dtype=torch.float32, device=device)
if with_sink
else None
)
scale = NOPE_ONLY_HEAD_DIM**-0.5
expected = _rocm_sparse_attn_prefill_ragged_triton(
q=q,
kv=kv,
indices=indices,
indptr=indptr,
scale=scale,
attn_sink=attn_sink,
nope_head_dim=NOPE_ONLY_HEAD_DIM,
rope_head_dim=0,
)
actual = _rocm_sparse_attn_decode_ragged_bf16_triton(
q=q,
kv=kv,
indices=indices,
indptr=indptr,
scale=scale,
attn_sink=attn_sink,
nope_head_dim=NOPE_ONLY_HEAD_DIM,
rope_head_dim=0,
num_splits=num_splits,
)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict:
device = torch.device("cuda")
set_random_seed(7)
@@ -1105,6 +1186,21 @@ def test_decode_num_splits_gfx950(monkeypatch) -> None:
assert mod._decode_gfx950_num_splits(1, 1, 128, 8192) == 32
assert mod._decode_gfx950_num_splits(17, 1, 128, 32) == 4
assert mod._decode_gfx950_num_splits(512, 1, 128, 7812) == 1
assert mod._decode_gfx950_num_splits(4, 1, 2048.0, 0.0, 32) == 32
def test_sparse_decode_bf16_num_splits_floor_and_tile_clamp(monkeypatch) -> None:
"""Short rows skip splitting, and splits never exceed the tiles they walk."""
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
monkeypatch.setattr(mod, "_decode_num_splits", lambda *args: 32)
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 32)
floor = mod._SPARSE_DECODE_BF16_MIN_SPLIT_LEN
block_k = mod._SPARSE_DECODE_BF16_BLOCK_K
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor - 1) == 1
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor) == floor // block_k
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, 2048) == 32
@requires_split_decode_arch
@@ -37,7 +37,9 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_sparse_attn_decode_bf16,
rocm_sparse_attn_prefill,
rocm_sparse_decode_bf16_num_splits,
)
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
from vllm.v1.worker.workspace import current_workspace_manager
@@ -904,20 +906,45 @@ class ROCMAiterMLASparseImpl(
self.sinks.reshape(1, self.num_heads, 1),
q.shape[1],
).reshape(-1)
rocm_sparse_attn_prefill(
q=q,
kv=kv_c_and_k_pe_cache.view(-1, 1, q.shape[-1]),
indices=None,
topk_length=None,
scale=self.scale,
head_dim=q.shape[-1],
nope_head_dim=self.kv_lora_rank,
rope_head_dim=q.shape[-1] - self.kv_lora_rank,
attn_sink=triton_sinks,
output=output,
ragged_indices=attn_metadata.paged_kv_indices,
ragged_indptr=attn_metadata.paged_kv_indptr,
kv = kv_c_and_k_pe_cache.view(-1, 1, q.shape[-1])
decode_num_splits = (
rocm_sparse_decode_bf16_num_splits(
num_tokens,
q.shape[1],
min(attn_metadata.max_seq_len, attn_metadata.topk_tokens),
)
if attn_metadata.num_decode_tokens == num_tokens
else 1
)
if decode_num_splits > 1:
rocm_sparse_attn_decode_bf16(
q=q,
kv=kv,
scale=self.scale,
head_dim=q.shape[-1],
nope_head_dim=self.kv_lora_rank,
rope_head_dim=q.shape[-1] - self.kv_lora_rank,
attn_sink=triton_sinks,
output=output,
ragged_indices=attn_metadata.paged_kv_indices,
ragged_indptr=attn_metadata.paged_kv_indptr,
num_splits=decode_num_splits,
)
else:
rocm_sparse_attn_prefill(
q=q,
kv=kv,
indices=None,
topk_length=None,
scale=self.scale,
head_dim=q.shape[-1],
nope_head_dim=self.kv_lora_rank,
rope_head_dim=q.shape[-1] - self.kv_lora_rank,
attn_sink=triton_sinks,
output=output,
ragged_indices=attn_metadata.paged_kv_indices,
ragged_indptr=attn_metadata.paged_kv_indptr,
)
output = AiterMLAHelper.get_mla_unpadded_o(self.num_heads, output)
return output, None
@@ -2208,6 +2208,119 @@ def _sparse_attn_prefill_ragged_kernel(
)
@triton.jit
def _sparse_attn_decode_ragged_bf16_partial_kernel(
q_ptr,
kv_ptr,
kv_indices_ptr,
kv_indptr_ptr,
part_m_ptr,
part_l_ptr,
part_acc_ptr,
q_stride_t,
q_stride_h,
q_stride_d,
kv_stride_n,
kv_stride_d,
pm_stride0,
pm_stride_s,
pa_stride0,
pa_stride_s,
pa_stride_h,
num_heads,
head_dim,
num_kv,
scale,
BLOCK_H: tl.constexpr,
BLOCK_D: tl.constexpr,
BLOCK_K: tl.constexpr,
NUM_SPLITS: tl.constexpr,
):
query_idx = tl.program_id(0)
split_id = tl.program_id(1)
pid_h = tl.program_id(2)
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
dim_offsets = tl.arange(0, BLOCK_D)
head_mask = head_offsets < num_heads
dim_mask = dim_offsets < head_dim
q = tl.load(
q_ptr
+ query_idx * q_stride_t
+ head_offsets[:, None] * q_stride_h
+ dim_offsets[None, :] * q_stride_d,
mask=head_mask[:, None] & dim_mask[None, :],
other=0.0,
)
neg_large = -3.4028234663852886e38
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32)
kv_start = tl.load(kv_indptr_ptr + query_idx)
kv_end = tl.load(kv_indptr_ptr + query_idx + 1)
kv_len = kv_end - kv_start
chunk = (kv_len + NUM_SPLITS - 1) // NUM_SPLITS
kv_lo = split_id * chunk
kv_hi = tl.minimum(kv_lo + chunk, kv_len)
k_offsets = tl.arange(0, BLOCK_K)
slot = tl.load(
kv_indices_ptr + kv_start + kv_lo + k_offsets,
mask=kv_lo + k_offsets < kv_hi,
other=-1,
)
for k_start in tl.range(kv_lo, kv_hi, BLOCK_K):
k_pos = k_start + k_offsets
in_range = k_pos < kv_hi
valid = in_range & (slot >= 0) & (slot < num_kv)
safe_slot = tl.where(valid, slot, 0)
kv = tl.load(
kv_ptr
+ _sparse_kv_row_offset(safe_slot[:, None], kv_stride_n)
+ dim_offsets[None, :] * kv_stride_d,
mask=valid[:, None] & dim_mask[None, :],
other=0.0,
)
next_k_pos = k_start + BLOCK_K + k_offsets
slot = tl.load(
kv_indices_ptr + kv_start + next_k_pos, mask=next_k_pos < kv_hi, other=-1
)
scores = tl.dot(q, tl.trans(kv)) * scale
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
m_block = tl.max(scores, axis=1)
m_new = tl.maximum(m_i, m_block)
alpha = tl.exp(m_i - m_new)
p = tl.exp(scores - m_new[:, None])
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
l_new = l_i * alpha + tl.sum(p, axis=1)
acc = acc * alpha[:, None] + tl.dot(p.to(kv.dtype), kv)
m_i = m_new
l_i = l_new
# Every launched slot is written; empty splits store the (neg_large, 0.0)
# sentinel the reduce combines to nothing.
pm_base = query_idx * pm_stride0 + split_id * pm_stride_s + head_offsets
tl.store(part_m_ptr + pm_base, m_i, mask=head_mask)
tl.store(part_l_ptr + pm_base, l_i, mask=head_mask)
tl.store(
part_acc_ptr
+ query_idx * pa_stride0
+ split_id * pa_stride_s
+ head_offsets[:, None] * pa_stride_h
+ dim_offsets[None, :],
acc,
mask=head_mask[:, None] & dim_mask[None, :],
)
@triton.jit
def _decode_e8m0_scales_triton(encoded_scales):
scale_bits = encoded_scales.to(tl.int32) << 23
@@ -3664,6 +3777,182 @@ def _decode_gfx950_num_splits(
return num_splits
# Empirically single-pass bf16 decode is faster than split-K below this number of
# selected slots per query, so we do not split.
_SPARSE_DECODE_BF16_MIN_SPLIT_LEN = 512
_SPARSE_DECODE_BF16_BLOCK_H = 16
_SPARSE_DECODE_BF16_BLOCK_K = 32
def rocm_sparse_decode_bf16_num_splits(
num_queries: int, num_heads: int, sparse_len: int
) -> int:
"""Number or kv splits in splitK for the sparse bf16 decode, or 1 for single-pass.
Args:
num_queries: Decode rows in the batch.
num_heads: Query heads per row.
sparse_len: Longest selected KV run any decode row can walk.
Returns:
The split count, or 1 when the caller should use the single-pass kernel.
"""
if sparse_len < _SPARSE_DECODE_BF16_MIN_SPLIT_LEN:
return 1
block_k = _SPARSE_DECODE_BF16_BLOCK_K
heads_blocks = triton.cdiv(num_heads, _SPARSE_DECODE_BF16_BLOCK_H)
select = _decode_gfx950_num_splits if _ON_GFX950 else _decode_num_splits
num_splits = select(num_queries, heads_blocks, sparse_len, 0.0, block_k)
# Number of splits cannot exceed the available k tiles
return max(1, min(num_splits, math.ceil(sparse_len / block_k)))
def _rocm_sparse_attn_decode_ragged_bf16_triton(
q: torch.Tensor,
kv: torch.Tensor,
indices: torch.Tensor,
indptr: torch.Tensor,
scale: float,
attn_sink: torch.Tensor | None,
nope_head_dim: int,
rope_head_dim: int,
num_splits: int,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Split-K decode over an bf16 ragged KV cache.
Partitions each query's selected tokens across different workgroups and combines
the partials through reduction.
Args:
q: Queries laid out as ``[sq, h, d]``.
kv: Unquantized KV rows laid out as ``[skv, d]``.
indices: Flattened per-query KV slots.
indptr: Segment offsets into ``indices``, ``[sq + 1]``.
scale: Softmax scale.
attn_sink: Optional per-head sink logits.
nope_head_dim: NoPE width of ``d``.
rope_head_dim: RoPE width of ``d``.
num_splits: Number of KV splits per query.
out: Optional destination with ``d`` trailing elements.
Returns:
The attention output, ``out`` when provided.
"""
assert q.ndim == 3, f"expected q=[sq,h,d], got {q.shape}"
assert kv.ndim == 2, f"expected kv=[skv,d], got {kv.shape}"
assert indices.ndim == 1, f"expected indices=[nnz], got {indices.shape}"
assert indptr.ndim == 1, f"expected indptr=[sq+1], got {indptr.shape}"
assert not q.is_cpu and not kv.is_cpu and not indices.is_cpu and not indptr.is_cpu
indices = _as_int32_contiguous_1d(indices)
indptr = _as_int32_contiguous_1d(indptr)
has_attn_sink = attn_sink is not None
if attn_sink is None:
attn_sink = torch.empty(1, device=q.device, dtype=torch.float32)
else:
attn_sink = attn_sink.contiguous()
num_queries, num_heads, head_dim = q.shape
assert indptr.numel() == num_queries + 1, (
f"expected indptr shape [{num_queries + 1}], got {indptr.shape}"
)
_validate_sparse_dims(
head_dim,
nope_head_dim,
rope_head_dim,
"_rocm_sparse_attn_decode_ragged_bf16_triton",
)
if out is None:
out = torch.empty_like(q)
assert out.shape[-1] == head_dim, (
f"expected out trailing dim {head_dim}, got {out.shape[-1]}"
)
block_h = _SPARSE_DECODE_BF16_BLOCK_H
block_d = triton.next_power_of_2(head_dim)
block_k = _SPARSE_DECODE_BF16_BLOCK_K
heads_blocks = triton.cdiv(num_heads, block_h)
comb_dim = nope_head_dim + rope_head_dim
part_m = torch.empty(
(num_queries, num_splits, num_heads), dtype=torch.float32, device=q.device
)
part_l = torch.empty_like(part_m)
part_acc = torch.empty(
(num_queries, num_splits, num_heads, comb_dim),
dtype=torch.float32,
device=q.device,
)
_sparse_attn_decode_ragged_bf16_partial_kernel[
(num_queries, num_splits, heads_blocks)
](
q,
kv,
indices,
indptr,
part_m,
part_l,
part_acc,
q.stride(0),
q.stride(1),
q.stride(2),
kv.stride(0),
kv.stride(1),
part_m.stride(0),
part_m.stride(1),
part_acc.stride(0),
part_acc.stride(1),
part_acc.stride(2),
num_heads,
head_dim,
kv.shape[0],
float(scale),
BLOCK_H=block_h,
BLOCK_D=block_d,
BLOCK_K=block_k,
NUM_SPLITS=num_splits,
num_warps=4,
)
_sparse_attn_decode_reduce_kernel[(num_queries, num_heads)](
part_m,
part_l,
part_acc,
attn_sink,
out,
None,
None,
None,
out.stride(0),
out.stride(1),
0,
head_dim // 32,
part_m.stride(0),
part_m.stride(1),
part_acc.stride(0),
part_acc.stride(1),
part_acc.stride(2),
0,
num_heads,
HAS_ATTN_SINK=has_attn_sink,
ADAPTIVE_SPLITS=False,
COMB_DIM=comb_dim,
BLOCK_H=1,
NUM_SPLITS=num_splits,
SPLITS_PAD=triton.next_power_of_2(num_splits),
FUSE_INV_ROPE=False,
NOPE=nope_head_dim,
HALF=rope_head_dim // 2,
QUANT_OUT=False,
num_warps=4,
)
return out
def _rocm_sparse_attn_decode_ragged_triton(
q: torch.Tensor,
main_cache: torch.Tensor,
@@ -4143,6 +4432,72 @@ def rocm_sparse_attn_prefill(
)
def rocm_sparse_attn_decode_bf16(
q: torch.Tensor,
kv: torch.Tensor,
scale: float,
head_dim: int,
nope_head_dim: int,
rope_head_dim: int,
attn_sink: torch.Tensor | None,
output: torch.Tensor,
ragged_indices: torch.Tensor,
ragged_indptr: torch.Tensor,
num_splits: int,
) -> None:
"""Run split-K sparse attention over decode rows using an unquantized KV cache.
Args:
q: Decode queries laid out as ``[sq, h, d]``.
kv: KV cache laid out as ``[skv, 1, d]``.
scale: Softmax scale.
head_dim: Post-absorption head width.
nope_head_dim: NoPE width of ``head_dim``.
rope_head_dim: RoPE width of ``head_dim``.
attn_sink: Optional per-head sink logits.
output: Destination, written in place.
ragged_indices: Flattened per-query KV slots.
ragged_indptr: Segment offsets into ``ragged_indices``, ``[sq + 1]``.
num_splits: KV splits per query, from
:func:`rocm_sparse_decode_bf16_num_splits`.
"""
assert kv.ndim == 3 and kv.shape[1] == 1, (
f"ROCm Triton sparse decode expects kv=[skv,1,d], got {kv.shape}"
)
_validate_sparse_dims(
head_dim,
nope_head_dim,
rope_head_dim,
"rocm_sparse_attn_decode_bf16",
)
num_queries, num_heads = q.shape[0], q.shape[1]
direct = output.shape[-1] == head_dim
out = (
output
if direct
else torch.empty(
(num_queries, num_heads, head_dim),
dtype=output.dtype,
device=output.device,
)
)
_rocm_sparse_attn_decode_ragged_bf16_triton(
q=q,
kv=kv.squeeze(1),
indices=ragged_indices,
indptr=ragged_indptr,
scale=scale,
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
nope_head_dim=nope_head_dim,
rope_head_dim=rope_head_dim,
num_splits=num_splits,
out=out,
)
if not direct:
output.copy_(out[..., : output.shape[-1]])
def rocm_sparse_attn_decode(
q: torch.Tensor,
kv_cache: torch.Tensor | None,