[ROCm][Perf][GLM-5.3-Flash] BF16 splitk kernel for sparse MLA decode (#58584)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Douglas Lehr
parent
3722645539
commit
87d9996abe
@@ -0,0 +1,130 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Benchmark the ROCm bf16 sparse-MLA decode split-K kernel in isolation.
|
||||
|
||||
Example usage:
|
||||
python3 benchmarks/kernels/benchmark_sparse_mla_decode_bf16.py \
|
||||
--rows 1 4 16 64 --lens 2048 --splits auto 4 16 32
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import json
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_decode_ragged_bf16_triton,
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
rocm_sparse_decode_bf16_num_splits,
|
||||
)
|
||||
|
||||
BLOCK_H = 16
|
||||
|
||||
|
||||
def make_inputs(rows: int, live: int, args):
|
||||
torch.manual_seed(args.seed)
|
||||
kv = (
|
||||
torch.randn(args.capacity, args.head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
* 0.125
|
||||
)
|
||||
q = (
|
||||
torch.randn(
|
||||
rows, args.heads, args.head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
* 0.125
|
||||
)
|
||||
slots = torch.stack(
|
||||
[torch.randperm(args.capacity, device="cuda")[:live] for _ in range(rows)]
|
||||
)
|
||||
indices = slots.reshape(-1).to(torch.int32)
|
||||
indptr = torch.arange(rows + 1, device="cuda", dtype=torch.int32) * live
|
||||
sink = (
|
||||
torch.randn(args.heads, device="cuda", dtype=torch.float32)
|
||||
if args.sink
|
||||
else None
|
||||
)
|
||||
return q, kv, indices, indptr, sink
|
||||
|
||||
|
||||
def resolve_splits(spec: str, rows: int, live: int, args) -> int:
|
||||
if spec != "auto":
|
||||
return int(spec)
|
||||
return rocm_sparse_decode_bf16_num_splits(rows, args.heads, live)
|
||||
|
||||
|
||||
def benchmark(backend: str, rows: int, live: int, spec: str, args) -> dict:
|
||||
q, kv, indices, indptr, sink = make_inputs(rows, live, args)
|
||||
nope, rope = args.head_dim, 0
|
||||
splits = resolve_splits(spec, rows, live, args)
|
||||
|
||||
def ragged():
|
||||
return _rocm_sparse_attn_prefill_ragged_triton(
|
||||
q, kv, indices, indptr, args.scale, sink, nope, rope
|
||||
)
|
||||
|
||||
def splitk():
|
||||
return _rocm_sparse_attn_decode_ragged_bf16_triton(
|
||||
q, kv, indices, indptr, args.scale, sink, nope, rope, splits
|
||||
)
|
||||
|
||||
run = ragged if backend == "ragged" else splitk
|
||||
torch.testing.assert_close(run(), ragged(), atol=2e-2, rtol=2e-2)
|
||||
ms = triton.testing.do_bench(run, warmup=args.warmup, rep=args.rep)
|
||||
return dict(
|
||||
backend=backend,
|
||||
rows=rows,
|
||||
live=live,
|
||||
heads=args.heads,
|
||||
head_dim=args.head_dim,
|
||||
splits=splits if backend == "splitk" else 1,
|
||||
workgroups=rows * triton.cdiv(args.heads, BLOCK_H) * splits,
|
||||
us=ms * 1000,
|
||||
kv_gbps=rows * live * args.head_dim * 2 / (ms * 1e6),
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--rows", type=int, nargs="+", default=[1, 2, 4, 8, 16, 64])
|
||||
parser.add_argument(
|
||||
"--lens",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[2048],
|
||||
help="Selected KV slots per row; GLM-5.3-Flash uses index_topk=2048",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--splits",
|
||||
nargs="+",
|
||||
default=["auto"],
|
||||
help="Split counts to sweep, or auto for the runtime heuristic",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backends",
|
||||
nargs="+",
|
||||
choices=["ragged", "splitk"],
|
||||
default=["ragged", "splitk"],
|
||||
)
|
||||
parser.add_argument(
|
||||
"--heads", type=int, default=8, help="Heads per rank; 64 at TP8 gives 8"
|
||||
)
|
||||
parser.add_argument("--head-dim", type=int, default=512)
|
||||
parser.add_argument("--capacity", type=int, default=65536)
|
||||
parser.add_argument("--scale", type=float, default=0.044194173824159216)
|
||||
parser.add_argument("--sink", action="store_true")
|
||||
parser.add_argument("--warmup", type=int, default=25)
|
||||
parser.add_argument("--rep", type=int, default=200)
|
||||
parser.add_argument("--seed", type=int, default=123)
|
||||
args = parser.parse_args()
|
||||
|
||||
print(json.dumps(dict(gpu=torch.cuda.get_device_name(0), args=vars(args))))
|
||||
for backend, rows, live in itertools.product(args.backends, args.rows, args.lens):
|
||||
specs = ["1"] if backend == "ragged" else args.splits
|
||||
for spec in specs:
|
||||
print(json.dumps(benchmark(backend, rows, live, spec, args)), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -156,6 +156,8 @@ def test_sparse_mla_sink_matches_ragged_reference(
|
||||
num_decodes=batch_size,
|
||||
num_decode_tokens=batch_size,
|
||||
max_query_len=1,
|
||||
max_seq_len=max(seq_lens),
|
||||
topk_tokens=max(seq_lens),
|
||||
)
|
||||
sinks = torch.linspace(-2.0, 6.0, real_heads, device=device)
|
||||
|
||||
@@ -367,12 +369,15 @@ def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache
|
||||
for page, offset in rows
|
||||
]
|
||||
lengths = torch.tensor([0, *(len(rows) for rows in selected_rows)])
|
||||
longest_row = max(len(rows) for rows in selected_rows)
|
||||
metadata = SimpleNamespace(
|
||||
block_size=block_size,
|
||||
num_prefills=0,
|
||||
num_decodes=len(selected_rows),
|
||||
num_decode_tokens=len(selected_rows),
|
||||
max_query_len=1,
|
||||
max_seq_len=longest_row,
|
||||
topk_tokens=longest_row,
|
||||
qo_indptr=torch.arange(
|
||||
len(selected_rows) + 1, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
@@ -479,6 +484,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
num_decodes=num_tokens,
|
||||
num_decode_tokens=num_tokens,
|
||||
max_query_len=1,
|
||||
max_seq_len=kv.shape[0],
|
||||
block_size=block_size,
|
||||
topk_tokens=impl.topk_indices_buffer.shape[1],
|
||||
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device="cuda"),
|
||||
|
||||
@@ -596,6 +596,87 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
NOPE_ONLY_HEAD_DIM = 512
|
||||
|
||||
|
||||
def _nope_only_ragged_inputs(
|
||||
kv_lens: list[int], num_heads: int, device: torch.device
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build a GLM-like 512+0 NoPE decode batch with ragged KV segments."""
|
||||
num_kv = 4096
|
||||
kv = (
|
||||
torch.randn(num_kv, NOPE_ONLY_HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
* 0.125
|
||||
)
|
||||
q = (
|
||||
torch.randn(
|
||||
len(kv_lens),
|
||||
num_heads,
|
||||
NOPE_ONLY_HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125
|
||||
)
|
||||
rows = [torch.randperm(num_kv)[:length].tolist() for length in kv_lens]
|
||||
indices, indptr = _ragged_from_rows(rows, device)
|
||||
return q, kv, indices, indptr
|
||||
|
||||
|
||||
# Lengths below the split count leave splits empty; non-multiples force a tail.
|
||||
_NOPE_ONLY_KV_LENS = [0, 1, 3, 37, 129, 2048]
|
||||
|
||||
|
||||
@requires_split_decode_arch
|
||||
@pytest.mark.parametrize("num_splits", [2, 32])
|
||||
@pytest.mark.parametrize("with_sink", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_bf16_split_k_matches_ragged(
|
||||
num_splits: int, with_sink: bool
|
||||
) -> None:
|
||||
"""Split-K decode must reproduce the single-workgroup ragged kernel."""
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_decode_ragged_bf16_triton,
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(0)
|
||||
num_heads = 16
|
||||
q, kv, indices, indptr = _nope_only_ragged_inputs(
|
||||
_NOPE_ONLY_KV_LENS, num_heads, device
|
||||
)
|
||||
attn_sink = (
|
||||
torch.randn(num_heads, dtype=torch.float32, device=device)
|
||||
if with_sink
|
||||
else None
|
||||
)
|
||||
scale = NOPE_ONLY_HEAD_DIM**-0.5
|
||||
|
||||
expected = _rocm_sparse_attn_prefill_ragged_triton(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
indptr=indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_ONLY_HEAD_DIM,
|
||||
rope_head_dim=0,
|
||||
)
|
||||
actual = _rocm_sparse_attn_decode_ragged_bf16_triton(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
indptr=indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_ONLY_HEAD_DIM,
|
||||
rope_head_dim=0,
|
||||
num_splits=num_splits,
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict:
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(7)
|
||||
@@ -1105,6 +1186,21 @@ def test_decode_num_splits_gfx950(monkeypatch) -> None:
|
||||
assert mod._decode_gfx950_num_splits(1, 1, 128, 8192) == 32
|
||||
assert mod._decode_gfx950_num_splits(17, 1, 128, 32) == 4
|
||||
assert mod._decode_gfx950_num_splits(512, 1, 128, 7812) == 1
|
||||
assert mod._decode_gfx950_num_splits(4, 1, 2048.0, 0.0, 32) == 32
|
||||
|
||||
|
||||
def test_sparse_decode_bf16_num_splits_floor_and_tile_clamp(monkeypatch) -> None:
|
||||
"""Short rows skip splitting, and splits never exceed the tiles they walk."""
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
monkeypatch.setattr(mod, "_decode_num_splits", lambda *args: 32)
|
||||
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 32)
|
||||
|
||||
floor = mod._SPARSE_DECODE_BF16_MIN_SPLIT_LEN
|
||||
block_k = mod._SPARSE_DECODE_BF16_BLOCK_K
|
||||
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor - 1) == 1
|
||||
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor) == floor // block_k
|
||||
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, 2048) == 32
|
||||
|
||||
|
||||
@requires_split_decode_arch
|
||||
|
||||
@@ -37,7 +37,9 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_sparse_attn_decode_bf16,
|
||||
rocm_sparse_attn_prefill,
|
||||
rocm_sparse_decode_bf16_num_splits,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
@@ -904,20 +906,45 @@ class ROCMAiterMLASparseImpl(
|
||||
self.sinks.reshape(1, self.num_heads, 1),
|
||||
q.shape[1],
|
||||
).reshape(-1)
|
||||
rocm_sparse_attn_prefill(
|
||||
q=q,
|
||||
kv=kv_c_and_k_pe_cache.view(-1, 1, q.shape[-1]),
|
||||
indices=None,
|
||||
topk_length=None,
|
||||
scale=self.scale,
|
||||
head_dim=q.shape[-1],
|
||||
nope_head_dim=self.kv_lora_rank,
|
||||
rope_head_dim=q.shape[-1] - self.kv_lora_rank,
|
||||
attn_sink=triton_sinks,
|
||||
output=output,
|
||||
ragged_indices=attn_metadata.paged_kv_indices,
|
||||
ragged_indptr=attn_metadata.paged_kv_indptr,
|
||||
kv = kv_c_and_k_pe_cache.view(-1, 1, q.shape[-1])
|
||||
decode_num_splits = (
|
||||
rocm_sparse_decode_bf16_num_splits(
|
||||
num_tokens,
|
||||
q.shape[1],
|
||||
min(attn_metadata.max_seq_len, attn_metadata.topk_tokens),
|
||||
)
|
||||
if attn_metadata.num_decode_tokens == num_tokens
|
||||
else 1
|
||||
)
|
||||
if decode_num_splits > 1:
|
||||
rocm_sparse_attn_decode_bf16(
|
||||
q=q,
|
||||
kv=kv,
|
||||
scale=self.scale,
|
||||
head_dim=q.shape[-1],
|
||||
nope_head_dim=self.kv_lora_rank,
|
||||
rope_head_dim=q.shape[-1] - self.kv_lora_rank,
|
||||
attn_sink=triton_sinks,
|
||||
output=output,
|
||||
ragged_indices=attn_metadata.paged_kv_indices,
|
||||
ragged_indptr=attn_metadata.paged_kv_indptr,
|
||||
num_splits=decode_num_splits,
|
||||
)
|
||||
else:
|
||||
rocm_sparse_attn_prefill(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=None,
|
||||
topk_length=None,
|
||||
scale=self.scale,
|
||||
head_dim=q.shape[-1],
|
||||
nope_head_dim=self.kv_lora_rank,
|
||||
rope_head_dim=q.shape[-1] - self.kv_lora_rank,
|
||||
attn_sink=triton_sinks,
|
||||
output=output,
|
||||
ragged_indices=attn_metadata.paged_kv_indices,
|
||||
ragged_indptr=attn_metadata.paged_kv_indptr,
|
||||
)
|
||||
output = AiterMLAHelper.get_mla_unpadded_o(self.num_heads, output)
|
||||
return output, None
|
||||
|
||||
|
||||
@@ -2208,6 +2208,119 @@ def _sparse_attn_prefill_ragged_kernel(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sparse_attn_decode_ragged_bf16_partial_kernel(
|
||||
q_ptr,
|
||||
kv_ptr,
|
||||
kv_indices_ptr,
|
||||
kv_indptr_ptr,
|
||||
part_m_ptr,
|
||||
part_l_ptr,
|
||||
part_acc_ptr,
|
||||
q_stride_t,
|
||||
q_stride_h,
|
||||
q_stride_d,
|
||||
kv_stride_n,
|
||||
kv_stride_d,
|
||||
pm_stride0,
|
||||
pm_stride_s,
|
||||
pa_stride0,
|
||||
pa_stride_s,
|
||||
pa_stride_h,
|
||||
num_heads,
|
||||
head_dim,
|
||||
num_kv,
|
||||
scale,
|
||||
BLOCK_H: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
NUM_SPLITS: tl.constexpr,
|
||||
):
|
||||
query_idx = tl.program_id(0)
|
||||
split_id = tl.program_id(1)
|
||||
pid_h = tl.program_id(2)
|
||||
|
||||
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||||
dim_offsets = tl.arange(0, BLOCK_D)
|
||||
head_mask = head_offsets < num_heads
|
||||
dim_mask = dim_offsets < head_dim
|
||||
|
||||
q = tl.load(
|
||||
q_ptr
|
||||
+ query_idx * q_stride_t
|
||||
+ head_offsets[:, None] * q_stride_h
|
||||
+ dim_offsets[None, :] * q_stride_d,
|
||||
mask=head_mask[:, None] & dim_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
neg_large = -3.4028234663852886e38
|
||||
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
|
||||
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||||
acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32)
|
||||
|
||||
kv_start = tl.load(kv_indptr_ptr + query_idx)
|
||||
kv_end = tl.load(kv_indptr_ptr + query_idx + 1)
|
||||
kv_len = kv_end - kv_start
|
||||
chunk = (kv_len + NUM_SPLITS - 1) // NUM_SPLITS
|
||||
kv_lo = split_id * chunk
|
||||
kv_hi = tl.minimum(kv_lo + chunk, kv_len)
|
||||
|
||||
k_offsets = tl.arange(0, BLOCK_K)
|
||||
slot = tl.load(
|
||||
kv_indices_ptr + kv_start + kv_lo + k_offsets,
|
||||
mask=kv_lo + k_offsets < kv_hi,
|
||||
other=-1,
|
||||
)
|
||||
for k_start in tl.range(kv_lo, kv_hi, BLOCK_K):
|
||||
k_pos = k_start + k_offsets
|
||||
in_range = k_pos < kv_hi
|
||||
valid = in_range & (slot >= 0) & (slot < num_kv)
|
||||
safe_slot = tl.where(valid, slot, 0)
|
||||
|
||||
kv = tl.load(
|
||||
kv_ptr
|
||||
+ _sparse_kv_row_offset(safe_slot[:, None], kv_stride_n)
|
||||
+ dim_offsets[None, :] * kv_stride_d,
|
||||
mask=valid[:, None] & dim_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
next_k_pos = k_start + BLOCK_K + k_offsets
|
||||
slot = tl.load(
|
||||
kv_indices_ptr + kv_start + next_k_pos, mask=next_k_pos < kv_hi, other=-1
|
||||
)
|
||||
|
||||
scores = tl.dot(q, tl.trans(kv)) * scale
|
||||
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
|
||||
|
||||
m_block = tl.max(scores, axis=1)
|
||||
m_new = tl.maximum(m_i, m_block)
|
||||
alpha = tl.exp(m_i - m_new)
|
||||
p = tl.exp(scores - m_new[:, None])
|
||||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||||
|
||||
acc = acc * alpha[:, None] + tl.dot(p.to(kv.dtype), kv)
|
||||
m_i = m_new
|
||||
l_i = l_new
|
||||
|
||||
# Every launched slot is written; empty splits store the (neg_large, 0.0)
|
||||
# sentinel the reduce combines to nothing.
|
||||
pm_base = query_idx * pm_stride0 + split_id * pm_stride_s + head_offsets
|
||||
tl.store(part_m_ptr + pm_base, m_i, mask=head_mask)
|
||||
tl.store(part_l_ptr + pm_base, l_i, mask=head_mask)
|
||||
tl.store(
|
||||
part_acc_ptr
|
||||
+ query_idx * pa_stride0
|
||||
+ split_id * pa_stride_s
|
||||
+ head_offsets[:, None] * pa_stride_h
|
||||
+ dim_offsets[None, :],
|
||||
acc,
|
||||
mask=head_mask[:, None] & dim_mask[None, :],
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _decode_e8m0_scales_triton(encoded_scales):
|
||||
scale_bits = encoded_scales.to(tl.int32) << 23
|
||||
@@ -3664,6 +3777,182 @@ def _decode_gfx950_num_splits(
|
||||
return num_splits
|
||||
|
||||
|
||||
# Empirically single-pass bf16 decode is faster than split-K below this number of
|
||||
# selected slots per query, so we do not split.
|
||||
_SPARSE_DECODE_BF16_MIN_SPLIT_LEN = 512
|
||||
_SPARSE_DECODE_BF16_BLOCK_H = 16
|
||||
_SPARSE_DECODE_BF16_BLOCK_K = 32
|
||||
|
||||
|
||||
def rocm_sparse_decode_bf16_num_splits(
|
||||
num_queries: int, num_heads: int, sparse_len: int
|
||||
) -> int:
|
||||
"""Number or kv splits in splitK for the sparse bf16 decode, or 1 for single-pass.
|
||||
|
||||
Args:
|
||||
num_queries: Decode rows in the batch.
|
||||
num_heads: Query heads per row.
|
||||
sparse_len: Longest selected KV run any decode row can walk.
|
||||
|
||||
Returns:
|
||||
The split count, or 1 when the caller should use the single-pass kernel.
|
||||
|
||||
"""
|
||||
if sparse_len < _SPARSE_DECODE_BF16_MIN_SPLIT_LEN:
|
||||
return 1
|
||||
block_k = _SPARSE_DECODE_BF16_BLOCK_K
|
||||
heads_blocks = triton.cdiv(num_heads, _SPARSE_DECODE_BF16_BLOCK_H)
|
||||
select = _decode_gfx950_num_splits if _ON_GFX950 else _decode_num_splits
|
||||
num_splits = select(num_queries, heads_blocks, sparse_len, 0.0, block_k)
|
||||
# Number of splits cannot exceed the available k tiles
|
||||
return max(1, min(num_splits, math.ceil(sparse_len / block_k)))
|
||||
|
||||
|
||||
def _rocm_sparse_attn_decode_ragged_bf16_triton(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
indptr: torch.Tensor,
|
||||
scale: float,
|
||||
attn_sink: torch.Tensor | None,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
num_splits: int,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Split-K decode over an bf16 ragged KV cache.
|
||||
|
||||
Partitions each query's selected tokens across different workgroups and combines
|
||||
the partials through reduction.
|
||||
|
||||
Args:
|
||||
q: Queries laid out as ``[sq, h, d]``.
|
||||
kv: Unquantized KV rows laid out as ``[skv, d]``.
|
||||
indices: Flattened per-query KV slots.
|
||||
indptr: Segment offsets into ``indices``, ``[sq + 1]``.
|
||||
scale: Softmax scale.
|
||||
attn_sink: Optional per-head sink logits.
|
||||
nope_head_dim: NoPE width of ``d``.
|
||||
rope_head_dim: RoPE width of ``d``.
|
||||
num_splits: Number of KV splits per query.
|
||||
out: Optional destination with ``d`` trailing elements.
|
||||
|
||||
Returns:
|
||||
The attention output, ``out`` when provided.
|
||||
|
||||
"""
|
||||
assert q.ndim == 3, f"expected q=[sq,h,d], got {q.shape}"
|
||||
assert kv.ndim == 2, f"expected kv=[skv,d], got {kv.shape}"
|
||||
assert indices.ndim == 1, f"expected indices=[nnz], got {indices.shape}"
|
||||
assert indptr.ndim == 1, f"expected indptr=[sq+1], got {indptr.shape}"
|
||||
assert not q.is_cpu and not kv.is_cpu and not indices.is_cpu and not indptr.is_cpu
|
||||
|
||||
indices = _as_int32_contiguous_1d(indices)
|
||||
indptr = _as_int32_contiguous_1d(indptr)
|
||||
has_attn_sink = attn_sink is not None
|
||||
if attn_sink is None:
|
||||
attn_sink = torch.empty(1, device=q.device, dtype=torch.float32)
|
||||
else:
|
||||
attn_sink = attn_sink.contiguous()
|
||||
|
||||
num_queries, num_heads, head_dim = q.shape
|
||||
assert indptr.numel() == num_queries + 1, (
|
||||
f"expected indptr shape [{num_queries + 1}], got {indptr.shape}"
|
||||
)
|
||||
_validate_sparse_dims(
|
||||
head_dim,
|
||||
nope_head_dim,
|
||||
rope_head_dim,
|
||||
"_rocm_sparse_attn_decode_ragged_bf16_triton",
|
||||
)
|
||||
if out is None:
|
||||
out = torch.empty_like(q)
|
||||
assert out.shape[-1] == head_dim, (
|
||||
f"expected out trailing dim {head_dim}, got {out.shape[-1]}"
|
||||
)
|
||||
|
||||
block_h = _SPARSE_DECODE_BF16_BLOCK_H
|
||||
block_d = triton.next_power_of_2(head_dim)
|
||||
block_k = _SPARSE_DECODE_BF16_BLOCK_K
|
||||
heads_blocks = triton.cdiv(num_heads, block_h)
|
||||
comb_dim = nope_head_dim + rope_head_dim
|
||||
|
||||
part_m = torch.empty(
|
||||
(num_queries, num_splits, num_heads), dtype=torch.float32, device=q.device
|
||||
)
|
||||
part_l = torch.empty_like(part_m)
|
||||
part_acc = torch.empty(
|
||||
(num_queries, num_splits, num_heads, comb_dim),
|
||||
dtype=torch.float32,
|
||||
device=q.device,
|
||||
)
|
||||
|
||||
_sparse_attn_decode_ragged_bf16_partial_kernel[
|
||||
(num_queries, num_splits, heads_blocks)
|
||||
](
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
indptr,
|
||||
part_m,
|
||||
part_l,
|
||||
part_acc,
|
||||
q.stride(0),
|
||||
q.stride(1),
|
||||
q.stride(2),
|
||||
kv.stride(0),
|
||||
kv.stride(1),
|
||||
part_m.stride(0),
|
||||
part_m.stride(1),
|
||||
part_acc.stride(0),
|
||||
part_acc.stride(1),
|
||||
part_acc.stride(2),
|
||||
num_heads,
|
||||
head_dim,
|
||||
kv.shape[0],
|
||||
float(scale),
|
||||
BLOCK_H=block_h,
|
||||
BLOCK_D=block_d,
|
||||
BLOCK_K=block_k,
|
||||
NUM_SPLITS=num_splits,
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
_sparse_attn_decode_reduce_kernel[(num_queries, num_heads)](
|
||||
part_m,
|
||||
part_l,
|
||||
part_acc,
|
||||
attn_sink,
|
||||
out,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
0,
|
||||
head_dim // 32,
|
||||
part_m.stride(0),
|
||||
part_m.stride(1),
|
||||
part_acc.stride(0),
|
||||
part_acc.stride(1),
|
||||
part_acc.stride(2),
|
||||
0,
|
||||
num_heads,
|
||||
HAS_ATTN_SINK=has_attn_sink,
|
||||
ADAPTIVE_SPLITS=False,
|
||||
COMB_DIM=comb_dim,
|
||||
BLOCK_H=1,
|
||||
NUM_SPLITS=num_splits,
|
||||
SPLITS_PAD=triton.next_power_of_2(num_splits),
|
||||
FUSE_INV_ROPE=False,
|
||||
NOPE=nope_head_dim,
|
||||
HALF=rope_head_dim // 2,
|
||||
QUANT_OUT=False,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _rocm_sparse_attn_decode_ragged_triton(
|
||||
q: torch.Tensor,
|
||||
main_cache: torch.Tensor,
|
||||
@@ -4143,6 +4432,72 @@ def rocm_sparse_attn_prefill(
|
||||
)
|
||||
|
||||
|
||||
def rocm_sparse_attn_decode_bf16(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
scale: float,
|
||||
head_dim: int,
|
||||
nope_head_dim: int,
|
||||
rope_head_dim: int,
|
||||
attn_sink: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
ragged_indices: torch.Tensor,
|
||||
ragged_indptr: torch.Tensor,
|
||||
num_splits: int,
|
||||
) -> None:
|
||||
"""Run split-K sparse attention over decode rows using an unquantized KV cache.
|
||||
|
||||
Args:
|
||||
q: Decode queries laid out as ``[sq, h, d]``.
|
||||
kv: KV cache laid out as ``[skv, 1, d]``.
|
||||
scale: Softmax scale.
|
||||
head_dim: Post-absorption head width.
|
||||
nope_head_dim: NoPE width of ``head_dim``.
|
||||
rope_head_dim: RoPE width of ``head_dim``.
|
||||
attn_sink: Optional per-head sink logits.
|
||||
output: Destination, written in place.
|
||||
ragged_indices: Flattened per-query KV slots.
|
||||
ragged_indptr: Segment offsets into ``ragged_indices``, ``[sq + 1]``.
|
||||
num_splits: KV splits per query, from
|
||||
:func:`rocm_sparse_decode_bf16_num_splits`.
|
||||
|
||||
"""
|
||||
assert kv.ndim == 3 and kv.shape[1] == 1, (
|
||||
f"ROCm Triton sparse decode expects kv=[skv,1,d], got {kv.shape}"
|
||||
)
|
||||
_validate_sparse_dims(
|
||||
head_dim,
|
||||
nope_head_dim,
|
||||
rope_head_dim,
|
||||
"rocm_sparse_attn_decode_bf16",
|
||||
)
|
||||
num_queries, num_heads = q.shape[0], q.shape[1]
|
||||
direct = output.shape[-1] == head_dim
|
||||
out = (
|
||||
output
|
||||
if direct
|
||||
else torch.empty(
|
||||
(num_queries, num_heads, head_dim),
|
||||
dtype=output.dtype,
|
||||
device=output.device,
|
||||
)
|
||||
)
|
||||
_rocm_sparse_attn_decode_ragged_bf16_triton(
|
||||
q=q,
|
||||
kv=kv.squeeze(1),
|
||||
indices=ragged_indices,
|
||||
indptr=ragged_indptr,
|
||||
scale=scale,
|
||||
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
|
||||
nope_head_dim=nope_head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
num_splits=num_splits,
|
||||
out=out,
|
||||
)
|
||||
if not direct:
|
||||
output.copy_(out[..., : output.shape[-1]])
|
||||
|
||||
|
||||
def rocm_sparse_attn_decode(
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor | None,
|
||||
|
||||
Reference in New Issue
Block a user