[ROCm][DSv4] implement flash sparse mla with triton kernels (#41812)

Signed-off-by: whx-sjtu <xiaowang990929@gmail.com>
This commit is contained in:
Hexiang Wang
2026-05-11 09:27:11 -07:00
committed by GitHub
parent 0d453e2336
commit 7863fff6e5
6 changed files with 1860 additions and 223 deletions
@@ -0,0 +1,377 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="Only used by ROCm"
)
NOPE_HEAD_DIM = 448
ROPE_HEAD_DIM = 64
HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM
def _ref_global_topk_ragged(
topk_indices: torch.Tensor,
token_to_req_indices: torch.Tensor,
block_table: torch.Tensor,
block_size: int,
is_valid_token: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
topk = topk_indices.reshape(topk_indices.shape[0], -1)
valid = (topk >= 0) & is_valid_token[:, None]
lens = valid.sum(dim=1, dtype=torch.int32)
indptr = torch.zeros(lens.shape[0] + 1, dtype=torch.int32, device=topk.device)
torch.cumsum(lens, dim=0, out=indptr[1:])
safe_topk = torch.clamp(topk, min=0)
block_indices = safe_topk // block_size
block_offsets = safe_topk % block_size
req_indices = token_to_req_indices[:, None].expand_as(topk)
slot_ids = block_table[req_indices, block_indices] * block_size + block_offsets
offsets = torch.arange(topk.shape[1], dtype=torch.int32, device=topk.device)
positions = indptr[:-1, None] + offsets[None, :]
return slot_ids[valid], positions[valid].to(torch.long), indptr, lens
def _ref_sparse_prefill_ragged(
q: torch.Tensor,
kv: torch.Tensor,
rows: list[list[int]],
scale: float,
attn_sink: torch.Tensor | None,
) -> torch.Tensor:
q_f32 = q.float()
kv_f32 = kv.float()
out = torch.empty_like(q_f32)
for query_idx in range(q.shape[0]):
row_indices = rows[query_idx]
for head_idx in range(q.shape[1]):
if row_indices:
selected_kv = kv_f32[row_indices]
scores = torch.mv(selected_kv, q_f32[query_idx, head_idx]) * scale
if attn_sink is not None:
scores_with_sink = torch.cat(
[scores, attn_sink[head_idx].float().reshape(1)]
)
probs = torch.softmax(scores_with_sink, dim=0)[:-1]
else:
probs = torch.softmax(scores, dim=0)
out[query_idx, head_idx] = torch.sum(
probs[:, None] * selected_kv, dim=0
)
else:
out[query_idx, head_idx] = 0
return out.to(torch.bfloat16)
def _pack_fp8_ds_mla_cache(kv: torch.Tensor, block_size: int) -> torch.Tensor:
assert kv.shape[-1] == HEAD_DIM
num_tokens = kv.shape[0]
num_blocks = (num_tokens + block_size - 1) // block_size
cache = torch.zeros(
(num_blocks, block_size, 584),
dtype=torch.uint8,
device=kv.device,
)
cache_flat = cache.view(torch.uint8).flatten()
kv_nope_fp8 = (
kv[:, :NOPE_HEAD_DIM].to(current_platform.fp8_dtype()).view(torch.uint8)
)
kv_rope_u8 = kv[:, NOPE_HEAD_DIM:].contiguous().view(torch.uint8)
for slot in range(num_tokens):
block_idx = slot // block_size
pos = slot % block_size
block_base = block_idx * cache.stride(0)
token_base = block_base + pos * 576
scale_base = block_base + block_size * 576 + pos * 8
cache_flat[token_base : token_base + NOPE_HEAD_DIM].copy_(kv_nope_fp8[slot])
cache_flat[
token_base + NOPE_HEAD_DIM : token_base + NOPE_HEAD_DIM + ROPE_HEAD_DIM * 2
].copy_(kv_rope_u8[slot])
cache_flat[scale_base : scale_base + 7].fill_(127)
return cache
def _read_fp8_ds_mla_cache(
cache: torch.Tensor, slot: int, block_size: int
) -> torch.Tensor:
cache_flat = cache.view(torch.uint8).flatten()
block_idx = slot // block_size
pos = slot % block_size
block_base = block_idx * cache.stride(0)
token_base = block_base + pos * 576
nope_u8 = cache_flat[token_base : token_base + NOPE_HEAD_DIM]
nope = nope_u8.view(current_platform.fp8_dtype()).to(torch.float32)
rope_u8 = cache_flat[
token_base + NOPE_HEAD_DIM : token_base + NOPE_HEAD_DIM + ROPE_HEAD_DIM * 2
]
rope = rope_u8.view(torch.bfloat16).to(torch.float32)
return torch.cat([nope, rope])
def _ref_sparse_decode_ragged(
q: torch.Tensor,
main_cache: torch.Tensor,
main_rows: list[list[int]],
scale: float,
attn_sink: torch.Tensor | None,
block_size: int,
extra_cache: torch.Tensor | None = None,
extra_rows: list[list[int]] | None = None,
) -> torch.Tensor:
q_f32 = q.float()
out = torch.empty_like(q_f32)
for query_idx in range(q.shape[0]):
row_kv = [
_read_fp8_ds_mla_cache(main_cache, int(slot), block_size)
for slot in main_rows[query_idx]
]
if extra_cache is not None and extra_rows is not None:
row_kv.extend(
_read_fp8_ds_mla_cache(extra_cache, int(slot), block_size)
for slot in extra_rows[query_idx]
)
kv = torch.stack(row_kv).to(q.device)
for head_idx in range(q.shape[1]):
scores = torch.mv(kv, q_f32[query_idx, head_idx]) * scale
if attn_sink is not None:
scores_with_sink = torch.cat(
[scores, attn_sink[head_idx].float().reshape(1)]
)
probs = torch.softmax(scores_with_sink, dim=0)[:-1]
else:
probs = torch.softmax(scores, dim=0)
out[query_idx, head_idx] = torch.sum(probs[:, None] * kv, dim=0)
return out.to(torch.bfloat16)
def _ref_combine_topk_swa_ragged(
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
expected_ragged = torch.tensor(
[
100,
101,
7,
8,
9,
110,
111,
8,
9,
10,
120,
121,
122,
9,
10,
11,
150,
27,
28,
29,
160,
161,
28,
29,
30,
],
dtype=torch.int32,
device=device,
)
expected_lens = torch.tensor([5, 5, 6, 4, 5], dtype=torch.int32, device=device)
expected_indptr = torch.zeros(6, dtype=torch.int32, device=device)
torch.cumsum(expected_lens, dim=0, out=expected_indptr[1:])
return expected_ragged, expected_indptr, expected_lens
@torch.inference_mode()
def test_compute_global_topk_ragged_indices_and_indptr() -> None:
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
compute_global_topk_ragged_indices_and_indptr,
)
device = torch.device("cuda")
block_size = 4
topk_indices = torch.tensor(
[
[0, 3, 4, -1],
[5, 8, -1, -1],
[2, 7, 9, -1],
],
dtype=torch.int32,
device=device,
)
token_to_req_indices = torch.tensor([0, 1, 1], dtype=torch.int32, device=device)
block_table = torch.tensor(
[
[10, 11, 12],
[20, 21, 22],
],
dtype=torch.int32,
device=device,
)
is_valid_token = torch.tensor([True, False, True], dtype=torch.bool, device=device)
actual_ragged, actual_indptr, actual_lens = (
compute_global_topk_ragged_indices_and_indptr(
topk_indices,
token_to_req_indices,
block_table,
block_size,
is_valid_token,
)
)
expected_values, expected_positions, expected_indptr, expected_lens = (
_ref_global_topk_ragged(
topk_indices,
token_to_req_indices,
block_table,
block_size,
is_valid_token,
)
)
torch.testing.assert_close(actual_ragged[expected_positions], expected_values)
torch.testing.assert_close(actual_indptr, expected_indptr)
torch.testing.assert_close(actual_lens, expected_lens)
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_kernel() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
device = torch.device("cuda")
torch.manual_seed(0)
q = torch.randn(3, 3, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
kv = torch.randn(5, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
indices = torch.tensor([0, 2, 1, 3, 4], dtype=torch.int32, device=device)
indptr = torch.tensor([0, 2, 5, 5], dtype=torch.int32, device=device)
attn_sink = torch.tensor([-0.25, 0.0, 0.25], dtype=torch.float32, device=device)
scale = HEAD_DIM**-0.5
actual = _rocm_sparse_attn_prefill_ragged_triton(
q=q,
kv=kv,
indices=indices,
indptr=indptr,
scale=scale,
attn_sink=attn_sink,
nope_head_dim=NOPE_HEAD_DIM,
rope_head_dim=ROPE_HEAD_DIM,
)
expected = _ref_sparse_prefill_ragged(
q, kv, [[0, 2], [1, 3, 4], []], scale, attn_sink
)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
@torch.inference_mode()
def test_sparse_attn_decode_ragged_kernel() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_decode_ragged_triton,
)
device = torch.device("cuda")
torch.manual_seed(1)
block_size = 4
q = torch.randn(2, 3, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
main_kv = torch.randn(6, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
extra_kv = torch.randn(5, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
main_cache = _pack_fp8_ds_mla_cache(main_kv, block_size)
extra_cache = _pack_fp8_ds_mla_cache(extra_kv, block_size)
main_indices = torch.tensor([0, 2, 4, 1], dtype=torch.int32, device=device)
main_indptr = torch.tensor([0, 2, 4], dtype=torch.int32, device=device)
extra_indices = torch.tensor([1, 3, 0], dtype=torch.int32, device=device)
extra_indptr = torch.tensor([0, 1, 3], dtype=torch.int32, device=device)
attn_sink = torch.tensor([-0.1, 0.0, 0.1], dtype=torch.float32, device=device)
scale = HEAD_DIM**-0.5
actual = _rocm_sparse_attn_decode_ragged_triton(
q=q,
main_cache=main_cache,
main_indices=main_indices,
main_indptr=main_indptr,
scale=scale,
attn_sink=attn_sink,
nope_head_dim=NOPE_HEAD_DIM,
rope_head_dim=ROPE_HEAD_DIM,
extra_cache=extra_cache,
extra_indices=extra_indices,
extra_indptr=extra_indptr,
)
expected = _ref_sparse_decode_ragged(
q=q,
main_cache=main_cache,
main_rows=[[0, 2], [4, 1]],
scale=scale,
attn_sink=attn_sink,
block_size=block_size,
extra_cache=extra_cache,
extra_rows=[[1], [3, 0]],
)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
@torch.inference_mode()
def test_combine_topk_swa_indices_ragged() -> None:
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
combine_topk_swa_indices_ragged,
)
device = torch.device("cuda")
topk_indices = torch.tensor(
[
[100, 101, 102, 103],
[110, 111, 112, 113],
[120, 121, 122, 123],
[130, 131, 132, 133],
[140, 141, 142, 143],
],
dtype=torch.int32,
device=device,
)
query_start_loc = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
seq_lens = torch.tensor([6, 4], dtype=torch.int32, device=device)
gather_lens = torch.tensor([4, 3], dtype=torch.int32, device=device)
window_size = 3
compress_ratio = 2
topk = 4
M = 20
N = 8
actual_ragged, actual_indptr, actual_lens = combine_topk_swa_indices_ragged(
topk_indices,
query_start_loc,
seq_lens,
gather_lens,
window_size,
compress_ratio,
topk,
M,
N,
)
expected_ragged, expected_indptr, expected_lens = _ref_combine_topk_swa_ragged(
device
)
torch.testing.assert_close(
actual_ragged[: expected_ragged.numel()], expected_ragged
)
torch.testing.assert_close(actual_indptr, expected_indptr)
torch.testing.assert_close(actual_lens, expected_lens)
@@ -28,11 +28,7 @@ from vllm.v1.attention.ops.deepseek_v4_ops import (
fused_inv_rope_fp8_quant,
fused_q_kv_rmsnorm,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_forward_decode_fallback,
rocm_inv_rope_einsum,
rocm_sparse_attn_prefill,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import rocm_inv_rope_einsum
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import (
@@ -725,6 +721,12 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
self.kv_cache = torch.tensor([])
def get_attn_backend(self) -> type[AttentionBackend]:
if current_platform.is_rocm():
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
DeepseekV4ROCMAiterMLASparseBackend,
)
return DeepseekV4ROCMAiterMLASparseBackend
return DeepseekV4FlashMLASparseBackend
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
@@ -757,6 +759,14 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}"
)
if current_platform.is_rocm():
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
DeepseekV4ROCMAiterMLASparseImpl,
)
DeepseekV4ROCMAiterMLASparseImpl.forward(self, q, kv, positions, output)
return
# Get SWA and indexer metadata from forward context
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
@@ -839,25 +849,6 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
swa_indices = swa_metadata.decode_swa_indices
swa_lens = swa_metadata.decode_swa_lens
if current_platform.is_rocm():
rocm_forward_decode_fallback(
q=q,
kv_cache=kv_cache,
swa_k_cache=self.swa_cache_layer.kv_cache,
swa_only=swa_only,
topk_indices=topk_indices,
topk_lens=topk_lens,
swa_indices=swa_indices,
swa_lens=swa_lens,
attn_sink=self.attn_sink,
scale=self.scale,
head_dim=self.head_dim,
nope_head_dim=self.nope_head_dim,
rope_head_dim=self.rope_head_dim,
output=output,
)
return
# We treat queries in the same seq as different queries
# and later we only attend by generated indices.
# q arrives pre-padded to self.padded_heads by the outer wrapper.
@@ -1021,28 +1012,15 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
M,
N,
)
if current_platform.is_rocm():
rocm_sparse_attn_prefill(
q=q[query_start:query_end],
kv=kv.view(-1, 1, q.shape[-1]),
indices=combined_indices.unsqueeze(1),
topk_length=combined_lens,
scale=self.scale,
head_dim=self.head_dim,
attn_sink=self.attn_sink,
output=output[query_start:query_end],
)
else:
output_chunk, _, _ = flash_mla_sparse_fwd(
q=q[query_start:query_end],
kv=kv.view(-1, 1, q.shape[-1]),
indices=combined_indices.unsqueeze(1),
sm_scale=self.scale,
attn_sink=self.attn_sink,
topk_length=combined_lens,
out=output[query_start:query_end],
)
flash_mla_sparse_fwd(
q=q[query_start:query_end],
kv=kv.view(-1, 1, q.shape[-1]),
indices=combined_indices.unsqueeze(1),
sm_scale=self.scale,
attn_sink=self.attn_sink,
topk_length=combined_lens,
out=output[query_start:query_end],
)
class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from typing import TYPE_CHECKING, Any, ClassVar
import numpy as np
import torch
@@ -112,7 +112,7 @@ class FlashMLASparseBackend(AttentionBackend):
return FlashMLASparseMetadataBuilder
@staticmethod
def get_impl_cls() -> type["FlashMLASparseImpl"]:
def get_impl_cls() -> type[SparseMLAAttentionImpl[Any]]:
return FlashMLASparseImpl
@classmethod
@@ -0,0 +1,682 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING, cast
import torch
from vllm.forward_context import get_forward_context
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backend import (
AttentionLayer,
CommonAttentionMetadata,
SparseMLAAttentionImpl,
)
from vllm.v1.attention.backends.mla.flashmla_sparse import (
DeepseekV4FlashMLASparseBackend,
FlashMLASparseMetadata,
FlashMLASparseMetadataBuilder,
)
from vllm.v1.attention.backends.mla.sparse_swa import (
DeepseekSparseSWAMetadata,
DeepseekSparseSWAMetadataBuilder,
)
from vllm.v1.attention.ops.deepseek_v4_ops import dequantize_and_gather_k_cache
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
build_ragged_indices_from_dense,
rocm_sparse_attn_decode,
rocm_sparse_attn_prefill,
)
from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING:
from vllm.model_executor.layers.deepseek_v4_attention import (
DeepseekV4MLAAttention,
)
def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
lengths = lengths.to(dtype=torch.int32).contiguous()
indptr = torch.zeros(lengths.shape[0] + 1, dtype=torch.int32, device=lengths.device)
torch.cumsum(lengths, dim=0, out=indptr[1:])
return indptr
@triton.jit
def _compute_topk_lens_kernel(
topk_lens_ptr,
topk_indices_ptr,
topk_indices_stride,
topk,
is_valid_token_ptr,
TRITON_BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0)
is_valid_token = tl.load(is_valid_token_ptr + token_idx)
count = tl.zeros((), dtype=tl.int32)
for i in range(0, topk, TRITON_BLOCK_SIZE):
offset = i + tl.arange(0, TRITON_BLOCK_SIZE)
mask = offset < topk
local_idx = tl.load(
topk_indices_ptr + token_idx * topk_indices_stride + offset,
mask=mask,
other=-1,
)
count += tl.sum((local_idx >= 0).to(tl.int32), axis=0)
tl.store(topk_lens_ptr + token_idx, tl.where(is_valid_token, count, 0))
@triton.jit
def _pack_global_topk_ragged_kernel(
global_topk_ragged_ptr,
topk_indptr_ptr,
topk_indices_ptr,
topk_indices_stride,
token_to_req_indices_ptr,
block_table_ptr,
block_table_stride,
block_size,
topk,
BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0)
block_idx = tl.program_id(1)
offset = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
out_start = tl.load(topk_indptr_ptr + token_idx)
out_end = tl.load(topk_indptr_ptr + token_idx + 1)
out_len = out_end - out_start
if block_idx * BLOCK_SIZE >= out_len:
return
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
mask = (offset < out_len) & (offset < topk)
local_idx = tl.load(
topk_indices_ptr + token_idx * topk_indices_stride + offset,
mask=mask,
other=-1,
)
valid = mask & (local_idx >= 0)
block_indices = local_idx // block_size
block_numbers = tl.load(
block_table_ptr + req_idx * block_table_stride + block_indices,
mask=valid,
other=0,
)
block_offsets = local_idx % block_size
slot_ids = tl.where(valid, block_numbers * block_size + block_offsets, -1)
tl.store(global_topk_ragged_ptr + out_start + offset, slot_ids, mask=mask)
def compute_global_topk_ragged_indices_and_indptr(
topk_indices: torch.Tensor,
token_to_req_indices: torch.Tensor,
block_table: torch.Tensor,
block_size: int,
is_valid_token: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous()
num_tokens = topk_indices.shape[0]
topk = topk_indices.shape[1]
topk_lens = torch.empty(num_tokens, dtype=torch.int32, device=topk_indices.device)
_compute_topk_lens_kernel[(num_tokens,)](
topk_lens,
topk_indices,
topk_indices.stride(0),
topk,
is_valid_token,
TRITON_BLOCK_SIZE=1024,
)
topk_indptr = _build_indptr_from_lengths(topk_lens)
global_topk_ragged = torch.empty(
num_tokens * topk,
dtype=torch.int32,
device=topk_indices.device,
)
if global_topk_ragged.numel() > 0:
block = 128
_pack_global_topk_ragged_kernel[(num_tokens, triton.cdiv(topk, block))](
global_topk_ragged,
topk_indptr,
topk_indices,
topk_indices.stride(0),
token_to_req_indices,
block_table,
block_table.stride(0),
block_size,
topk,
BLOCK_SIZE=block,
)
return global_topk_ragged, topk_indptr, topk_lens
@triton.jit
def _compute_combined_lens_kernel(
combined_lens_ptr,
query_start_loc_ptr,
seq_lens_ptr,
TOP_K: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
WINDOW_SIZE: tl.constexpr,
):
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
num_workers = tl.num_programs(1)
base = tl.load(query_start_loc_ptr)
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
query_len = query_end - query_start
seq_len = tl.load(seq_lens_ptr + batch_idx)
start_pos = seq_len - query_len
for token_idx in range(query_start + worker_id, query_end, num_workers):
token_idx_in_query = token_idx - query_start
pos = start_pos + token_idx_in_query
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
@triton.jit
def _combine_topk_swa_indices_ragged_kernel(
combined_ragged_ptr,
combined_indptr_ptr,
topk_indices_ptr,
topk_indices_stride,
query_start_loc_ptr,
seq_lens_ptr,
gather_lens_ptr,
M,
N,
topk_width,
TOP_K: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
WINDOW_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
block_idx = tl.program_id(2)
num_workers = tl.num_programs(1)
base = tl.load(query_start_loc_ptr)
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
query_len = query_end - query_start
seq_len = tl.load(seq_lens_ptr + batch_idx)
gather_len = tl.load(gather_lens_ptr + batch_idx)
start_pos = seq_len - query_len
gather_start = seq_len - gather_len
for token_idx in range(query_start + worker_id, query_end, num_workers):
token_idx_in_query = token_idx - query_start
pos = start_pos + token_idx_in_query
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
combined_len = topk_len + swa_len
offset = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
if block_idx * BLOCK_SIZE < combined_len:
out_start = tl.load(combined_indptr_ptr + token_idx)
topk_mask = (offset < topk_len) & (offset < topk_width)
topk_vals = tl.load(
topk_indices_ptr + token_idx * topk_indices_stride + offset,
mask=topk_mask,
other=-1,
)
tl.store(
combined_ragged_ptr + out_start + offset,
topk_vals + M * batch_idx,
mask=topk_mask,
)
swa_offset = offset - topk_len
swa_mask = (offset >= topk_len) & (swa_offset < swa_len)
tl.store(
combined_ragged_ptr + out_start + offset,
M * batch_idx + N + swa_offset + pos - swa_len + 1 - gather_start,
mask=swa_mask,
)
def combine_topk_swa_indices_ragged(
topk_indices: torch.Tensor,
query_start_loc: torch.Tensor,
seq_lens: torch.Tensor,
gather_lens: torch.Tensor,
window_size: int,
compress_ratio: int,
topk: int,
M: int,
N: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous()
num_tokens = topk_indices.shape[0]
num_reqs = seq_lens.shape[0]
combined_lens = torch.empty(
num_tokens, dtype=torch.int32, device=topk_indices.device
)
num_workers = 128
_compute_combined_lens_kernel[(num_reqs, num_workers)](
combined_lens,
query_start_loc,
seq_lens,
TOP_K=topk,
COMPRESS_RATIO=compress_ratio,
WINDOW_SIZE=window_size,
)
combined_indptr = _build_indptr_from_lengths(combined_lens)
combined_ragged = torch.empty(
num_tokens * (topk + window_size),
dtype=torch.int32,
device=topk_indices.device,
)
if combined_ragged.numel() > 0:
block = 128
_combine_topk_swa_indices_ragged_kernel[
(num_reqs, num_workers, triton.cdiv(topk + window_size, block))
](
combined_ragged,
combined_indptr,
topk_indices,
topk_indices.stride(0),
query_start_loc,
seq_lens,
gather_lens,
M,
N,
topk_indices.shape[-1],
TOP_K=topk,
COMPRESS_RATIO=compress_ratio,
WINDOW_SIZE=window_size,
BLOCK_SIZE=block,
)
return combined_ragged, combined_indptr, combined_lens
@dataclass
class DeepseekV4ROCMAiterMLASparseMetadata(FlashMLASparseMetadata):
"""ROCm-specific DeepSeek V4 metadata carrying ragged decode topk."""
c128a_decode_topk_ragged_indices: torch.Tensor | None = None
c128a_decode_topk_ragged_indptr: torch.Tensor | None = None
@dataclass
class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata):
decode_swa_ragged_indices: torch.Tensor | None = None
decode_swa_ragged_indptr: torch.Tensor | None = None
class DeepseekV4ROCMAiterMLASparseMetadataBuilder(FlashMLASparseMetadataBuilder):
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> DeepseekV4ROCMAiterMLASparseMetadata:
base = super().build(
common_prefix_len=common_prefix_len,
common_attn_metadata=common_attn_metadata,
fast_build=fast_build,
)
ragged_indices = None
ragged_indptr = None
dense_decode = base.c128a_global_decode_topk_indices
decode_lens = base.c128a_decode_topk_lens
if dense_decode is not None and decode_lens is not None:
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
dense_decode.reshape(dense_decode.shape[0], -1),
decode_lens,
)
return DeepseekV4ROCMAiterMLASparseMetadata(
**vars(base),
c128a_decode_topk_ragged_indices=ragged_indices,
c128a_decode_topk_ragged_indptr=ragged_indptr,
)
class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> DeepseekV4ROCMAiterSparseSWAMetadata:
base = super().build(
common_prefix_len=common_prefix_len,
common_attn_metadata=common_attn_metadata,
fast_build=fast_build,
)
ragged_indices = None
ragged_indptr = None
if (
base.num_decode_tokens > 0
and base.decode_swa_indices is not None
and base.decode_swa_lens is not None
):
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
base.decode_swa_indices.reshape(base.num_decode_tokens, -1),
base.decode_swa_lens,
)
return DeepseekV4ROCMAiterSparseSWAMetadata(
**vars(base),
decode_swa_ragged_indices=ragged_indices,
decode_swa_ragged_indptr=ragged_indptr,
)
class DeepseekV4ROCMAiterMLASparseImpl(
SparseMLAAttentionImpl[DeepseekV4ROCMAiterMLASparseMetadata]
):
"""ROCm sparse MLA implementation used by DeepSeek V4's custom MLA layer."""
_PREFILL_CHUNK_SIZE = 4
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int,
alibi_slopes: list[float] | None,
sliding_window: int | None,
kv_cache_dtype: str,
logits_soft_cap: float | None,
attn_type: str,
kv_sharing_target_layer_name: str | None,
**_: object,
) -> None:
self.num_heads = num_heads
self.head_size = head_size
self.scale = float(scale)
self.num_kv_heads = num_kv_heads
self.kv_cache_dtype = kv_cache_dtype
def forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
kv_c_and_k_pe_cache: torch.Tensor,
attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata,
layer: AttentionLayer,
) -> tuple[torch.Tensor, torch.Tensor | None]:
raise NotImplementedError(
"DeepseekV4ROCMAiterMLASparseImpl is driven by "
"DeepseekV4MLAAttention.forward."
)
@classmethod
def forward(
cls,
layer: "DeepseekV4MLAAttention",
q: torch.Tensor,
kv: torch.Tensor,
positions: torch.Tensor,
output: torch.Tensor,
) -> None:
assert output.shape == q.shape, (
f"output buffer shape {output.shape} must match q shape {q.shape}"
)
assert output.dtype == q.dtype, (
f"output buffer dtype {output.dtype} must match q dtype {q.dtype}"
)
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
assert isinstance(attn_metadata, dict)
rocm_metadata = cast(
DeepseekV4ROCMAiterMLASparseMetadata | None,
attn_metadata.get(layer.prefix),
)
swa_metadata = cast(
DeepseekV4ROCMAiterSparseSWAMetadata | None,
attn_metadata.get(layer.swa_cache_layer.prefix),
)
assert swa_metadata is not None
swa_only = layer.compress_ratio <= 1
self_kv_cache = layer.kv_cache if not swa_only else None
swa_kv_cache = layer.swa_cache_layer.kv_cache
num_decodes = swa_metadata.num_decodes
num_prefills = swa_metadata.num_prefills
num_decode_tokens = swa_metadata.num_decode_tokens
if num_prefills > 0:
cls._forward_prefill(
layer=layer,
q=q[num_decode_tokens:],
positions=positions[num_decode_tokens:],
compressed_k_cache=self_kv_cache,
swa_k_cache=swa_kv_cache,
output=output[num_decode_tokens:],
attn_metadata=rocm_metadata,
swa_metadata=swa_metadata,
)
if num_decodes > 0:
cls._forward_decode(
layer=layer,
q=q[:num_decode_tokens],
kv_cache=self_kv_cache,
swa_metadata=swa_metadata,
attn_metadata=rocm_metadata,
swa_only=swa_only,
output=output[:num_decode_tokens],
)
@classmethod
def _forward_decode(
cls,
layer: "DeepseekV4MLAAttention",
q: torch.Tensor,
kv_cache: torch.Tensor | None,
swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata,
attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None,
swa_only: bool,
output: torch.Tensor,
) -> None:
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
topk_indices = None
topk_lens = None
topk_ragged_indices = None
topk_ragged_indptr = None
if not swa_only:
assert attn_metadata is not None
assert swa_metadata.is_valid_token is not None
block_size = attn_metadata.block_size // layer.compress_ratio
is_valid = swa_metadata.is_valid_token[:num_decode_tokens]
if layer.compress_ratio == 4:
assert layer.topk_indices_buffer is not None
(
topk_ragged_indices,
topk_ragged_indptr,
topk_lens,
) = compute_global_topk_ragged_indices_and_indptr(
layer.topk_indices_buffer[:num_decode_tokens],
swa_metadata.token_to_req_indices,
attn_metadata.block_table[:num_decodes],
block_size,
is_valid,
)
else:
topk_indices = attn_metadata.c128a_global_decode_topk_indices
topk_lens = attn_metadata.c128a_decode_topk_lens
topk_ragged_indices = attn_metadata.c128a_decode_topk_ragged_indices
topk_ragged_indptr = attn_metadata.c128a_decode_topk_ragged_indptr
rocm_sparse_attn_decode(
q=q,
kv_cache=kv_cache,
swa_k_cache=layer.swa_cache_layer.kv_cache,
swa_only=swa_only,
topk_indices=topk_indices,
topk_lens=topk_lens,
swa_indices=swa_metadata.decode_swa_indices,
swa_lens=swa_metadata.decode_swa_lens,
swa_ragged_indices=swa_metadata.decode_swa_ragged_indices,
swa_ragged_indptr=swa_metadata.decode_swa_ragged_indptr,
topk_ragged_indices=topk_ragged_indices,
topk_ragged_indptr=topk_ragged_indptr,
attn_sink=layer.attn_sink,
scale=layer.scale,
head_dim=layer.head_dim,
nope_head_dim=layer.nope_head_dim,
rope_head_dim=layer.rope_head_dim,
output=output,
)
@classmethod
def _forward_prefill(
cls,
layer: "DeepseekV4MLAAttention",
q: torch.Tensor,
positions: torch.Tensor,
compressed_k_cache: torch.Tensor | None,
swa_k_cache: torch.Tensor,
output: torch.Tensor,
attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None,
swa_metadata: DeepseekV4ROCMAiterSparseSWAMetadata,
) -> None:
swa_only = attn_metadata is None
num_prefills = swa_metadata.num_prefills
num_prefill_tokens = swa_metadata.num_prefill_tokens
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
seq_lens = swa_metadata.prefill_seq_lens
gather_lens = swa_metadata.prefill_gather_lens
assert seq_lens is not None
assert gather_lens is not None
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
query_start_loc = swa_metadata.query_start_loc
assert query_start_loc_cpu is not None
assert query_start_loc is not None
prefill_token_base = query_start_loc_cpu[num_decodes]
if not swa_only:
if layer.compress_ratio == 4:
assert layer.topk_indices_buffer is not None
topk_indices = layer.topk_indices_buffer[num_decode_tokens:]
topk_indices = topk_indices[:num_prefill_tokens]
else:
assert attn_metadata is not None
topk_indices = attn_metadata.c128a_prefill_topk_indices
assert topk_indices is not None
top_k = topk_indices.shape[-1]
N = (layer.max_model_len + layer.compress_ratio - 1) // layer.compress_ratio
else:
assert layer.topk_indices_buffer is not None
topk_indices = layer.topk_indices_buffer[num_decode_tokens:]
top_k = 0
N = 0
M = N + layer.window_size + layer.max_num_batched_tokens
num_chunks = (num_prefills + cls._PREFILL_CHUNK_SIZE - 1) // (
cls._PREFILL_CHUNK_SIZE
)
workspace_manager = current_workspace_manager()
kv = workspace_manager.get_simultaneous(
((cls._PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16),
)[0]
for chunk_idx in range(num_chunks):
chunk_start = chunk_idx * cls._PREFILL_CHUNK_SIZE
chunk_end = min(chunk_start + cls._PREFILL_CHUNK_SIZE, num_prefills)
chunk_size = chunk_end - chunk_start
if not swa_only:
assert attn_metadata is not None
assert compressed_k_cache is not None
block_table = attn_metadata.block_table[num_decodes:]
dequantize_and_gather_k_cache(
kv[:chunk_size],
compressed_k_cache,
seq_lens=seq_lens[chunk_start:chunk_end] // layer.compress_ratio,
gather_lens=None,
block_table=block_table[chunk_start:chunk_end],
block_size=attn_metadata.block_size // layer.compress_ratio,
offset=0,
)
swa_block_table = swa_metadata.block_table[num_decodes:]
dequantize_and_gather_k_cache(
kv[:chunk_size],
swa_k_cache,
seq_lens=seq_lens[chunk_start:chunk_end],
gather_lens=gather_lens[chunk_start:chunk_end],
block_table=swa_block_table[chunk_start:chunk_end],
block_size=swa_metadata.block_size,
offset=N,
)
query_start = (
query_start_loc_cpu[num_decodes + chunk_start] - prefill_token_base
)
query_end = (
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
)
combined_ragged_indices, combined_ragged_indptr, combined_lens = (
combine_topk_swa_indices_ragged(
topk_indices[query_start:query_end],
query_start_loc[
num_decodes + chunk_start : num_decodes + chunk_end + 1
],
seq_lens[chunk_start:chunk_end],
gather_lens[chunk_start:chunk_end],
layer.window_size,
layer.compress_ratio,
top_k,
M,
N,
)
)
rocm_sparse_attn_prefill(
q=q[query_start:query_end],
kv=kv.view(-1, 1, q.shape[-1]),
indices=torch.empty(
q[query_start:query_end].shape[0],
1,
0,
dtype=torch.int32,
device=q.device,
),
topk_length=combined_lens,
scale=layer.scale,
head_dim=layer.head_dim,
nope_head_dim=layer.nope_head_dim,
rope_head_dim=layer.rope_head_dim,
attn_sink=layer.attn_sink,
output=output[query_start:query_end],
ragged_indices=combined_ragged_indices,
ragged_indptr=combined_ragged_indptr,
)
class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4FlashMLASparseBackend):
@staticmethod
def get_name() -> str:
return "ROCM_V4_FLASHMLA_SPARSE"
@staticmethod
def get_builder_cls() -> type["DeepseekV4ROCMAiterMLASparseMetadataBuilder"]:
return DeepseekV4ROCMAiterMLASparseMetadataBuilder
@staticmethod
def get_impl_cls() -> type["DeepseekV4ROCMAiterMLASparseImpl"]:
return DeepseekV4ROCMAiterMLASparseImpl
@@ -112,6 +112,12 @@ class DeepseekSparseSWABackend(AttentionBackend):
@staticmethod
def get_builder_cls() -> type["DeepseekSparseSWAMetadataBuilder"]:
if current_platform.is_rocm():
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
)
return DeepseekV4ROCMAiterSparseSWAMetadataBuilder
return DeepseekSparseSWAMetadataBuilder
@staticmethod
+769 -175
View File
@@ -905,50 +905,705 @@ def rocm_inv_rope_einsum(
return torch.einsum("tgd,grd->tgr", o_ref, wo_a_weight)
def rocm_ref_sparse_attn_prefill(
_DSV4_SPARSE_NOPE_DIM = 448
_DSV4_SPARSE_ROPE_DIM = 64
def _validate_dsv4_sparse_dims(
head_dim: int,
nope_head_dim: int,
rope_head_dim: int,
op_name: str,
) -> None:
assert head_dim == nope_head_dim + rope_head_dim, (
f"{op_name} expected head_dim={nope_head_dim + rope_head_dim}, got {head_dim}"
)
assert (
nope_head_dim == _DSV4_SPARSE_NOPE_DIM
and rope_head_dim == _DSV4_SPARSE_ROPE_DIM
), (
f"{op_name} expects {_DSV4_SPARSE_NOPE_DIM} NoPE dims and "
f"{_DSV4_SPARSE_ROPE_DIM} RoPE dims"
)
@triton.jit
def _pack_dense_prefix_to_ragged_kernel(
indices_ptr,
lengths_ptr,
indptr_ptr,
out_ptr,
indices_stride0,
num_rows_limit,
row_width,
BLOCK_SIZE: tl.constexpr,
):
row_idx = tl.program_id(0)
block_idx = tl.program_id(1)
offsets = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
row_len = tl.load(lengths_ptr + row_idx)
if block_idx * BLOCK_SIZE >= row_len:
return
mask = offsets < row_len
vals = tl.load(
indices_ptr + row_idx * indices_stride0 + offsets,
mask=mask & (offsets < row_width),
other=-1,
).to(tl.int32)
if num_rows_limit >= 0:
vals = tl.where((vals >= 0) & (vals < num_rows_limit), vals, -1)
out_start = tl.load(indptr_ptr + row_idx)
tl.store(out_ptr + out_start + offsets, vals, mask=mask)
def build_ragged_indices_from_dense(
indices: torch.Tensor,
lengths: torch.Tensor,
num_rows: int = -1,
) -> tuple[torch.Tensor, torch.Tensor]:
indices = indices.reshape(indices.shape[0], -1)
lengths = lengths.to(device=indices.device, dtype=torch.int32).reshape(-1)
assert lengths.numel() == indices.shape[0], (
f"Expected one length per row, got {lengths.shape} for indices {indices.shape}"
)
max_width = indices.shape[1] if indices.ndim == 2 else 0
lengths = lengths.clamp(min=0, max=max_width).contiguous()
indptr = torch.empty(indices.shape[0] + 1, dtype=torch.int32, device=indices.device)
indptr[0] = 0
torch.cumsum(lengths, dim=0, out=indptr[1:])
if indices.numel() == 0:
flat = torch.empty(0, dtype=torch.int32, device=indices.device)
else:
flat = torch.empty(
int(indptr[-1].item()), dtype=torch.int32, device=indices.device
)
if flat.numel() > 0:
block_size = 128
_pack_dense_prefix_to_ragged_kernel[
(indices.shape[0], triton.cdiv(max_width, block_size))
](
indices,
lengths,
indptr,
flat,
indices.stride(0),
int(num_rows),
max_width,
BLOCK_SIZE=block_size,
)
return flat, indptr
def _as_int32_contiguous_1d(x: torch.Tensor) -> torch.Tensor:
if x.dtype == torch.int32 and x.ndim == 1 and x.is_contiguous():
return x
return x.to(torch.int32).contiguous()
@triton.jit
def _sparse_attn_prefill_ragged_kernel(
q_ptr,
kv_ptr,
kv_indices_ptr,
kv_indptr_ptr,
attn_sink_ptr,
out_ptr,
q_stride_t,
q_stride_h,
q_stride_d,
kv_stride_n,
kv_stride_d,
out_stride_t,
out_stride_h,
out_stride_d,
num_heads,
head_dim,
num_kv,
scale,
HAS_ATTN_SINK: tl.constexpr,
BLOCK_H: tl.constexpr,
BLOCK_D: tl.constexpr,
BLOCK_K: tl.constexpr,
):
query_idx = tl.program_id(0)
pid_h = tl.program_id(1)
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
dim_offsets = tl.arange(0, BLOCK_D)
head_mask = head_offsets < num_heads
dim_mask = dim_offsets < head_dim
q = tl.load(
q_ptr
+ query_idx * q_stride_t
+ head_offsets[:, None] * q_stride_h
+ dim_offsets[None, :] * q_stride_d,
mask=head_mask[:, None] & dim_mask[None, :],
other=0.0,
)
neg_large = -3.4028234663852886e38
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32)
kv_start = tl.load(kv_indptr_ptr + query_idx)
kv_end = tl.load(kv_indptr_ptr + query_idx + 1)
kv_len = kv_end - kv_start
k_offsets = tl.arange(0, BLOCK_K)
for k_start in tl.range(0, kv_len, BLOCK_K):
k_pos = k_start + k_offsets
in_range = k_pos < kv_len
slot = tl.load(kv_indices_ptr + kv_start + k_pos, mask=in_range, other=-1)
valid = in_range & (slot >= 0) & (slot < num_kv)
kv = tl.load(
kv_ptr + slot[:, None] * kv_stride_n + dim_offsets[None, :] * kv_stride_d,
mask=valid[:, None] & dim_mask[None, :],
other=0.0,
)
kv = tl.where(valid[:, None] & dim_mask[None, :], kv, 0.0)
scores = tl.dot(q, tl.trans(kv)) * scale
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
m_block = tl.max(scores, axis=1)
m_new = tl.maximum(m_i, m_block)
alpha = tl.exp(m_i - m_new)
p = tl.exp(scores - m_new[:, None])
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
l_new = l_i * alpha + tl.sum(p, axis=1)
acc = acc * alpha[:, None] + tl.dot(p.to(kv.dtype), kv)
m_i = m_new
l_i = l_new
if HAS_ATTN_SINK:
sink = tl.load(
attn_sink_ptr + head_offsets, mask=head_mask, other=neg_large
).to(tl.float32)
m_final = tl.maximum(m_i, sink)
alpha = tl.exp(m_i - m_final)
l_final = l_i * alpha + tl.exp(sink - m_final)
denom = tl.maximum(l_final, 1.0e-30)
out = tl.where(
l_final[:, None] > 0.0,
(acc * alpha[:, None]) / denom[:, None],
0.0,
)
else:
denom = tl.maximum(l_i, 1.0e-30)
out = tl.where(l_i[:, None] > 0.0, acc / denom[:, None], 0.0)
tl.store(
out_ptr
+ query_idx * out_stride_t
+ head_offsets[:, None] * out_stride_h
+ dim_offsets[None, :] * out_stride_d,
out,
mask=head_mask[:, None] & dim_mask[None, :],
)
@triton.jit
def _sparse_attn_decode_ragged_kernel(
q_ptr,
main_cache_ptr,
main_indices_ptr,
main_indptr_ptr,
extra_cache_ptr,
extra_indices_ptr,
extra_indptr_ptr,
attn_sink_ptr,
out_ptr,
q_stride0,
q_stride1,
out_stride0,
out_stride1,
main_cache_stride0,
extra_cache_stride0,
main_num_rows,
extra_num_rows,
main_block_size,
extra_block_size,
scale,
num_heads,
HAS_ATTN_SINK: tl.constexpr,
HAS_EXTRA: tl.constexpr,
NOPE_DIM: tl.constexpr,
NOPE_BLOCK: tl.constexpr,
ROPE_DIM: tl.constexpr,
IS_FNUZ: tl.constexpr,
BLOCK_H: tl.constexpr,
BLOCK_K: tl.constexpr,
):
query_idx = tl.program_id(0)
pid_h = tl.program_id(1)
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
head_mask = head_offsets < num_heads
nope_offsets = tl.arange(0, NOPE_BLOCK)
nope_mask = nope_offsets < NOPE_DIM
rope_offsets = tl.arange(0, ROPE_DIM)
q_row_ptr = q_ptr + query_idx * q_stride0 + head_offsets[:, None] * q_stride1
q_nope = tl.load(
q_row_ptr + nope_offsets[None, :],
mask=head_mask[:, None] & nope_mask[None, :],
other=0.0,
)
q_rope = tl.load(
q_row_ptr + NOPE_DIM + rope_offsets[None, :],
mask=head_mask[:, None],
other=0.0,
)
neg_large = -3.4028234663852886e38
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
acc_nope = tl.zeros((BLOCK_H, NOPE_BLOCK), dtype=tl.float32)
acc_rope = tl.zeros((BLOCK_H, ROPE_DIM), dtype=tl.float32)
k_offsets = tl.arange(0, BLOCK_K)
main_start = tl.load(main_indptr_ptr + query_idx)
main_end = tl.load(main_indptr_ptr + query_idx + 1)
main_len = main_end - main_start
zero_nope = tl.zeros((BLOCK_K, NOPE_BLOCK), dtype=tl.bfloat16)
zero_rope = tl.zeros((BLOCK_K, ROPE_DIM), dtype=tl.bfloat16)
for k_start in tl.range(0, main_len, BLOCK_K):
k_pos = k_start + k_offsets
in_range = k_pos < main_len
slot = tl.load(main_indices_ptr + main_start + k_pos, mask=in_range, other=-1)
valid = in_range & (slot >= 0) & (slot < main_num_rows)
safe_slot = tl.where(valid, slot, 0)
block_idx = safe_slot // main_block_size
pos_in_block = safe_slot % main_block_size
cache_block_ptr = main_cache_ptr + block_idx.to(tl.int64) * main_cache_stride0
token_data_ptr = cache_block_ptr + pos_in_block * 576
token_scale_ptr = cache_block_ptr + main_block_size * 576 + pos_in_block * 8
x_uint8 = tl.load(
token_data_ptr[:, None] + nope_offsets[None, :],
mask=valid[:, None] & nope_mask[None, :],
other=0,
)
if IS_FNUZ:
x_fp8 = x_uint8.to(tl.float8e4b15, bitcast=True)
else:
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
encoded_scales = tl.load(
token_scale_ptr[:, None] + nope_offsets[None, :] // 64,
mask=valid[:, None] & nope_mask[None, :],
other=127,
)
scales = tl.exp2(encoded_scales.to(tl.float32) - 127.0)
k_nope = x_fp8.to(tl.bfloat16) * scales.to(tl.bfloat16)
k_nope = tl.where(valid[:, None] & nope_mask[None, :], k_nope, zero_nope)
k_nope = tl.where(k_nope == k_nope, k_nope, zero_nope)
rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16))
k_rope = tl.load(
rope_ptr[:, None] + rope_offsets[None, :],
mask=valid[:, None],
other=0.0,
)
k_rope = tl.where(valid[:, None], k_rope, zero_rope)
k_rope = tl.where(k_rope == k_rope, k_rope, zero_rope)
scores = tl.dot(q_nope, tl.trans(k_nope)) + tl.dot(q_rope, tl.trans(k_rope))
scores *= scale
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
m_block = tl.max(scores, axis=1)
m_new = tl.maximum(m_i, m_block)
alpha = tl.exp(m_i - m_new)
p = tl.exp(scores - m_new[:, None])
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
l_new = l_i * alpha + tl.sum(p, axis=1)
acc_nope = acc_nope * alpha[:, None] + tl.dot(p.to(k_nope.dtype), k_nope)
acc_rope = acc_rope * alpha[:, None] + tl.dot(p.to(k_rope.dtype), k_rope)
m_i = m_new
l_i = l_new
if HAS_EXTRA:
extra_start = tl.load(extra_indptr_ptr + query_idx)
extra_end = tl.load(extra_indptr_ptr + query_idx + 1)
extra_len = extra_end - extra_start
for k_start in tl.range(0, extra_len, BLOCK_K):
k_pos = k_start + k_offsets
in_range = k_pos < extra_len
slot = tl.load(
extra_indices_ptr + extra_start + k_pos, mask=in_range, other=-1
)
valid = in_range & (slot >= 0) & (slot < extra_num_rows)
safe_slot = tl.where(valid, slot, 0)
block_idx = safe_slot // extra_block_size
pos_in_block = safe_slot % extra_block_size
cache_block_ptr = (
extra_cache_ptr + block_idx.to(tl.int64) * extra_cache_stride0
)
token_data_ptr = cache_block_ptr + pos_in_block * 576
token_scale_ptr = (
cache_block_ptr + extra_block_size * 576 + pos_in_block * 8
)
x_uint8 = tl.load(
token_data_ptr[:, None] + nope_offsets[None, :],
mask=valid[:, None] & nope_mask[None, :],
other=0,
)
if IS_FNUZ:
x_fp8 = x_uint8.to(tl.float8e4b15, bitcast=True)
else:
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
encoded_scales = tl.load(
token_scale_ptr[:, None] + nope_offsets[None, :] // 64,
mask=valid[:, None] & nope_mask[None, :],
other=127,
)
scales = tl.exp2(encoded_scales.to(tl.float32) - 127.0)
k_nope = x_fp8.to(tl.bfloat16) * scales.to(tl.bfloat16)
k_nope = tl.where(valid[:, None] & nope_mask[None, :], k_nope, zero_nope)
k_nope = tl.where(k_nope == k_nope, k_nope, zero_nope)
rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16))
k_rope = tl.load(
rope_ptr[:, None] + rope_offsets[None, :],
mask=valid[:, None],
other=0.0,
)
k_rope = tl.where(valid[:, None], k_rope, zero_rope)
k_rope = tl.where(k_rope == k_rope, k_rope, zero_rope)
scores = tl.dot(q_nope, tl.trans(k_nope)) + tl.dot(
q_rope,
tl.trans(k_rope),
)
scores *= scale
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
m_block = tl.max(scores, axis=1)
m_new = tl.maximum(m_i, m_block)
alpha = tl.exp(m_i - m_new)
p = tl.exp(scores - m_new[:, None])
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
l_new = l_i * alpha + tl.sum(p, axis=1)
acc_nope = acc_nope * alpha[:, None] + tl.dot(p.to(k_nope.dtype), k_nope)
acc_rope = acc_rope * alpha[:, None] + tl.dot(p.to(k_rope.dtype), k_rope)
m_i = m_new
l_i = l_new
if HAS_ATTN_SINK:
sink = tl.load(
attn_sink_ptr + head_offsets, mask=head_mask, other=neg_large
).to(tl.float32)
m_final = tl.maximum(m_i, sink)
alpha = tl.exp(m_i - m_final)
l_final = l_i * alpha + tl.exp(sink - m_final)
denom = tl.maximum(l_final, 1.0e-30)
out_nope = tl.where(
l_final[:, None] > 0.0,
(acc_nope * alpha[:, None]) / denom[:, None],
0.0,
)
out_rope = tl.where(
l_final[:, None] > 0.0,
(acc_rope * alpha[:, None]) / denom[:, None],
0.0,
)
else:
denom = tl.maximum(l_i, 1.0e-30)
out_nope = tl.where(l_i[:, None] > 0.0, acc_nope / denom[:, None], 0.0)
out_rope = tl.where(l_i[:, None] > 0.0, acc_rope / denom[:, None], 0.0)
out_row_ptr = (
out_ptr + query_idx * out_stride0 + head_offsets[:, None] * out_stride1
)
tl.store(
out_row_ptr + nope_offsets[None, :],
out_nope,
mask=head_mask[:, None] & nope_mask[None, :],
)
tl.store(
out_row_ptr + NOPE_DIM + rope_offsets[None, :],
out_rope,
mask=head_mask[:, None],
)
def _rocm_sparse_attn_prefill_ragged_triton(
q: torch.Tensor,
kv: torch.Tensor,
indices: torch.Tensor,
topk_length: torch.Tensor | None,
indptr: torch.Tensor,
scale: float,
head_dim: int,
attn_sink: torch.Tensor | None,
nope_head_dim: int,
rope_head_dim: int,
) -> torch.Tensor:
indices = indices.clone().squeeze(1)
s_q, h_q, d_qk = q.shape
topk = indices.shape[-1]
s_kv = kv.shape[0]
if topk_length is not None:
mask = torch.arange(topk, device=indices.device).unsqueeze(
0
) >= topk_length.unsqueeze(1)
indices[mask] = -1
invalid_mask = (indices < 0) | (indices >= s_kv)
indices[invalid_mask] = 0
assert q.ndim == 3, f"expected q=[sq,h,d], got {q.shape}"
assert kv.ndim == 2, f"expected kv=[skv,d], got {kv.shape}"
assert indices.ndim == 1, f"expected indices=[nnz], got {indices.shape}"
assert indptr.ndim == 1, f"expected indptr=[sq+1], got {indptr.shape}"
assert q.is_cuda and kv.is_cuda and indices.is_cuda and indptr.is_cuda
qf = q.float()
gathered_kv = kv.index_select(0, indices.flatten()).reshape(s_q, topk, d_qk).float()
scores = qf @ gathered_kv.transpose(1, 2)
scores *= scale
scores[invalid_mask.unsqueeze(1).expand_as(scores)] = float("-inf")
indices = _as_int32_contiguous_1d(indices)
indptr = _as_int32_contiguous_1d(indptr)
has_attn_sink = attn_sink is not None
if attn_sink is None:
attn_sink = torch.empty(1, device=q.device, dtype=torch.float32)
else:
attn_sink = attn_sink.contiguous()
orig_lse = torch.logsumexp(scores, dim=-1)
lse_for_o = orig_lse
if attn_sink is not None:
lse_for_o = torch.logsumexp(
torch.stack(
[orig_lse, attn_sink[:h_q].view(1, h_q).expand_as(orig_lse)],
dim=0,
),
dim=0,
num_queries, num_heads, head_dim = q.shape
assert indptr.numel() == num_queries + 1, (
f"expected indptr shape [{num_queries + 1}], got {indptr.shape}"
)
_validate_dsv4_sparse_dims(
head_dim,
nope_head_dim,
rope_head_dim,
"_rocm_sparse_attn_prefill_ragged_triton",
)
block_h = 16
block_d = triton.next_power_of_2(head_dim)
block_k = 16 if head_dim >= 256 else 32
out = torch.empty_like(q, dtype=torch.bfloat16)
_sparse_attn_prefill_ragged_kernel[(num_queries, triton.cdiv(num_heads, block_h))](
q,
kv,
indices,
indptr,
attn_sink,
out,
q.stride(0),
q.stride(1),
q.stride(2),
kv.stride(0),
kv.stride(1),
out.stride(0),
out.stride(1),
out.stride(2),
num_heads,
head_dim,
kv.shape[0],
float(scale),
HAS_ATTN_SINK=has_attn_sink,
BLOCK_H=block_h,
BLOCK_D=block_d,
BLOCK_K=block_k,
num_warps=8,
)
return out
def _rocm_sparse_attn_prefill_triton(
q: torch.Tensor,
kv: torch.Tensor,
indices: torch.Tensor,
scale: float,
attn_sink: torch.Tensor | None,
nope_head_dim: int,
rope_head_dim: int,
topk_length: torch.Tensor | None = None,
) -> torch.Tensor:
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
indices,
topk_length
if topk_length is not None
else (indices >= 0).sum(dim=-1, dtype=torch.int32),
num_rows=kv.shape[0],
)
return _rocm_sparse_attn_prefill_ragged_triton(
q=q,
kv=kv,
indices=ragged_indices,
indptr=ragged_indptr,
scale=scale,
attn_sink=attn_sink,
nope_head_dim=nope_head_dim,
rope_head_dim=rope_head_dim,
)
def _rocm_sparse_attn_decode_ragged_triton(
q: torch.Tensor,
main_cache: torch.Tensor,
main_indices: torch.Tensor,
main_indptr: torch.Tensor,
scale: float,
attn_sink: torch.Tensor | None,
nope_head_dim: int,
rope_head_dim: int,
extra_cache: torch.Tensor | None = None,
extra_indices: torch.Tensor | None = None,
extra_indptr: torch.Tensor | None = None,
) -> torch.Tensor:
assert q.ndim == 3, f"expected q=[b,h,d], got {q.shape}"
assert main_cache.ndim == 3, (
f"expected main_cache=[blocks,block,bytes], got {main_cache.shape}"
)
assert main_indices.ndim == 1, (
f"expected main_indices=[nnz], got {main_indices.shape}"
)
assert main_indptr.ndim == 1, f"expected main_indptr=[b+1], got {main_indptr.shape}"
assert (
q.is_cuda
and main_cache.is_cuda
and main_indices.is_cuda
and main_indptr.is_cuda
)
main_indices = _as_int32_contiguous_1d(main_indices)
main_indptr = _as_int32_contiguous_1d(main_indptr)
has_attn_sink = attn_sink is not None
if attn_sink is None:
attn_sink = torch.empty(1, device=q.device, dtype=torch.float32)
else:
attn_sink = attn_sink.contiguous()
num_queries, num_heads, head_dim = q.shape
assert main_indptr.numel() == num_queries + 1, (
f"expected main_indptr shape [{num_queries + 1}], got {main_indptr.shape}"
)
_validate_dsv4_sparse_dims(
head_dim,
nope_head_dim,
rope_head_dim,
"_rocm_sparse_attn_decode_ragged_triton",
)
has_extra = (
extra_cache is not None
and extra_indices is not None
and extra_indptr is not None
)
if has_extra:
assert extra_cache is not None
assert extra_indices is not None
assert extra_indptr is not None
assert extra_indices.ndim == 1, (
f"expected extra_indices=[nnz], got {extra_indices.shape}"
)
lse_for_o = lse_for_o.clone()
lse_for_o[lse_for_o == float("-inf")] = float("+inf")
probs = torch.exp(scores - lse_for_o.unsqueeze(-1))
out = probs @ gathered_kv[..., :head_dim]
lonely_q_mask = orig_lse == float("-inf")
out[lonely_q_mask.unsqueeze(-1).expand_as(out)] = 0.0
return out.to(torch.bfloat16)
assert extra_indptr.ndim == 1, (
f"expected extra_indptr=[b+1], got {extra_indptr.shape}"
)
extra_indices = _as_int32_contiguous_1d(extra_indices)
extra_indptr = _as_int32_contiguous_1d(extra_indptr)
assert extra_indptr.numel() == num_queries + 1, (
f"expected extra_indptr shape [{num_queries + 1}], got {extra_indptr.shape}"
)
else:
extra_cache = main_cache
extra_indices = torch.empty(0, device=q.device, dtype=torch.int32)
extra_indptr = torch.zeros(num_queries + 1, device=q.device, dtype=torch.int32)
block_h = 16
block_k = 16 if head_dim >= 256 else 32
out = torch.empty_like(q, dtype=torch.bfloat16)
_sparse_attn_decode_ragged_kernel[(num_queries, triton.cdiv(num_heads, block_h))](
q,
main_cache,
main_indices,
main_indptr,
extra_cache,
extra_indices,
extra_indptr,
attn_sink,
out,
q.stride(0),
q.stride(1),
out.stride(0),
out.stride(1),
main_cache.stride(0),
extra_cache.stride(0),
main_cache.shape[0] * main_cache.shape[1],
extra_cache.shape[0] * extra_cache.shape[1],
main_cache.shape[1],
extra_cache.shape[1],
scale,
num_heads,
HAS_ATTN_SINK=has_attn_sink,
HAS_EXTRA=has_extra,
NOPE_DIM=nope_head_dim,
NOPE_BLOCK=triton.next_power_of_2(nope_head_dim),
ROPE_DIM=rope_head_dim,
IS_FNUZ=current_platform.is_fp8_fnuz(),
BLOCK_H=block_h,
BLOCK_K=block_k,
num_warps=8,
)
return out
def _rocm_sparse_attn_decode_triton(
q: torch.Tensor,
main_cache: torch.Tensor,
main_indices: torch.Tensor,
scale: float,
attn_sink: torch.Tensor | None,
nope_head_dim: int,
rope_head_dim: int,
extra_cache: torch.Tensor | None = None,
extra_indices: torch.Tensor | None = None,
main_lengths: torch.Tensor | None = None,
extra_lengths: torch.Tensor | None = None,
main_ragged_indices: torch.Tensor | None = None,
main_ragged_indptr: torch.Tensor | None = None,
extra_ragged_indices: torch.Tensor | None = None,
extra_ragged_indptr: torch.Tensor | None = None,
) -> torch.Tensor:
if main_ragged_indices is None or main_ragged_indptr is None:
main_ragged_indices, main_ragged_indptr = build_ragged_indices_from_dense(
main_indices,
main_lengths
if main_lengths is not None
else (main_indices >= 0).sum(dim=-1, dtype=torch.int32),
num_rows=main_cache.shape[0] * main_cache.shape[1],
)
if (
(extra_ragged_indices is None or extra_ragged_indptr is None)
and extra_cache is not None
and extra_indices is not None
):
extra_ragged_indices, extra_ragged_indptr = build_ragged_indices_from_dense(
extra_indices,
extra_lengths
if extra_lengths is not None
else (extra_indices >= 0).sum(dim=-1, dtype=torch.int32),
num_rows=extra_cache.shape[0] * extra_cache.shape[1],
)
return _rocm_sparse_attn_decode_ragged_triton(
q=q,
main_cache=main_cache,
main_indices=main_ragged_indices,
main_indptr=main_ragged_indptr,
scale=scale,
attn_sink=attn_sink,
nope_head_dim=nope_head_dim,
rope_head_dim=rope_head_dim,
extra_cache=extra_cache,
extra_indices=extra_ragged_indices,
extra_indptr=extra_ragged_indptr,
)
def rocm_sparse_attn_prefill(
@@ -958,132 +1613,49 @@ def rocm_sparse_attn_prefill(
topk_length: torch.Tensor | None,
scale: float,
head_dim: int,
nope_head_dim: int,
rope_head_dim: int,
attn_sink: torch.Tensor | None,
output: torch.Tensor,
ragged_indices: torch.Tensor | None = None,
ragged_indptr: torch.Tensor | None = None,
) -> None:
output_chunk = rocm_ref_sparse_attn_prefill(
q=q,
kv=kv,
indices=indices,
topk_length=topk_length,
scale=scale,
head_dim=head_dim,
attn_sink=attn_sink,
assert kv.ndim == 3 and kv.shape[1] == 1, (
f"ROCm Triton sparse prefill expects kv=[skv,1,d], got {kv.shape}"
)
_validate_dsv4_sparse_dims(
head_dim,
nope_head_dim,
rope_head_dim,
"rocm_sparse_attn_prefill",
)
if ragged_indices is not None and ragged_indptr is not None:
output_chunk = _rocm_sparse_attn_prefill_ragged_triton(
q=q,
kv=kv.squeeze(1),
indices=ragged_indices,
indptr=ragged_indptr,
scale=scale,
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
nope_head_dim=nope_head_dim,
rope_head_dim=rope_head_dim,
)
else:
indices_2d = indices.reshape(indices.shape[0], -1)
output_chunk = _rocm_sparse_attn_prefill_triton(
q=q,
kv=kv.squeeze(1),
indices=indices_2d,
scale=scale,
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
nope_head_dim=nope_head_dim,
rope_head_dim=rope_head_dim,
topk_length=topk_length,
)
output.copy_(output_chunk.to(output.dtype))
def rocm_dequantize_blocked_k_cache(
quant_k_cache: torch.Tensor,
head_dim: int,
nope_head_dim: int,
rope_head_dim: int,
) -> torch.Tensor:
fp8_dtype = current_platform.fp8_dtype()
tile_size = 64
num_tiles = nope_head_dim // tile_size
num_blocks, block_size, _ = quant_k_cache.shape
quant_k_cache = quant_k_cache.view(num_blocks, -1)
input_nope_rope = quant_k_cache[
:, : block_size * (nope_head_dim + 2 * rope_head_dim)
].view(num_blocks, block_size, nope_head_dim + 2 * rope_head_dim)
input_nope = input_nope_rope[:, :, :nope_head_dim].view(fp8_dtype)
input_rope = input_nope_rope[:, :, nope_head_dim:].view(torch.bfloat16)
input_scale = (
quant_k_cache[:, block_size * (nope_head_dim + 2 * rope_head_dim) :]
.view(num_blocks, block_size, 8)[:, :, :num_tiles]
.view(torch.float8_e8m0fnu)
)
result = torch.empty(
(num_blocks, block_size, 1, head_dim),
dtype=torch.bfloat16,
device=quant_k_cache.device,
)
result[..., nope_head_dim:] = input_rope.unsqueeze(2)
for tile_idx in range(num_tiles):
cur_nope = input_nope[
..., tile_idx * tile_size : (tile_idx + 1) * tile_size
].to(torch.bfloat16)
cur_scales = input_scale[:, :, tile_idx].to(torch.bfloat16).unsqueeze(-1)
result[..., tile_idx * tile_size : (tile_idx + 1) * tile_size] = (
cur_nope * cur_scales
).unsqueeze(2)
return result
def rocm_ref_sparse_attn_decode(
q: torch.Tensor,
blocked_k: torch.Tensor,
indices_in_kvcache: torch.Tensor,
topk_length: torch.Tensor | None,
scale: float,
head_dim: int,
attn_sink: torch.Tensor | None,
extra_blocked_k: torch.Tensor | None = None,
extra_indices_in_kvcache: torch.Tensor | None = None,
extra_topk_length: torch.Tensor | None = None,
) -> torch.Tensor:
b, s_q, h_q, d_qk = q.shape
def process_scope(
cur_blocked_k: torch.Tensor,
cur_indices: torch.Tensor,
cur_topk_length: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
cur_indices = cur_indices.reshape(b, s_q, -1)
topk = cur_indices.size(-1)
fixed_indices = torch.clamp_min(cur_indices, 0)
gathered_kv = (
cur_blocked_k.view(-1, d_qk)
.index_select(0, fixed_indices.view(-1))
.view(b, s_q, topk, d_qk)
)
invalid_mask = cur_indices == -1
if cur_topk_length is not None:
cur_topk_length = cur_topk_length.reshape(b)
invalid_mask |= torch.arange(0, topk, device=invalid_mask.device).view(
1, 1, topk
) >= cur_topk_length.view(b, 1, 1)
return gathered_kv, invalid_mask
gathered_kv, invalid_mask = process_scope(
blocked_k, indices_in_kvcache, topk_length
)
if extra_blocked_k is not None:
assert extra_indices_in_kvcache is not None
gathered_kv1, invalid_mask1 = process_scope(
extra_blocked_k, extra_indices_in_kvcache, extra_topk_length
)
gathered_kv = torch.cat([gathered_kv, gathered_kv1], dim=2)
invalid_mask = torch.cat([invalid_mask, invalid_mask1], dim=2)
gathered_kv = gathered_kv.view(b * s_q, -1, d_qk).float()
gathered_kv[gathered_kv != gathered_kv] = 0.0
qf = q.float().view(b * s_q, h_q, d_qk)
attn_weight = qf @ gathered_kv.transpose(-1, -2)
attn_weight *= scale
attn_weight[
invalid_mask.view(b * s_q, 1, -1).expand(b * s_q, h_q, invalid_mask.size(-1))
] = float("-inf")
lse = attn_weight.logsumexp(dim=-1)
attn_weight = torch.exp(attn_weight - lse.unsqueeze(-1))
output = attn_weight @ gathered_kv[..., :head_dim]
output = output.view(b, s_q, h_q, head_dim)
lse = lse.view(b, s_q, h_q)
if attn_sink is not None:
output *= (1.0 / (1.0 + torch.exp(attn_sink.view(1, 1, h_q) - lse))).unsqueeze(
-1
)
lonely_q_mask = lse == float("-inf")
output[lonely_q_mask.unsqueeze(-1).expand_as(output)] = 0.0
return output.squeeze(1).to(torch.bfloat16)
def rocm_forward_decode_fallback(
def rocm_sparse_attn_decode(
q: torch.Tensor,
kv_cache: torch.Tensor | None,
swa_k_cache: torch.Tensor,
@@ -1092,6 +1664,10 @@ def rocm_forward_decode_fallback(
topk_lens: torch.Tensor | None,
swa_indices: torch.Tensor,
swa_lens: torch.Tensor,
swa_ragged_indices: torch.Tensor | None,
swa_ragged_indptr: torch.Tensor | None,
topk_ragged_indices: torch.Tensor | None,
topk_ragged_indptr: torch.Tensor | None,
attn_sink: torch.Tensor | None,
scale: float,
head_dim: int,
@@ -1099,31 +1675,49 @@ def rocm_forward_decode_fallback(
rope_head_dim: int,
output: torch.Tensor,
) -> None:
blocked_swa = rocm_dequantize_blocked_k_cache(
swa_k_cache,
head_dim=head_dim,
nope_head_dim=nope_head_dim,
rope_head_dim=rope_head_dim,
assert swa_k_cache.dtype == torch.uint8, (
"ROCm Triton sparse decode expects uint8 fp8_ds_mla SWA cache, "
f"got {swa_k_cache.dtype}"
)
blocked_extra = None
_validate_dsv4_sparse_dims(
head_dim,
nope_head_dim,
rope_head_dim,
"rocm_sparse_attn_decode",
)
main_indices = swa_indices.reshape(swa_indices.shape[0], -1)
extra_cache = None
extra_indices = None
if not swa_only:
assert kv_cache is not None
blocked_extra = rocm_dequantize_blocked_k_cache(
kv_cache,
head_dim=head_dim,
nope_head_dim=nope_head_dim,
rope_head_dim=rope_head_dim,
assert topk_indices is not None or (
topk_ragged_indices is not None and topk_ragged_indptr is not None
)
attn_out = rocm_ref_sparse_attn_decode(
q=q.unsqueeze(1),
blocked_k=blocked_swa,
indices_in_kvcache=swa_indices.unsqueeze(1),
topk_length=swa_lens,
assert kv_cache.dtype == torch.uint8, (
"ROCm Triton sparse decode expects uint8 fp8_ds_mla extra cache, "
f"got {kv_cache.dtype}"
)
extra_cache = kv_cache
if topk_indices is not None:
extra_indices = topk_indices.reshape(topk_indices.shape[0], -1)
attn_out = _rocm_sparse_attn_decode_triton(
q=q,
main_cache=swa_k_cache,
main_indices=main_indices,
scale=scale,
head_dim=head_dim,
attn_sink=attn_sink[: q.shape[1]] if attn_sink is not None else None,
extra_blocked_k=blocked_extra,
extra_indices_in_kvcache=topk_indices,
extra_topk_length=topk_lens,
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
nope_head_dim=nope_head_dim,
rope_head_dim=rope_head_dim,
extra_cache=extra_cache,
extra_indices=extra_indices,
main_lengths=swa_lens,
extra_lengths=topk_lens,
main_ragged_indices=swa_ragged_indices,
main_ragged_indptr=swa_ragged_indptr,
extra_ragged_indices=topk_ragged_indices,
extra_ragged_indptr=topk_ragged_indptr,
)
output.copy_(attn_out.to(output.dtype))