[ROCm][DSv4] implement flash sparse mla with triton kernels (#41812)
Signed-off-by: whx-sjtu <xiaowang990929@gmail.com>
This commit is contained in:
@@ -0,0 +1,377 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="Only used by ROCm"
|
||||
)
|
||||
|
||||
NOPE_HEAD_DIM = 448
|
||||
ROPE_HEAD_DIM = 64
|
||||
HEAD_DIM = NOPE_HEAD_DIM + ROPE_HEAD_DIM
|
||||
|
||||
|
||||
def _ref_global_topk_ragged(
|
||||
topk_indices: torch.Tensor,
|
||||
token_to_req_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
is_valid_token: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
topk = topk_indices.reshape(topk_indices.shape[0], -1)
|
||||
valid = (topk >= 0) & is_valid_token[:, None]
|
||||
lens = valid.sum(dim=1, dtype=torch.int32)
|
||||
indptr = torch.zeros(lens.shape[0] + 1, dtype=torch.int32, device=topk.device)
|
||||
torch.cumsum(lens, dim=0, out=indptr[1:])
|
||||
|
||||
safe_topk = torch.clamp(topk, min=0)
|
||||
block_indices = safe_topk // block_size
|
||||
block_offsets = safe_topk % block_size
|
||||
req_indices = token_to_req_indices[:, None].expand_as(topk)
|
||||
slot_ids = block_table[req_indices, block_indices] * block_size + block_offsets
|
||||
|
||||
offsets = torch.arange(topk.shape[1], dtype=torch.int32, device=topk.device)
|
||||
positions = indptr[:-1, None] + offsets[None, :]
|
||||
return slot_ids[valid], positions[valid].to(torch.long), indptr, lens
|
||||
|
||||
|
||||
def _ref_sparse_prefill_ragged(
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
rows: list[list[int]],
|
||||
scale: float,
|
||||
attn_sink: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
q_f32 = q.float()
|
||||
kv_f32 = kv.float()
|
||||
out = torch.empty_like(q_f32)
|
||||
|
||||
for query_idx in range(q.shape[0]):
|
||||
row_indices = rows[query_idx]
|
||||
for head_idx in range(q.shape[1]):
|
||||
if row_indices:
|
||||
selected_kv = kv_f32[row_indices]
|
||||
scores = torch.mv(selected_kv, q_f32[query_idx, head_idx]) * scale
|
||||
if attn_sink is not None:
|
||||
scores_with_sink = torch.cat(
|
||||
[scores, attn_sink[head_idx].float().reshape(1)]
|
||||
)
|
||||
probs = torch.softmax(scores_with_sink, dim=0)[:-1]
|
||||
else:
|
||||
probs = torch.softmax(scores, dim=0)
|
||||
out[query_idx, head_idx] = torch.sum(
|
||||
probs[:, None] * selected_kv, dim=0
|
||||
)
|
||||
else:
|
||||
out[query_idx, head_idx] = 0
|
||||
return out.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _pack_fp8_ds_mla_cache(kv: torch.Tensor, block_size: int) -> torch.Tensor:
|
||||
assert kv.shape[-1] == HEAD_DIM
|
||||
num_tokens = kv.shape[0]
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size
|
||||
cache = torch.zeros(
|
||||
(num_blocks, block_size, 584),
|
||||
dtype=torch.uint8,
|
||||
device=kv.device,
|
||||
)
|
||||
cache_flat = cache.view(torch.uint8).flatten()
|
||||
kv_nope_fp8 = (
|
||||
kv[:, :NOPE_HEAD_DIM].to(current_platform.fp8_dtype()).view(torch.uint8)
|
||||
)
|
||||
kv_rope_u8 = kv[:, NOPE_HEAD_DIM:].contiguous().view(torch.uint8)
|
||||
|
||||
for slot in range(num_tokens):
|
||||
block_idx = slot // block_size
|
||||
pos = slot % block_size
|
||||
block_base = block_idx * cache.stride(0)
|
||||
token_base = block_base + pos * 576
|
||||
scale_base = block_base + block_size * 576 + pos * 8
|
||||
cache_flat[token_base : token_base + NOPE_HEAD_DIM].copy_(kv_nope_fp8[slot])
|
||||
cache_flat[
|
||||
token_base + NOPE_HEAD_DIM : token_base + NOPE_HEAD_DIM + ROPE_HEAD_DIM * 2
|
||||
].copy_(kv_rope_u8[slot])
|
||||
cache_flat[scale_base : scale_base + 7].fill_(127)
|
||||
return cache
|
||||
|
||||
|
||||
def _read_fp8_ds_mla_cache(
|
||||
cache: torch.Tensor, slot: int, block_size: int
|
||||
) -> torch.Tensor:
|
||||
cache_flat = cache.view(torch.uint8).flatten()
|
||||
block_idx = slot // block_size
|
||||
pos = slot % block_size
|
||||
block_base = block_idx * cache.stride(0)
|
||||
token_base = block_base + pos * 576
|
||||
|
||||
nope_u8 = cache_flat[token_base : token_base + NOPE_HEAD_DIM]
|
||||
nope = nope_u8.view(current_platform.fp8_dtype()).to(torch.float32)
|
||||
rope_u8 = cache_flat[
|
||||
token_base + NOPE_HEAD_DIM : token_base + NOPE_HEAD_DIM + ROPE_HEAD_DIM * 2
|
||||
]
|
||||
rope = rope_u8.view(torch.bfloat16).to(torch.float32)
|
||||
return torch.cat([nope, rope])
|
||||
|
||||
|
||||
def _ref_sparse_decode_ragged(
|
||||
q: torch.Tensor,
|
||||
main_cache: torch.Tensor,
|
||||
main_rows: list[list[int]],
|
||||
scale: float,
|
||||
attn_sink: torch.Tensor | None,
|
||||
block_size: int,
|
||||
extra_cache: torch.Tensor | None = None,
|
||||
extra_rows: list[list[int]] | None = None,
|
||||
) -> torch.Tensor:
|
||||
q_f32 = q.float()
|
||||
out = torch.empty_like(q_f32)
|
||||
|
||||
for query_idx in range(q.shape[0]):
|
||||
row_kv = [
|
||||
_read_fp8_ds_mla_cache(main_cache, int(slot), block_size)
|
||||
for slot in main_rows[query_idx]
|
||||
]
|
||||
if extra_cache is not None and extra_rows is not None:
|
||||
row_kv.extend(
|
||||
_read_fp8_ds_mla_cache(extra_cache, int(slot), block_size)
|
||||
for slot in extra_rows[query_idx]
|
||||
)
|
||||
|
||||
kv = torch.stack(row_kv).to(q.device)
|
||||
for head_idx in range(q.shape[1]):
|
||||
scores = torch.mv(kv, q_f32[query_idx, head_idx]) * scale
|
||||
if attn_sink is not None:
|
||||
scores_with_sink = torch.cat(
|
||||
[scores, attn_sink[head_idx].float().reshape(1)]
|
||||
)
|
||||
probs = torch.softmax(scores_with_sink, dim=0)[:-1]
|
||||
else:
|
||||
probs = torch.softmax(scores, dim=0)
|
||||
out[query_idx, head_idx] = torch.sum(probs[:, None] * kv, dim=0)
|
||||
return out.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _ref_combine_topk_swa_ragged(
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
expected_ragged = torch.tensor(
|
||||
[
|
||||
100,
|
||||
101,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
110,
|
||||
111,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
120,
|
||||
121,
|
||||
122,
|
||||
9,
|
||||
10,
|
||||
11,
|
||||
150,
|
||||
27,
|
||||
28,
|
||||
29,
|
||||
160,
|
||||
161,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
expected_lens = torch.tensor([5, 5, 6, 4, 5], dtype=torch.int32, device=device)
|
||||
expected_indptr = torch.zeros(6, dtype=torch.int32, device=device)
|
||||
torch.cumsum(expected_lens, dim=0, out=expected_indptr[1:])
|
||||
return expected_ragged, expected_indptr, expected_lens
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_compute_global_topk_ragged_indices_and_indptr() -> None:
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
|
||||
compute_global_topk_ragged_indices_and_indptr,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
block_size = 4
|
||||
topk_indices = torch.tensor(
|
||||
[
|
||||
[0, 3, 4, -1],
|
||||
[5, 8, -1, -1],
|
||||
[2, 7, 9, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
token_to_req_indices = torch.tensor([0, 1, 1], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor(
|
||||
[
|
||||
[10, 11, 12],
|
||||
[20, 21, 22],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
is_valid_token = torch.tensor([True, False, True], dtype=torch.bool, device=device)
|
||||
|
||||
actual_ragged, actual_indptr, actual_lens = (
|
||||
compute_global_topk_ragged_indices_and_indptr(
|
||||
topk_indices,
|
||||
token_to_req_indices,
|
||||
block_table,
|
||||
block_size,
|
||||
is_valid_token,
|
||||
)
|
||||
)
|
||||
expected_values, expected_positions, expected_indptr, expected_lens = (
|
||||
_ref_global_topk_ragged(
|
||||
topk_indices,
|
||||
token_to_req_indices,
|
||||
block_table,
|
||||
block_size,
|
||||
is_valid_token,
|
||||
)
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual_ragged[expected_positions], expected_values)
|
||||
torch.testing.assert_close(actual_indptr, expected_indptr)
|
||||
torch.testing.assert_close(actual_lens, expected_lens)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_ragged_kernel() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
q = torch.randn(3, 3, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
kv = torch.randn(5, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
indices = torch.tensor([0, 2, 1, 3, 4], dtype=torch.int32, device=device)
|
||||
indptr = torch.tensor([0, 2, 5, 5], dtype=torch.int32, device=device)
|
||||
attn_sink = torch.tensor([-0.25, 0.0, 0.25], dtype=torch.float32, device=device)
|
||||
scale = HEAD_DIM**-0.5
|
||||
|
||||
actual = _rocm_sparse_attn_prefill_ragged_triton(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
indptr=indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
)
|
||||
expected = _ref_sparse_prefill_ragged(
|
||||
q, kv, [[0, 2], [1, 3, 4], []], scale, attn_sink
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_ragged_kernel() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_decode_ragged_triton,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(1)
|
||||
block_size = 4
|
||||
q = torch.randn(2, 3, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
main_kv = torch.randn(6, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
extra_kv = torch.randn(5, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125
|
||||
main_cache = _pack_fp8_ds_mla_cache(main_kv, block_size)
|
||||
extra_cache = _pack_fp8_ds_mla_cache(extra_kv, block_size)
|
||||
main_indices = torch.tensor([0, 2, 4, 1], dtype=torch.int32, device=device)
|
||||
main_indptr = torch.tensor([0, 2, 4], dtype=torch.int32, device=device)
|
||||
extra_indices = torch.tensor([1, 3, 0], dtype=torch.int32, device=device)
|
||||
extra_indptr = torch.tensor([0, 1, 3], dtype=torch.int32, device=device)
|
||||
attn_sink = torch.tensor([-0.1, 0.0, 0.1], dtype=torch.float32, device=device)
|
||||
scale = HEAD_DIM**-0.5
|
||||
|
||||
actual = _rocm_sparse_attn_decode_ragged_triton(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
main_indices=main_indices,
|
||||
main_indptr=main_indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
extra_cache=extra_cache,
|
||||
extra_indices=extra_indices,
|
||||
extra_indptr=extra_indptr,
|
||||
)
|
||||
expected = _ref_sparse_decode_ragged(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
main_rows=[[0, 2], [4, 1]],
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
block_size=block_size,
|
||||
extra_cache=extra_cache,
|
||||
extra_rows=[[1], [3, 0]],
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_combine_topk_swa_indices_ragged() -> None:
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse_dsv4 import (
|
||||
combine_topk_swa_indices_ragged,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
topk_indices = torch.tensor(
|
||||
[
|
||||
[100, 101, 102, 103],
|
||||
[110, 111, 112, 113],
|
||||
[120, 121, 122, 123],
|
||||
[130, 131, 132, 133],
|
||||
[140, 141, 142, 143],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
query_start_loc = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
|
||||
seq_lens = torch.tensor([6, 4], dtype=torch.int32, device=device)
|
||||
gather_lens = torch.tensor([4, 3], dtype=torch.int32, device=device)
|
||||
window_size = 3
|
||||
compress_ratio = 2
|
||||
topk = 4
|
||||
M = 20
|
||||
N = 8
|
||||
|
||||
actual_ragged, actual_indptr, actual_lens = combine_topk_swa_indices_ragged(
|
||||
topk_indices,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
gather_lens,
|
||||
window_size,
|
||||
compress_ratio,
|
||||
topk,
|
||||
M,
|
||||
N,
|
||||
)
|
||||
expected_ragged, expected_indptr, expected_lens = _ref_combine_topk_swa_ragged(
|
||||
device
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual_ragged[: expected_ragged.numel()], expected_ragged
|
||||
)
|
||||
torch.testing.assert_close(actual_indptr, expected_indptr)
|
||||
torch.testing.assert_close(actual_lens, expected_lens)
|
||||
Reference in New Issue
Block a user