[ROCm][DSV4][Perf] Optimize Triton sparse-MLA decode on gfx950 (#52212)
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Signed-off-by: fai <fangzhouai@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Cursor Grok 4.6 <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Nick Hill
Cursor Grok 4.6
parent
1f0e0bf612
commit
ef43e3101b
@@ -24,12 +24,27 @@ def _on_split_decode_arch() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
try:
|
||||
from vllm.platforms.rocm import _ON_GFX950
|
||||
|
||||
return _ON_GFX950
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
# The flash-decode split-K decode path is only tuned for AMD gfx942/gfx950; other
|
||||
# architectures take the fallback decode kernel, so its tests are skipped there.
|
||||
requires_split_decode_arch = pytest.mark.skipif(
|
||||
not _on_split_decode_arch(),
|
||||
reason="split-K decode kernel is only tuned for AMD gfx942/gfx950",
|
||||
)
|
||||
requires_gfx950 = pytest.mark.skipif(
|
||||
not _on_gfx950(),
|
||||
reason="optimized sparse decode partial is gfx950-only",
|
||||
)
|
||||
|
||||
NOPE_HEAD_DIM = 448
|
||||
ROPE_HEAD_DIM = 64
|
||||
@@ -118,28 +133,48 @@ def _pack_fp8_ds_mla_cache(
|
||||
return cache
|
||||
|
||||
|
||||
def _read_fp8_ds_mla_cache(
|
||||
cache: torch.Tensor, slot: int, block_size: int, use_fnuz: bool
|
||||
) -> torch.Tensor:
|
||||
cache_flat = cache.view(torch.uint8).flatten()
|
||||
def _poison_fp8_ds_mla_cache_row(
|
||||
cache: torch.Tensor, block_size: int, slot: int = 0
|
||||
) -> None:
|
||||
flat = cache.flatten()
|
||||
block_idx = slot // block_size
|
||||
pos = slot % block_size
|
||||
block_base = block_idx * cache.stride(0)
|
||||
token_base = block_base + pos * 576
|
||||
scale_base = block_base + block_size * 576 + pos * 8
|
||||
flat[token_base] = 0x7F
|
||||
flat[scale_base : scale_base + 7] = 255
|
||||
flat[token_base + NOPE_HEAD_DIM : token_base + 576].view(torch.bfloat16)[0] = float(
|
||||
"nan"
|
||||
)
|
||||
|
||||
|
||||
def _read_fp8_ds_mla_cache_rows(
|
||||
cache: torch.Tensor,
|
||||
slots: torch.Tensor,
|
||||
block_size: int,
|
||||
use_fnuz: bool,
|
||||
) -> torch.Tensor:
|
||||
cache_flat = cache.view(torch.uint8).flatten()
|
||||
block_idx = slots // block_size
|
||||
pos = slots % block_size
|
||||
block_base = block_idx * cache.stride(0)
|
||||
token_base = block_base + pos * 576
|
||||
scale_base = block_base + block_size * 576 + pos * 8
|
||||
|
||||
fp8_dtype = torch.float8_e4m3fnuz if use_fnuz else torch.float8_e4m3fn
|
||||
nope_u8 = cache_flat[token_base : token_base + NOPE_HEAD_DIM]
|
||||
nope_offsets = torch.arange(NOPE_HEAD_DIM, device=cache.device)
|
||||
nope_u8 = cache_flat[token_base[:, None] + nope_offsets]
|
||||
nope = nope_u8.view(fp8_dtype).to(torch.float32)
|
||||
scale_offsets = torch.arange(7, device=cache.device)
|
||||
scales = torch.exp2(
|
||||
cache_flat[scale_base : scale_base + 7].to(torch.float32) - 127.0
|
||||
cache_flat[scale_base[:, None] + scale_offsets].to(torch.float32) - 127.0
|
||||
)
|
||||
nope = nope * scales.repeat_interleave(64)
|
||||
rope_u8 = cache_flat[
|
||||
token_base + NOPE_HEAD_DIM : token_base + NOPE_HEAD_DIM + ROPE_HEAD_DIM * 2
|
||||
]
|
||||
rope = rope_u8.view(torch.bfloat16).to(torch.float32)
|
||||
return torch.cat([nope, rope])
|
||||
nope = nope * scales.repeat_interleave(64, dim=1)
|
||||
rope_offsets = torch.arange(ROPE_HEAD_DIM * 2, device=cache.device)
|
||||
rope_u8 = cache_flat[token_base[:, None] + NOPE_HEAD_DIM + rope_offsets]
|
||||
rope = rope_u8.contiguous().view(torch.bfloat16).to(torch.float32)
|
||||
return torch.cat([nope, rope], dim=1)
|
||||
|
||||
|
||||
def _ref_sparse_decode_ragged(
|
||||
@@ -158,19 +193,30 @@ def _ref_sparse_decode_ragged(
|
||||
out = torch.empty_like(q_f32)
|
||||
|
||||
for query_idx in range(q.shape[0]):
|
||||
row_kv = [
|
||||
_read_fp8_ds_mla_cache(main_cache, int(slot), block_size, main_use_fnuz)
|
||||
for slot in main_rows[query_idx]
|
||||
]
|
||||
if extra_cache is not None and extra_rows is not None:
|
||||
row_kv.extend(
|
||||
_read_fp8_ds_mla_cache(
|
||||
extra_cache, int(slot), block_size, extra_use_fnuz
|
||||
row_kv = []
|
||||
if main_rows[query_idx]:
|
||||
main_slots = torch.tensor(
|
||||
main_rows[query_idx], dtype=torch.int64, device=q.device
|
||||
)
|
||||
row_kv.append(
|
||||
_read_fp8_ds_mla_cache_rows(
|
||||
main_cache, main_slots, block_size, main_use_fnuz
|
||||
)
|
||||
)
|
||||
if extra_cache is not None and extra_rows is not None and extra_rows[query_idx]:
|
||||
extra_slots = torch.tensor(
|
||||
extra_rows[query_idx], dtype=torch.int64, device=q.device
|
||||
)
|
||||
row_kv.append(
|
||||
_read_fp8_ds_mla_cache_rows(
|
||||
extra_cache, extra_slots, block_size, extra_use_fnuz
|
||||
)
|
||||
for slot in extra_rows[query_idx]
|
||||
)
|
||||
|
||||
kv = torch.stack(row_kv).to(q.device)
|
||||
if not row_kv:
|
||||
out[query_idx] = 0
|
||||
continue
|
||||
kv = torch.cat(row_kv)
|
||||
for head_idx in range(q.shape[1]):
|
||||
scores = torch.mv(kv, q_f32[query_idx, head_idx]) * scale
|
||||
if attn_sink is not None:
|
||||
@@ -198,6 +244,46 @@ def _ragged_from_rows(
|
||||
)
|
||||
|
||||
|
||||
def _launch_sparse_decode_reduce(
|
||||
part_m: torch.Tensor,
|
||||
part_l: torch.Tensor,
|
||||
part_acc: torch.Tensor,
|
||||
adaptive_splits: bool,
|
||||
) -> torch.Tensor:
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
num_queries, num_splits, num_heads = part_m.shape
|
||||
out = torch.empty(
|
||||
(num_queries, num_heads, HEAD_DIM),
|
||||
dtype=torch.bfloat16,
|
||||
device=part_m.device,
|
||||
)
|
||||
attn_sink = torch.empty(1, dtype=torch.float32, device=part_m.device)
|
||||
mod._sparse_attn_decode_reduce_kernel[(num_queries, num_heads)](
|
||||
part_m,
|
||||
part_l,
|
||||
part_acc,
|
||||
attn_sink,
|
||||
out,
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
part_m.stride(0),
|
||||
part_m.stride(1),
|
||||
part_acc.stride(0),
|
||||
part_acc.stride(1),
|
||||
part_acc.stride(2),
|
||||
num_heads,
|
||||
HAS_ATTN_SINK=False,
|
||||
ADAPTIVE_SPLITS=adaptive_splits,
|
||||
COMB_DIM=HEAD_DIM,
|
||||
BLOCK_H=1,
|
||||
NUM_SPLITS=num_splits,
|
||||
SPLITS_PAD=1 << (num_splits - 1).bit_length(),
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_paged_mqa_logits_do_not_contain_nan(monkeypatch) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
@@ -312,6 +398,19 @@ def test_compute_global_topk_ragged_indices_and_indptr() -> None:
|
||||
torch.testing.assert_close(actual_lens, expected_lens)
|
||||
|
||||
|
||||
def test_extra_cache_nan_free_provenance_gate(monkeypatch) -> None:
|
||||
from vllm.models.deepseek_v4.amd import rocm as mod
|
||||
|
||||
monkeypatch.setattr(mod, "_ON_GFX950", True)
|
||||
assert mod._trust_dsv4_extra_cache_nan_free("fp8_ds_mla", False, True)
|
||||
assert not mod._trust_dsv4_extra_cache_nan_free("fp8_ds_mla", True, True)
|
||||
assert not mod._trust_dsv4_extra_cache_nan_free("bfloat16", False, True)
|
||||
assert not mod._trust_dsv4_extra_cache_nan_free("fp8_ds_mla", False, False)
|
||||
|
||||
monkeypatch.setattr(mod, "_ON_GFX950", False)
|
||||
assert not mod._trust_dsv4_extra_cache_nan_free("fp8_ds_mla", False, True)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_ragged_kernel() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
@@ -366,6 +465,7 @@ def test_sparse_attn_decode_ragged_kernel() -> None:
|
||||
attn_sink = torch.tensor([-0.1, 0.0, 0.1], dtype=torch.float32, device=device)
|
||||
scale = HEAD_DIM**-0.5
|
||||
|
||||
out = torch.empty_like(q)
|
||||
actual = _rocm_sparse_attn_decode_ragged_triton(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
@@ -378,6 +478,7 @@ def test_sparse_attn_decode_ragged_kernel() -> None:
|
||||
extra_cache=extra_cache,
|
||||
extra_indices=extra_indices,
|
||||
extra_indptr=extra_indptr,
|
||||
out=out,
|
||||
)
|
||||
expected = _ref_sparse_decode_ragged(
|
||||
q=q,
|
||||
@@ -391,9 +492,111 @@ def test_sparse_attn_decode_ragged_kernel() -> None:
|
||||
main_use_fnuz=main_use_fnuz,
|
||||
)
|
||||
|
||||
assert actual.data_ptr() == out.data_ptr()
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_scrubs_untrusted_cache_by_default() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_decode_ragged_triton,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
block_size = 4
|
||||
main_cache = torch.zeros(1, block_size, 584, dtype=torch.uint8, device=device)
|
||||
extra_cache = torch.zeros_like(main_cache)
|
||||
_poison_fp8_ds_mla_cache_row(main_cache, block_size)
|
||||
_poison_fp8_ds_mla_cache_row(extra_cache, block_size)
|
||||
indices = torch.zeros(1, dtype=torch.int32, device=device)
|
||||
indptr = torch.tensor([0, 1], dtype=torch.int32, device=device)
|
||||
|
||||
actual = _rocm_sparse_attn_decode_ragged_triton(
|
||||
q=torch.ones(1, 1, HEAD_DIM, dtype=torch.bfloat16, device=device),
|
||||
main_cache=main_cache,
|
||||
main_indices=indices,
|
||||
main_indptr=indptr,
|
||||
scale=HEAD_DIM**-0.5,
|
||||
attn_sink=None,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
extra_cache=extra_cache,
|
||||
extra_indices=indices,
|
||||
extra_indptr=indptr,
|
||||
)
|
||||
|
||||
assert not torch.isnan(actual).any()
|
||||
assert torch.equal(actual, torch.zeros_like(actual))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("on_gfx950", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_ragged_graph_buffer_view_tracks_source_width(
|
||||
monkeypatch, on_gfx950: bool
|
||||
) -> None:
|
||||
from vllm.models.deepseek_v4.amd import rocm as rocm_mod
|
||||
|
||||
monkeypatch.setattr(rocm_mod, "_ON_GFX950", on_gfx950)
|
||||
|
||||
indices_buffer = torch.full((16,), -1, dtype=torch.int32)
|
||||
indptr_buffer = torch.full((3,), -1, dtype=torch.int32)
|
||||
first_indices = torch.tensor([3, 5, 7], dtype=torch.int32)
|
||||
first_indptr = torch.tensor([0, 1, 3], dtype=torch.int32)
|
||||
first_view, first_indptr_view = rocm_mod._copy_ragged_to_graph_buffers(
|
||||
first_indices,
|
||||
first_indptr,
|
||||
indices_buffer,
|
||||
indptr_buffer,
|
||||
num_rows=2,
|
||||
max_entries_per_row=8,
|
||||
)
|
||||
|
||||
second_indices = torch.tensor([1, 2, 3, 4, 5, 6], dtype=torch.int32)
|
||||
second_indptr = torch.tensor([0, 2, 6], dtype=torch.int32)
|
||||
second_view, second_indptr_view = rocm_mod._copy_ragged_to_graph_buffers(
|
||||
second_indices,
|
||||
second_indptr,
|
||||
indices_buffer,
|
||||
indptr_buffer,
|
||||
num_rows=2,
|
||||
max_entries_per_row=8,
|
||||
)
|
||||
|
||||
expected_first_entries = (
|
||||
first_indices.numel() if on_gfx950 else indices_buffer.numel()
|
||||
)
|
||||
expected_second_entries = (
|
||||
second_indices.numel() if on_gfx950 else indices_buffer.numel()
|
||||
)
|
||||
assert first_view.numel() == expected_first_entries
|
||||
assert second_view.numel() == expected_second_entries
|
||||
assert first_view.data_ptr() == second_view.data_ptr() == indices_buffer.data_ptr()
|
||||
assert first_indptr_view.data_ptr() == second_indptr_view.data_ptr()
|
||||
assert torch.equal(second_view[: second_indices.numel()], second_indices)
|
||||
assert torch.equal(second_indptr_view, second_indptr)
|
||||
|
||||
|
||||
def test_rocm_capture_metadata_sets_adaptive_marker(monkeypatch) -> None:
|
||||
from vllm.models.deepseek_v4.amd import rocm as rocm_mod
|
||||
from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4SparseMLAMetadataBuilder,
|
||||
)
|
||||
|
||||
metadata = SimpleNamespace(for_cudagraph_capture=False)
|
||||
monkeypatch.setattr(
|
||||
DeepseekV4SparseMLAMetadataBuilder,
|
||||
"build_for_cudagraph_capture",
|
||||
lambda *_: metadata,
|
||||
)
|
||||
builder = object.__new__(rocm_mod.DeepseekV4ROCMAiterMLASparseMetadataBuilder)
|
||||
|
||||
actual = builder.build_for_cudagraph_capture(SimpleNamespace())
|
||||
|
||||
assert actual is metadata
|
||||
assert actual.for_cudagraph_capture is _on_gfx950()
|
||||
|
||||
|
||||
@requires_split_decode_arch
|
||||
@torch.inference_mode()
|
||||
def test_decode_num_splits_heuristic(monkeypatch) -> None:
|
||||
@@ -408,6 +611,9 @@ def test_decode_num_splits_heuristic(monkeypatch) -> None:
|
||||
# A tiny batch on a large device should split to add parallelism.
|
||||
assert mod._decode_num_splits(2, 1, avg_main_len=256.0, avg_extra_len=0.0) > 1
|
||||
|
||||
# The shared gfx942 selector retains its original 16-split ceiling.
|
||||
assert mod._decode_num_splits(1, 1, 128.0, 8192.0) == 16
|
||||
|
||||
# The chosen count always stays within the searched [1, 16] range, and a
|
||||
# zero-length workload never splits (no work to parallelize).
|
||||
for num_queries in (1, 4, 24, 224, 1024):
|
||||
@@ -418,6 +624,16 @@ def test_decode_num_splits_heuristic(monkeypatch) -> None:
|
||||
assert mod._decode_num_splits(2, 1, avg_main_len=0.0, avg_extra_len=0.0) >= 1
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_decode_num_splits_gfx950(monkeypatch) -> None:
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
monkeypatch.setattr(mod, "_decode_cu_count", lambda: 256)
|
||||
assert mod._decode_gfx950_num_splits(1, 1, 128, 8192) == 32
|
||||
assert mod._decode_gfx950_num_splits(17, 1, 128, 32) == 4
|
||||
assert mod._decode_gfx950_num_splits(512, 1, 128, 7812) == 1
|
||||
|
||||
|
||||
@requires_split_decode_arch
|
||||
@pytest.mark.parametrize("num_splits", [1, 2, 3, 4, 8])
|
||||
@pytest.mark.parametrize("with_extra", [True, False])
|
||||
@@ -473,7 +689,14 @@ def test_sparse_attn_decode_split_k_kernel(
|
||||
scale = HEAD_DIM**-0.5
|
||||
|
||||
# Pin the split count so each parametrized value is exercised deterministically.
|
||||
monkeypatch.setattr(mod, "_decode_num_splits", lambda *args, **kwargs: num_splits)
|
||||
split_fn = "_decode_gfx950_num_splits" if _on_gfx950() else "_decode_num_splits"
|
||||
other_split_fn = (
|
||||
"_decode_num_splits" if _on_gfx950() else "_decode_gfx950_num_splits"
|
||||
)
|
||||
monkeypatch.setattr(mod, split_fn, lambda *args, **kwargs: num_splits)
|
||||
monkeypatch.setattr(
|
||||
mod, other_split_fn, lambda *args, **kwargs: pytest.fail("wrong selector")
|
||||
)
|
||||
|
||||
actual = mod._rocm_sparse_attn_decode_ragged_triton(
|
||||
q=q,
|
||||
@@ -503,6 +726,218 @@ def test_sparse_attn_decode_split_k_kernel(
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_gfx950_adaptive_reduce_ignores_stale_scratch() -> None:
|
||||
device = torch.device("cuda")
|
||||
part_m = torch.full((1, 8, 1), torch.finfo(torch.float32).min, device=device)
|
||||
part_l = torch.zeros_like(part_m)
|
||||
part_acc = torch.full((1, 8, 1, HEAD_DIM), float("nan"), device=device)
|
||||
part_m[:, :2] = 0
|
||||
part_l[:, :2] = 1
|
||||
part_acc[:, 0] = 1
|
||||
part_acc[:, 1] = 3
|
||||
|
||||
actual = _launch_sparse_decode_reduce(part_m, part_l, part_acc, True)
|
||||
|
||||
assert torch.isfinite(actual).all()
|
||||
assert torch.equal(actual, torch.full_like(actual, 2))
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@pytest.mark.parametrize("extra_len", [0, 1, 31, 32, 33, 63, 64, 65])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_gfx950_outer64_boundaries(
|
||||
monkeypatch, extra_len: int
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(13)
|
||||
block_size = 4
|
||||
num_heads = 16
|
||||
num_extra_rows = 80
|
||||
q = torch.randn(2, num_heads, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
q *= 0.125
|
||||
main_cache = torch.zeros(1, block_size, 584, dtype=torch.uint8, device=device)
|
||||
main_indices = torch.empty(0, dtype=torch.int32, device=device)
|
||||
main_indptr = torch.zeros(3, dtype=torch.int32, device=device)
|
||||
extra_cache = _pack_fp8_ds_mla_cache(
|
||||
torch.randn(num_extra_rows, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
* 0.125,
|
||||
block_size,
|
||||
use_fnuz=False,
|
||||
)
|
||||
_poison_fp8_ds_mla_cache_row(extra_cache, block_size)
|
||||
|
||||
raw_row = list(range(1, extra_len + 1))
|
||||
if extra_len > 3:
|
||||
raw_row[3] = -1
|
||||
if extra_len > 40:
|
||||
raw_row[40] = num_extra_rows
|
||||
if extra_len > 64:
|
||||
raw_row[64] = num_extra_rows + 1024
|
||||
extra_indices, extra_indptr = _ragged_from_rows([raw_row, []], device)
|
||||
valid_row = [slot for slot in raw_row if 0 <= slot < num_extra_rows]
|
||||
attn_sink = torch.linspace(-0.1, 0.1, num_heads, dtype=torch.float32, device=device)
|
||||
|
||||
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 1)
|
||||
actual = mod._rocm_sparse_attn_decode_ragged_triton(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
main_indices=main_indices,
|
||||
main_indptr=main_indptr,
|
||||
scale=HEAD_DIM**-0.5,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
extra_cache=extra_cache,
|
||||
extra_indices=extra_indices,
|
||||
extra_indptr=extra_indptr,
|
||||
)
|
||||
expected = _ref_sparse_decode_ragged(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
main_rows=[[], []],
|
||||
scale=HEAD_DIM**-0.5,
|
||||
attn_sink=attn_sink,
|
||||
block_size=block_size,
|
||||
extra_cache=extra_cache,
|
||||
extra_rows=[valid_row, []],
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
assert torch.equal(actual[1], torch.zeros_like(actual[1]))
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_gfx950_graph_replay(monkeypatch) -> None:
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(17)
|
||||
block_size = 64
|
||||
num_queries = 16
|
||||
num_heads = 16
|
||||
num_splits = 8
|
||||
extra_per_query = 65 * num_splits
|
||||
max_extra_per_query = 8192
|
||||
q = (
|
||||
torch.randn(
|
||||
num_queries,
|
||||
num_heads,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125
|
||||
)
|
||||
main_cache = _pack_fp8_ds_mla_cache(
|
||||
torch.randn(num_queries, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125,
|
||||
block_size,
|
||||
use_fnuz=False,
|
||||
)
|
||||
extra_cache = _pack_fp8_ds_mla_cache(
|
||||
torch.randn(
|
||||
num_queries * extra_per_query,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125,
|
||||
block_size,
|
||||
use_fnuz=False,
|
||||
)
|
||||
main_rows = [[query_idx] for query_idx in range(num_queries)]
|
||||
extra_rows = [
|
||||
list(range(query_idx * extra_per_query, (query_idx + 1) * extra_per_query))
|
||||
for query_idx in range(num_queries)
|
||||
]
|
||||
main_indices, main_indptr = _ragged_from_rows(main_rows, device)
|
||||
short_extra_rows = [row[:64] for row in extra_rows]
|
||||
long_indices, long_indptr = _ragged_from_rows(extra_rows, device)
|
||||
short_indices, short_indptr = _ragged_from_rows(short_extra_rows, device)
|
||||
extra_indices = torch.full(
|
||||
(num_queries * max_extra_per_query,),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
extra_indices[: long_indices.numel()].copy_(long_indices)
|
||||
extra_indptr = long_indptr.clone()
|
||||
extra_indices_ptr = extra_indices.data_ptr()
|
||||
attn_sink = torch.linspace(-0.1, 0.1, num_heads, dtype=torch.float32, device=device)
|
||||
out = torch.empty_like(q)
|
||||
|
||||
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: num_splits)
|
||||
|
||||
def run_decode() -> torch.Tensor:
|
||||
return mod._rocm_sparse_attn_decode_ragged_triton(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
main_indices=main_indices,
|
||||
main_indptr=main_indptr,
|
||||
scale=HEAD_DIM**-0.5,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
extra_cache=extra_cache,
|
||||
extra_indices=extra_indices,
|
||||
extra_indptr=extra_indptr,
|
||||
out=out,
|
||||
extra_cache_nan_free=True,
|
||||
adaptive_splits=True,
|
||||
)
|
||||
|
||||
run_decode()
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured_out = run_decode()
|
||||
torch.accelerator.synchronize()
|
||||
captured_long = out.clone()
|
||||
expected_long = _ref_sparse_decode_ragged(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
main_rows=main_rows,
|
||||
scale=HEAD_DIM**-0.5,
|
||||
attn_sink=attn_sink,
|
||||
block_size=block_size,
|
||||
extra_cache=extra_cache,
|
||||
extra_rows=extra_rows,
|
||||
)
|
||||
torch.testing.assert_close(captured_long, expected_long, atol=2e-2, rtol=2e-2)
|
||||
|
||||
extra_indices[: short_indices.numel()].copy_(short_indices)
|
||||
extra_indptr.copy_(short_indptr)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
short_out = out.clone()
|
||||
expected_short = _ref_sparse_decode_ragged(
|
||||
q=q,
|
||||
main_cache=main_cache,
|
||||
main_rows=main_rows,
|
||||
scale=HEAD_DIM**-0.5,
|
||||
attn_sink=attn_sink,
|
||||
block_size=block_size,
|
||||
extra_cache=extra_cache,
|
||||
extra_rows=short_extra_rows,
|
||||
)
|
||||
|
||||
assert captured_out.data_ptr() == out.data_ptr()
|
||||
assert extra_indices.data_ptr() == extra_indices_ptr
|
||||
assert extra_indices.numel() == num_queries * max_extra_per_query
|
||||
assert not torch.equal(short_out, captured_long)
|
||||
torch.testing.assert_close(short_out, expected_short, atol=2e-2, rtol=2e-2)
|
||||
|
||||
extra_indices[: long_indices.numel()].copy_(long_indices)
|
||||
extra_indptr.copy_(long_indptr)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(out, expected_long, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# o-projection: fused inverse-RoPE + cached bf16 wo_a (rocm_inv_rope_einsum)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -28,16 +28,81 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
|
||||
_fused_kv_compress_norm_rope_insert_indexer_attn,
|
||||
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn,
|
||||
_launch_two_stage_sparse_attn_compressor,
|
||||
compress_norm_rope_store_triton,
|
||||
)
|
||||
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
cp_gather_indexer_k_quant_cache_triton,
|
||||
indexer_k_quant_and_cache_triton,
|
||||
)
|
||||
|
||||
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
try:
|
||||
from vllm.platforms.rocm import _ON_GFX950
|
||||
|
||||
return _ON_GFX950
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
|
||||
def test_cp_gather_despecialized_kernel_is_gfx950_only(monkeypatch):
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
class FakeKernel:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def __getitem__(self, grid):
|
||||
def launch(*args):
|
||||
self.calls.append((grid, args))
|
||||
|
||||
return launch
|
||||
|
||||
legacy_kernel = FakeKernel()
|
||||
gfx950_kernel = FakeKernel()
|
||||
monkeypatch.setattr(mod, "_cp_gather_indexer_quant_cache_kernel", legacy_kernel)
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"_cp_gather_indexer_quant_cache_gfx950_kernel",
|
||||
gfx950_kernel,
|
||||
)
|
||||
|
||||
k_cache = torch.zeros((4, 1, 132), dtype=torch.uint8)
|
||||
k_fp8 = torch.empty((5, 128), dtype=current_platform.fp8_dtype())
|
||||
k_scale = torch.empty((5, 4), dtype=torch.uint8)
|
||||
block_table = torch.zeros((2, 7), dtype=torch.int32)
|
||||
cu_seqlen = torch.tensor([0, 2, 5], dtype=torch.int32)
|
||||
token_to_seq = torch.tensor([0, 0, 1, 1, 1], dtype=torch.int32)
|
||||
args = (k_cache, k_fp8, k_scale, block_table, cu_seqlen, token_to_seq)
|
||||
|
||||
monkeypatch.setattr(mod, "_ON_GFX950", True)
|
||||
mod.cp_gather_indexer_k_quant_cache_triton(*args)
|
||||
assert len(gfx950_kernel.calls) == 1
|
||||
assert not legacy_kernel.calls
|
||||
gfx950_grid, gfx950_args = gfx950_kernel.calls[0]
|
||||
assert gfx950_grid == (5,)
|
||||
assert len(gfx950_args) == 18
|
||||
assert gfx950_args[-3:] == (2, 7, 4)
|
||||
|
||||
monkeypatch.setattr(mod, "_ON_GFX950", False)
|
||||
mod.cp_gather_indexer_k_quant_cache_triton(*args)
|
||||
assert len(legacy_kernel.calls) == 1
|
||||
legacy_grid, legacy_args = legacy_kernel.calls[0]
|
||||
assert legacy_grid == (5,)
|
||||
assert len(legacy_args) == 19
|
||||
assert legacy_args[-4:] == (5, 2, 7, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("window_size", "num_speculative_tokens", "expected"),
|
||||
[(128, 5, 192), (512, 5, 576), (1024, 0, 1024)],
|
||||
@@ -91,6 +156,129 @@ def _ue8m0_reference(x: torch.Tensor, block_size: int, fp8_max: float):
|
||||
return x_fp8, scales
|
||||
|
||||
|
||||
def _decode_dsv4_cache_row(
|
||||
cache: torch.Tensor, block_size: int, scrub_nan: bool
|
||||
) -> torch.Tensor:
|
||||
flat = cache.flatten()
|
||||
nope = flat[:448].view(torch.float8_e4m3fn).to(torch.bfloat16)
|
||||
encoded = flat[block_size * 576 : block_size * 576 + 7]
|
||||
scales = torch.exp2(encoded.to(torch.float32) - 127.0).to(torch.bfloat16)
|
||||
nope = nope * scales.repeat_interleave(64)
|
||||
rope = flat[448:576].view(torch.bfloat16)
|
||||
decoded = torch.cat((nope, rope))
|
||||
if scrub_nan:
|
||||
decoded = torch.where(decoded == decoded, decoded, 0.0)
|
||||
return decoded
|
||||
|
||||
|
||||
def _assert_nan_free_cache_matches_legacy_scrub(
|
||||
cache: torch.Tensor, block_size: int
|
||||
) -> None:
|
||||
flat = cache.flatten()
|
||||
scale_base = block_size * 576
|
||||
scale_codes = flat[scale_base : scale_base + 8]
|
||||
assert scale_codes[0].item() == 254
|
||||
assert scale_codes[1].item() == 247
|
||||
assert scale_codes[:7].max().item() <= 254
|
||||
nope_bytes = flat[:448]
|
||||
assert not ((nope_bytes == 0x7F) | (nope_bytes == 0xFF)).any()
|
||||
|
||||
rope = flat[448:576].view(torch.bfloat16)
|
||||
assert not torch.isnan(rope).any()
|
||||
assert torch.isposinf(rope[0])
|
||||
assert torch.equal(rope[1:4], torch.zeros_like(rope[1:4]))
|
||||
|
||||
legacy_cache = cache.clone()
|
||||
legacy_flat = legacy_cache.flatten()
|
||||
legacy_flat[scale_base] = 255
|
||||
legacy_rope = legacy_flat[448:576].view(torch.bfloat16)
|
||||
legacy_rope[1:4] = float("nan")
|
||||
canonical = _decode_dsv4_cache_row(cache, block_size, scrub_nan=False)
|
||||
legacy = _decode_dsv4_cache_row(legacy_cache, block_size, scrub_nan=True)
|
||||
torch.testing.assert_close(canonical, legacy, rtol=0, atol=0)
|
||||
assert torch.isinf(canonical[0])
|
||||
assert torch.isposinf(canonical[64])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _on_gfx950(),
|
||||
reason="NaN-free fp8_ds_mla compressed-cache contract is gfx950-only",
|
||||
)
|
||||
@pytest.mark.parametrize("writer", ["single_pass", "two_stage_finalizer"])
|
||||
def test_gfx950_compressed_cache_canonicalizes_nonfinite(writer: str) -> None:
|
||||
head_dim = 512
|
||||
rope_dim = 64
|
||||
block_size = 4
|
||||
device = "cuda"
|
||||
|
||||
positions = torch.zeros(1, dtype=torch.int64, device=device)
|
||||
slot_mapping = torch.zeros(1, dtype=torch.int64, device=device)
|
||||
rms_weight = torch.ones(head_dim, dtype=torch.bfloat16, device=device)
|
||||
rms_weight[0] = float("inf")
|
||||
rms_weight[64] = torch.finfo(torch.bfloat16).max
|
||||
rms_weight[448] = float("inf")
|
||||
rms_weight[450] = float("nan")
|
||||
cos_sin_cache = torch.zeros(1, rope_dim, dtype=torch.float32, device=device)
|
||||
cos_sin_cache[:, : rope_dim // 2] = 1.0
|
||||
cache = torch.zeros(1, block_size, 584, dtype=torch.uint8, device=device)
|
||||
|
||||
state_cache = torch.zeros(1, 1, 2 * head_dim, dtype=torch.float32, device=device)
|
||||
state_cache[..., :head_dim] = 1.0
|
||||
token_to_req = torch.zeros(1, dtype=torch.int32, device=device)
|
||||
block_table = torch.zeros(1, 1, dtype=torch.int32, device=device)
|
||||
|
||||
if writer == "single_pass":
|
||||
compress_norm_rope_store_triton(
|
||||
state_cache=state_cache,
|
||||
num_actual=1,
|
||||
token_to_req_indices=token_to_req,
|
||||
positions=positions,
|
||||
slot_mapping=slot_mapping,
|
||||
block_table=block_table,
|
||||
block_size=1,
|
||||
state_width=head_dim,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
kv_cache=cache,
|
||||
k_cache_metadata=SimpleNamespace(slot_mapping=slot_mapping),
|
||||
pdl_kwargs={},
|
||||
head_dim=head_dim,
|
||||
rope_head_dim=rope_dim,
|
||||
compress_ratio=1,
|
||||
overlap=False,
|
||||
use_fp4_cache=False,
|
||||
rms_norm_weight=rms_weight,
|
||||
rms_norm_eps=1e-6,
|
||||
quant_block=64,
|
||||
token_stride=576,
|
||||
scale_dim=8,
|
||||
)
|
||||
else:
|
||||
_launch_two_stage_sparse_attn_compressor(
|
||||
state_cache,
|
||||
token_to_req,
|
||||
positions,
|
||||
slot_mapping,
|
||||
block_table,
|
||||
1,
|
||||
head_dim,
|
||||
1,
|
||||
cos_sin_cache,
|
||||
cache,
|
||||
slot_mapping,
|
||||
rms_weight,
|
||||
1e-6,
|
||||
64,
|
||||
576,
|
||||
8,
|
||||
head_dim,
|
||||
rope_dim,
|
||||
1,
|
||||
torch.empty(1, head_dim, dtype=torch.float32, device=device),
|
||||
)
|
||||
|
||||
_assert_nan_free_cache_matches_legacy_scrub(cache, block_size)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("starts", "query_start_loc", "expected"),
|
||||
[
|
||||
@@ -402,7 +590,8 @@ def test_indexer_gather_accepts_upper_bound_output():
|
||||
valid_tokens = 9
|
||||
upper_bound_tokens = 13
|
||||
block_size = 16
|
||||
num_blocks = 2
|
||||
num_seqs = 3
|
||||
num_blocks = num_seqs
|
||||
sentinel = 123
|
||||
device = "cuda"
|
||||
|
||||
@@ -410,13 +599,15 @@ def test_indexer_gather_accepts_upper_bound_output():
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks, block_size, cache_stride, dtype=torch.uint8, device=device
|
||||
)
|
||||
slot_mapping = torch.arange(valid_tokens, dtype=torch.int64, device=device)
|
||||
slot_mapping = torch.tensor(
|
||||
[0, 1, 2, 16, 17, 18, 32, 33, 34], dtype=torch.int64, device=device
|
||||
)
|
||||
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, quant_block_size, "ue8m0")
|
||||
|
||||
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(
|
||||
0
|
||||
1
|
||||
)
|
||||
cu_seq_lens = torch.tensor([0, valid_tokens], dtype=torch.int32, device=device)
|
||||
cu_seq_lens = torch.tensor([0, 3, 6, 9], dtype=torch.int32, device=device)
|
||||
dst_k = torch.full(
|
||||
(upper_bound_tokens, head_dim), sentinel, dtype=torch.uint8, device=device
|
||||
)
|
||||
@@ -431,8 +622,52 @@ def test_indexer_gather_accepts_upper_bound_output():
|
||||
ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache, dst_k, dst_scale, block_table, cu_seq_lens
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
triton_kv_cache = torch.zeros_like(kv_cache)
|
||||
indexer_k_quant_and_cache_triton(
|
||||
k,
|
||||
triton_kv_cache,
|
||||
slot_mapping,
|
||||
quant_block_size,
|
||||
"ue8m0",
|
||||
)
|
||||
triton_dst_k = torch.full_like(dst_k, sentinel)
|
||||
triton_dst_scale = torch.full_like(dst_scale, sentinel)
|
||||
token_to_seq = torch.cat(
|
||||
(
|
||||
torch.repeat_interleave(
|
||||
torch.arange(num_seqs, dtype=torch.int32, device=device), 3
|
||||
),
|
||||
torch.full(
|
||||
(upper_bound_tokens - valid_tokens,),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
),
|
||||
)
|
||||
)
|
||||
cp_gather_indexer_k_quant_cache_triton(
|
||||
triton_kv_cache,
|
||||
triton_dst_k.view(current_platform.fp8_dtype()),
|
||||
triton_dst_scale,
|
||||
block_table,
|
||||
cu_seq_lens,
|
||||
token_to_seq,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
if current_platform.is_rocm():
|
||||
triton_recovered = triton_dst_k[:valid_tokens].view(
|
||||
current_platform.fp8_dtype()
|
||||
).float() * triton_dst_scale[:valid_tokens].view(torch.float32)
|
||||
triton_error = (triton_recovered - k.float()).abs().amax(dim=1)
|
||||
max_triton_error = (
|
||||
16.0 * triton_dst_scale[:valid_tokens].view(torch.float32).flatten()
|
||||
)
|
||||
assert torch.all(triton_error <= max_triton_error)
|
||||
assert torch.all(triton_dst_k[valid_tokens:] == sentinel)
|
||||
assert torch.all(triton_dst_scale[valid_tokens:] == sentinel)
|
||||
k_recovered = dst_k[:valid_tokens].view(torch.float8_e4m3fn).float() * dst_scale[
|
||||
:valid_tokens
|
||||
].view(torch.float32)
|
||||
|
||||
Reference in New Issue
Block a user