[ROCm][DSV4][Perf] Optimize Triton sparse-MLA decode on gfx950 (#52212)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Signed-off-by: fai <fangzhouai@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Cursor Grok 4.6 <cursoragent@cursor.com>
This commit is contained in:
Fangzhou Ai
2026-08-16 12:16:46 -07:00
committed by GitHub
co-authored by Nick Hill Cursor Grok 4.6
parent 1f0e0bf612
commit ef43e3101b
5 changed files with 1550 additions and 94 deletions
@@ -24,12 +24,27 @@ def _on_split_decode_arch() -> bool:
return False
def _on_gfx950() -> bool:
if not current_platform.is_rocm():
return False
try:
from vllm.platforms.rocm import _ON_GFX950
return _ON_GFX950
except ImportError:
return False
# The flash-decode split-K decode path is only tuned for AMD gfx942/gfx950; other
# architectures take the fallback decode kernel, so its tests are skipped there.
requires_split_decode_arch = pytest.mark.skipif(
not _on_split_decode_arch(),
reason="split-K decode kernel is only tuned for AMD gfx942/gfx950",
)
requires_gfx950 = pytest.mark.skipif(
not _on_gfx950(),
reason="optimized sparse decode partial is gfx950-only",
)
NOPE_HEAD_DIM = 448
ROPE_HEAD_DIM = 64
@@ -118,28 +133,48 @@ def _pack_fp8_ds_mla_cache(
return cache
def _read_fp8_ds_mla_cache(
cache: torch.Tensor, slot: int, block_size: int, use_fnuz: bool
) -> torch.Tensor:
cache_flat = cache.view(torch.uint8).flatten()
def _poison_fp8_ds_mla_cache_row(
cache: torch.Tensor, block_size: int, slot: int = 0
) -> None:
flat = cache.flatten()
block_idx = slot // block_size
pos = slot % block_size
block_base = block_idx * cache.stride(0)
token_base = block_base + pos * 576
scale_base = block_base + block_size * 576 + pos * 8
flat[token_base] = 0x7F
flat[scale_base : scale_base + 7] = 255
flat[token_base + NOPE_HEAD_DIM : token_base + 576].view(torch.bfloat16)[0] = float(
"nan"
)
def _read_fp8_ds_mla_cache_rows(
cache: torch.Tensor,
slots: torch.Tensor,
block_size: int,
use_fnuz: bool,
) -> torch.Tensor:
cache_flat = cache.view(torch.uint8).flatten()
block_idx = slots // block_size
pos = slots % block_size
block_base = block_idx * cache.stride(0)
token_base = block_base + pos * 576
scale_base = block_base + block_size * 576 + pos * 8
fp8_dtype = torch.float8_e4m3fnuz if use_fnuz else torch.float8_e4m3fn
nope_u8 = cache_flat[token_base : token_base + NOPE_HEAD_DIM]
nope_offsets = torch.arange(NOPE_HEAD_DIM, device=cache.device)
nope_u8 = cache_flat[token_base[:, None] + nope_offsets]
nope = nope_u8.view(fp8_dtype).to(torch.float32)
scale_offsets = torch.arange(7, device=cache.device)
scales = torch.exp2(
cache_flat[scale_base : scale_base + 7].to(torch.float32) - 127.0
cache_flat[scale_base[:, None] + scale_offsets].to(torch.float32) - 127.0
)
nope = nope * scales.repeat_interleave(64)
rope_u8 = cache_flat[
token_base + NOPE_HEAD_DIM : token_base + NOPE_HEAD_DIM + ROPE_HEAD_DIM * 2
]
rope = rope_u8.view(torch.bfloat16).to(torch.float32)
return torch.cat([nope, rope])
nope = nope * scales.repeat_interleave(64, dim=1)
rope_offsets = torch.arange(ROPE_HEAD_DIM * 2, device=cache.device)
rope_u8 = cache_flat[token_base[:, None] + NOPE_HEAD_DIM + rope_offsets]
rope = rope_u8.contiguous().view(torch.bfloat16).to(torch.float32)
return torch.cat([nope, rope], dim=1)
def _ref_sparse_decode_ragged(
@@ -158,19 +193,30 @@ def _ref_sparse_decode_ragged(
out = torch.empty_like(q_f32)
for query_idx in range(q.shape[0]):
row_kv = [
_read_fp8_ds_mla_cache(main_cache, int(slot), block_size, main_use_fnuz)
for slot in main_rows[query_idx]
]
if extra_cache is not None and extra_rows is not None:
row_kv.extend(
_read_fp8_ds_mla_cache(
extra_cache, int(slot), block_size, extra_use_fnuz
row_kv = []
if main_rows[query_idx]:
main_slots = torch.tensor(
main_rows[query_idx], dtype=torch.int64, device=q.device
)
row_kv.append(
_read_fp8_ds_mla_cache_rows(
main_cache, main_slots, block_size, main_use_fnuz
)
)
if extra_cache is not None and extra_rows is not None and extra_rows[query_idx]:
extra_slots = torch.tensor(
extra_rows[query_idx], dtype=torch.int64, device=q.device
)
row_kv.append(
_read_fp8_ds_mla_cache_rows(
extra_cache, extra_slots, block_size, extra_use_fnuz
)
for slot in extra_rows[query_idx]
)
kv = torch.stack(row_kv).to(q.device)
if not row_kv:
out[query_idx] = 0
continue
kv = torch.cat(row_kv)
for head_idx in range(q.shape[1]):
scores = torch.mv(kv, q_f32[query_idx, head_idx]) * scale
if attn_sink is not None:
@@ -198,6 +244,46 @@ def _ragged_from_rows(
)
def _launch_sparse_decode_reduce(
part_m: torch.Tensor,
part_l: torch.Tensor,
part_acc: torch.Tensor,
adaptive_splits: bool,
) -> torch.Tensor:
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
num_queries, num_splits, num_heads = part_m.shape
out = torch.empty(
(num_queries, num_heads, HEAD_DIM),
dtype=torch.bfloat16,
device=part_m.device,
)
attn_sink = torch.empty(1, dtype=torch.float32, device=part_m.device)
mod._sparse_attn_decode_reduce_kernel[(num_queries, num_heads)](
part_m,
part_l,
part_acc,
attn_sink,
out,
out.stride(0),
out.stride(1),
part_m.stride(0),
part_m.stride(1),
part_acc.stride(0),
part_acc.stride(1),
part_acc.stride(2),
num_heads,
HAS_ATTN_SINK=False,
ADAPTIVE_SPLITS=adaptive_splits,
COMB_DIM=HEAD_DIM,
BLOCK_H=1,
NUM_SPLITS=num_splits,
SPLITS_PAD=1 << (num_splits - 1).bit_length(),
num_warps=4,
)
return out
@torch.inference_mode()
def test_paged_mqa_logits_do_not_contain_nan(monkeypatch) -> None:
from vllm._aiter_ops import rocm_aiter_ops
@@ -312,6 +398,19 @@ def test_compute_global_topk_ragged_indices_and_indptr() -> None:
torch.testing.assert_close(actual_lens, expected_lens)
def test_extra_cache_nan_free_provenance_gate(monkeypatch) -> None:
from vllm.models.deepseek_v4.amd import rocm as mod
monkeypatch.setattr(mod, "_ON_GFX950", True)
assert mod._trust_dsv4_extra_cache_nan_free("fp8_ds_mla", False, True)
assert not mod._trust_dsv4_extra_cache_nan_free("fp8_ds_mla", True, True)
assert not mod._trust_dsv4_extra_cache_nan_free("bfloat16", False, True)
assert not mod._trust_dsv4_extra_cache_nan_free("fp8_ds_mla", False, False)
monkeypatch.setattr(mod, "_ON_GFX950", False)
assert not mod._trust_dsv4_extra_cache_nan_free("fp8_ds_mla", False, True)
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_kernel() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
@@ -366,6 +465,7 @@ def test_sparse_attn_decode_ragged_kernel() -> None:
attn_sink = torch.tensor([-0.1, 0.0, 0.1], dtype=torch.float32, device=device)
scale = HEAD_DIM**-0.5
out = torch.empty_like(q)
actual = _rocm_sparse_attn_decode_ragged_triton(
q=q,
main_cache=main_cache,
@@ -378,6 +478,7 @@ def test_sparse_attn_decode_ragged_kernel() -> None:
extra_cache=extra_cache,
extra_indices=extra_indices,
extra_indptr=extra_indptr,
out=out,
)
expected = _ref_sparse_decode_ragged(
q=q,
@@ -391,9 +492,111 @@ def test_sparse_attn_decode_ragged_kernel() -> None:
main_use_fnuz=main_use_fnuz,
)
assert actual.data_ptr() == out.data_ptr()
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
@requires_gfx950
@torch.inference_mode()
def test_sparse_attn_decode_scrubs_untrusted_cache_by_default() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_decode_ragged_triton,
)
device = torch.device("cuda")
block_size = 4
main_cache = torch.zeros(1, block_size, 584, dtype=torch.uint8, device=device)
extra_cache = torch.zeros_like(main_cache)
_poison_fp8_ds_mla_cache_row(main_cache, block_size)
_poison_fp8_ds_mla_cache_row(extra_cache, block_size)
indices = torch.zeros(1, dtype=torch.int32, device=device)
indptr = torch.tensor([0, 1], dtype=torch.int32, device=device)
actual = _rocm_sparse_attn_decode_ragged_triton(
q=torch.ones(1, 1, HEAD_DIM, dtype=torch.bfloat16, device=device),
main_cache=main_cache,
main_indices=indices,
main_indptr=indptr,
scale=HEAD_DIM**-0.5,
attn_sink=None,
nope_head_dim=NOPE_HEAD_DIM,
rope_head_dim=ROPE_HEAD_DIM,
extra_cache=extra_cache,
extra_indices=indices,
extra_indptr=indptr,
)
assert not torch.isnan(actual).any()
assert torch.equal(actual, torch.zeros_like(actual))
@pytest.mark.parametrize("on_gfx950", [False, True])
@torch.inference_mode()
def test_rocm_ragged_graph_buffer_view_tracks_source_width(
monkeypatch, on_gfx950: bool
) -> None:
from vllm.models.deepseek_v4.amd import rocm as rocm_mod
monkeypatch.setattr(rocm_mod, "_ON_GFX950", on_gfx950)
indices_buffer = torch.full((16,), -1, dtype=torch.int32)
indptr_buffer = torch.full((3,), -1, dtype=torch.int32)
first_indices = torch.tensor([3, 5, 7], dtype=torch.int32)
first_indptr = torch.tensor([0, 1, 3], dtype=torch.int32)
first_view, first_indptr_view = rocm_mod._copy_ragged_to_graph_buffers(
first_indices,
first_indptr,
indices_buffer,
indptr_buffer,
num_rows=2,
max_entries_per_row=8,
)
second_indices = torch.tensor([1, 2, 3, 4, 5, 6], dtype=torch.int32)
second_indptr = torch.tensor([0, 2, 6], dtype=torch.int32)
second_view, second_indptr_view = rocm_mod._copy_ragged_to_graph_buffers(
second_indices,
second_indptr,
indices_buffer,
indptr_buffer,
num_rows=2,
max_entries_per_row=8,
)
expected_first_entries = (
first_indices.numel() if on_gfx950 else indices_buffer.numel()
)
expected_second_entries = (
second_indices.numel() if on_gfx950 else indices_buffer.numel()
)
assert first_view.numel() == expected_first_entries
assert second_view.numel() == expected_second_entries
assert first_view.data_ptr() == second_view.data_ptr() == indices_buffer.data_ptr()
assert first_indptr_view.data_ptr() == second_indptr_view.data_ptr()
assert torch.equal(second_view[: second_indices.numel()], second_indices)
assert torch.equal(second_indptr_view, second_indptr)
def test_rocm_capture_metadata_sets_adaptive_marker(monkeypatch) -> None:
from vllm.models.deepseek_v4.amd import rocm as rocm_mod
from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4SparseMLAMetadataBuilder,
)
metadata = SimpleNamespace(for_cudagraph_capture=False)
monkeypatch.setattr(
DeepseekV4SparseMLAMetadataBuilder,
"build_for_cudagraph_capture",
lambda *_: metadata,
)
builder = object.__new__(rocm_mod.DeepseekV4ROCMAiterMLASparseMetadataBuilder)
actual = builder.build_for_cudagraph_capture(SimpleNamespace())
assert actual is metadata
assert actual.for_cudagraph_capture is _on_gfx950()
@requires_split_decode_arch
@torch.inference_mode()
def test_decode_num_splits_heuristic(monkeypatch) -> None:
@@ -408,6 +611,9 @@ def test_decode_num_splits_heuristic(monkeypatch) -> None:
# A tiny batch on a large device should split to add parallelism.
assert mod._decode_num_splits(2, 1, avg_main_len=256.0, avg_extra_len=0.0) > 1
# The shared gfx942 selector retains its original 16-split ceiling.
assert mod._decode_num_splits(1, 1, 128.0, 8192.0) == 16
# The chosen count always stays within the searched [1, 16] range, and a
# zero-length workload never splits (no work to parallelize).
for num_queries in (1, 4, 24, 224, 1024):
@@ -418,6 +624,16 @@ def test_decode_num_splits_heuristic(monkeypatch) -> None:
assert mod._decode_num_splits(2, 1, avg_main_len=0.0, avg_extra_len=0.0) >= 1
@torch.inference_mode()
def test_decode_num_splits_gfx950(monkeypatch) -> None:
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
monkeypatch.setattr(mod, "_decode_cu_count", lambda: 256)
assert mod._decode_gfx950_num_splits(1, 1, 128, 8192) == 32
assert mod._decode_gfx950_num_splits(17, 1, 128, 32) == 4
assert mod._decode_gfx950_num_splits(512, 1, 128, 7812) == 1
@requires_split_decode_arch
@pytest.mark.parametrize("num_splits", [1, 2, 3, 4, 8])
@pytest.mark.parametrize("with_extra", [True, False])
@@ -473,7 +689,14 @@ def test_sparse_attn_decode_split_k_kernel(
scale = HEAD_DIM**-0.5
# Pin the split count so each parametrized value is exercised deterministically.
monkeypatch.setattr(mod, "_decode_num_splits", lambda *args, **kwargs: num_splits)
split_fn = "_decode_gfx950_num_splits" if _on_gfx950() else "_decode_num_splits"
other_split_fn = (
"_decode_num_splits" if _on_gfx950() else "_decode_gfx950_num_splits"
)
monkeypatch.setattr(mod, split_fn, lambda *args, **kwargs: num_splits)
monkeypatch.setattr(
mod, other_split_fn, lambda *args, **kwargs: pytest.fail("wrong selector")
)
actual = mod._rocm_sparse_attn_decode_ragged_triton(
q=q,
@@ -503,6 +726,218 @@ def test_sparse_attn_decode_split_k_kernel(
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
@requires_gfx950
@torch.inference_mode()
def test_sparse_attn_decode_gfx950_adaptive_reduce_ignores_stale_scratch() -> None:
device = torch.device("cuda")
part_m = torch.full((1, 8, 1), torch.finfo(torch.float32).min, device=device)
part_l = torch.zeros_like(part_m)
part_acc = torch.full((1, 8, 1, HEAD_DIM), float("nan"), device=device)
part_m[:, :2] = 0
part_l[:, :2] = 1
part_acc[:, 0] = 1
part_acc[:, 1] = 3
actual = _launch_sparse_decode_reduce(part_m, part_l, part_acc, True)
assert torch.isfinite(actual).all()
assert torch.equal(actual, torch.full_like(actual, 2))
@requires_gfx950
@pytest.mark.parametrize("extra_len", [0, 1, 31, 32, 33, 63, 64, 65])
@torch.inference_mode()
def test_sparse_attn_decode_gfx950_outer64_boundaries(
monkeypatch, extra_len: int
) -> None:
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
device = torch.device("cuda")
torch.manual_seed(13)
block_size = 4
num_heads = 16
num_extra_rows = 80
q = torch.randn(2, num_heads, HEAD_DIM, dtype=torch.bfloat16, device=device)
q *= 0.125
main_cache = torch.zeros(1, block_size, 584, dtype=torch.uint8, device=device)
main_indices = torch.empty(0, dtype=torch.int32, device=device)
main_indptr = torch.zeros(3, dtype=torch.int32, device=device)
extra_cache = _pack_fp8_ds_mla_cache(
torch.randn(num_extra_rows, HEAD_DIM, dtype=torch.bfloat16, device=device)
* 0.125,
block_size,
use_fnuz=False,
)
_poison_fp8_ds_mla_cache_row(extra_cache, block_size)
raw_row = list(range(1, extra_len + 1))
if extra_len > 3:
raw_row[3] = -1
if extra_len > 40:
raw_row[40] = num_extra_rows
if extra_len > 64:
raw_row[64] = num_extra_rows + 1024
extra_indices, extra_indptr = _ragged_from_rows([raw_row, []], device)
valid_row = [slot for slot in raw_row if 0 <= slot < num_extra_rows]
attn_sink = torch.linspace(-0.1, 0.1, num_heads, dtype=torch.float32, device=device)
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 1)
actual = mod._rocm_sparse_attn_decode_ragged_triton(
q=q,
main_cache=main_cache,
main_indices=main_indices,
main_indptr=main_indptr,
scale=HEAD_DIM**-0.5,
attn_sink=attn_sink,
nope_head_dim=NOPE_HEAD_DIM,
rope_head_dim=ROPE_HEAD_DIM,
extra_cache=extra_cache,
extra_indices=extra_indices,
extra_indptr=extra_indptr,
)
expected = _ref_sparse_decode_ragged(
q=q,
main_cache=main_cache,
main_rows=[[], []],
scale=HEAD_DIM**-0.5,
attn_sink=attn_sink,
block_size=block_size,
extra_cache=extra_cache,
extra_rows=[valid_row, []],
)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
assert torch.equal(actual[1], torch.zeros_like(actual[1]))
@requires_gfx950
@torch.inference_mode()
def test_sparse_attn_decode_gfx950_graph_replay(monkeypatch) -> None:
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
device = torch.device("cuda")
torch.manual_seed(17)
block_size = 64
num_queries = 16
num_heads = 16
num_splits = 8
extra_per_query = 65 * num_splits
max_extra_per_query = 8192
q = (
torch.randn(
num_queries,
num_heads,
HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125
)
main_cache = _pack_fp8_ds_mla_cache(
torch.randn(num_queries, HEAD_DIM, dtype=torch.bfloat16, device=device) * 0.125,
block_size,
use_fnuz=False,
)
extra_cache = _pack_fp8_ds_mla_cache(
torch.randn(
num_queries * extra_per_query,
HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125,
block_size,
use_fnuz=False,
)
main_rows = [[query_idx] for query_idx in range(num_queries)]
extra_rows = [
list(range(query_idx * extra_per_query, (query_idx + 1) * extra_per_query))
for query_idx in range(num_queries)
]
main_indices, main_indptr = _ragged_from_rows(main_rows, device)
short_extra_rows = [row[:64] for row in extra_rows]
long_indices, long_indptr = _ragged_from_rows(extra_rows, device)
short_indices, short_indptr = _ragged_from_rows(short_extra_rows, device)
extra_indices = torch.full(
(num_queries * max_extra_per_query,),
-1,
dtype=torch.int32,
device=device,
)
extra_indices[: long_indices.numel()].copy_(long_indices)
extra_indptr = long_indptr.clone()
extra_indices_ptr = extra_indices.data_ptr()
attn_sink = torch.linspace(-0.1, 0.1, num_heads, dtype=torch.float32, device=device)
out = torch.empty_like(q)
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: num_splits)
def run_decode() -> torch.Tensor:
return mod._rocm_sparse_attn_decode_ragged_triton(
q=q,
main_cache=main_cache,
main_indices=main_indices,
main_indptr=main_indptr,
scale=HEAD_DIM**-0.5,
attn_sink=attn_sink,
nope_head_dim=NOPE_HEAD_DIM,
rope_head_dim=ROPE_HEAD_DIM,
extra_cache=extra_cache,
extra_indices=extra_indices,
extra_indptr=extra_indptr,
out=out,
extra_cache_nan_free=True,
adaptive_splits=True,
)
run_decode()
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured_out = run_decode()
torch.accelerator.synchronize()
captured_long = out.clone()
expected_long = _ref_sparse_decode_ragged(
q=q,
main_cache=main_cache,
main_rows=main_rows,
scale=HEAD_DIM**-0.5,
attn_sink=attn_sink,
block_size=block_size,
extra_cache=extra_cache,
extra_rows=extra_rows,
)
torch.testing.assert_close(captured_long, expected_long, atol=2e-2, rtol=2e-2)
extra_indices[: short_indices.numel()].copy_(short_indices)
extra_indptr.copy_(short_indptr)
graph.replay()
torch.accelerator.synchronize()
short_out = out.clone()
expected_short = _ref_sparse_decode_ragged(
q=q,
main_cache=main_cache,
main_rows=main_rows,
scale=HEAD_DIM**-0.5,
attn_sink=attn_sink,
block_size=block_size,
extra_cache=extra_cache,
extra_rows=short_extra_rows,
)
assert captured_out.data_ptr() == out.data_ptr()
assert extra_indices.data_ptr() == extra_indices_ptr
assert extra_indices.numel() == num_queries * max_extra_per_query
assert not torch.equal(short_out, captured_long)
torch.testing.assert_close(short_out, expected_short, atol=2e-2, rtol=2e-2)
extra_indices[: long_indices.numel()].copy_(long_indices)
extra_indptr.copy_(long_indptr)
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(out, expected_long, atol=2e-2, rtol=2e-2)
# ---------------------------------------------------------------------------
# o-projection: fused inverse-RoPE + cached bf16 wo_a (rocm_inv_rope_einsum)
# ---------------------------------------------------------------------------
+239 -4
View File
@@ -28,16 +28,81 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
_fused_kv_compress_norm_rope_insert_indexer_attn,
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn,
_launch_two_stage_sparse_attn_compressor,
compress_norm_rope_store_triton,
)
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
cp_gather_indexer_k_quant_cache_triton,
indexer_k_quant_and_cache_triton,
)
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
def _on_gfx950() -> bool:
if not current_platform.is_rocm():
return False
try:
from vllm.platforms.rocm import _ON_GFX950
return _ON_GFX950
except Exception:
return False
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
def test_cp_gather_despecialized_kernel_is_gfx950_only(monkeypatch):
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
class FakeKernel:
def __init__(self):
self.calls = []
def __getitem__(self, grid):
def launch(*args):
self.calls.append((grid, args))
return launch
legacy_kernel = FakeKernel()
gfx950_kernel = FakeKernel()
monkeypatch.setattr(mod, "_cp_gather_indexer_quant_cache_kernel", legacy_kernel)
monkeypatch.setattr(
mod,
"_cp_gather_indexer_quant_cache_gfx950_kernel",
gfx950_kernel,
)
k_cache = torch.zeros((4, 1, 132), dtype=torch.uint8)
k_fp8 = torch.empty((5, 128), dtype=current_platform.fp8_dtype())
k_scale = torch.empty((5, 4), dtype=torch.uint8)
block_table = torch.zeros((2, 7), dtype=torch.int32)
cu_seqlen = torch.tensor([0, 2, 5], dtype=torch.int32)
token_to_seq = torch.tensor([0, 0, 1, 1, 1], dtype=torch.int32)
args = (k_cache, k_fp8, k_scale, block_table, cu_seqlen, token_to_seq)
monkeypatch.setattr(mod, "_ON_GFX950", True)
mod.cp_gather_indexer_k_quant_cache_triton(*args)
assert len(gfx950_kernel.calls) == 1
assert not legacy_kernel.calls
gfx950_grid, gfx950_args = gfx950_kernel.calls[0]
assert gfx950_grid == (5,)
assert len(gfx950_args) == 18
assert gfx950_args[-3:] == (2, 7, 4)
monkeypatch.setattr(mod, "_ON_GFX950", False)
mod.cp_gather_indexer_k_quant_cache_triton(*args)
assert len(legacy_kernel.calls) == 1
legacy_grid, legacy_args = legacy_kernel.calls[0]
assert legacy_grid == (5,)
assert len(legacy_args) == 19
assert legacy_args[-4:] == (5, 2, 7, 4)
@pytest.mark.parametrize(
("window_size", "num_speculative_tokens", "expected"),
[(128, 5, 192), (512, 5, 576), (1024, 0, 1024)],
@@ -91,6 +156,129 @@ def _ue8m0_reference(x: torch.Tensor, block_size: int, fp8_max: float):
return x_fp8, scales
def _decode_dsv4_cache_row(
cache: torch.Tensor, block_size: int, scrub_nan: bool
) -> torch.Tensor:
flat = cache.flatten()
nope = flat[:448].view(torch.float8_e4m3fn).to(torch.bfloat16)
encoded = flat[block_size * 576 : block_size * 576 + 7]
scales = torch.exp2(encoded.to(torch.float32) - 127.0).to(torch.bfloat16)
nope = nope * scales.repeat_interleave(64)
rope = flat[448:576].view(torch.bfloat16)
decoded = torch.cat((nope, rope))
if scrub_nan:
decoded = torch.where(decoded == decoded, decoded, 0.0)
return decoded
def _assert_nan_free_cache_matches_legacy_scrub(
cache: torch.Tensor, block_size: int
) -> None:
flat = cache.flatten()
scale_base = block_size * 576
scale_codes = flat[scale_base : scale_base + 8]
assert scale_codes[0].item() == 254
assert scale_codes[1].item() == 247
assert scale_codes[:7].max().item() <= 254
nope_bytes = flat[:448]
assert not ((nope_bytes == 0x7F) | (nope_bytes == 0xFF)).any()
rope = flat[448:576].view(torch.bfloat16)
assert not torch.isnan(rope).any()
assert torch.isposinf(rope[0])
assert torch.equal(rope[1:4], torch.zeros_like(rope[1:4]))
legacy_cache = cache.clone()
legacy_flat = legacy_cache.flatten()
legacy_flat[scale_base] = 255
legacy_rope = legacy_flat[448:576].view(torch.bfloat16)
legacy_rope[1:4] = float("nan")
canonical = _decode_dsv4_cache_row(cache, block_size, scrub_nan=False)
legacy = _decode_dsv4_cache_row(legacy_cache, block_size, scrub_nan=True)
torch.testing.assert_close(canonical, legacy, rtol=0, atol=0)
assert torch.isinf(canonical[0])
assert torch.isposinf(canonical[64])
@pytest.mark.skipif(
not _on_gfx950(),
reason="NaN-free fp8_ds_mla compressed-cache contract is gfx950-only",
)
@pytest.mark.parametrize("writer", ["single_pass", "two_stage_finalizer"])
def test_gfx950_compressed_cache_canonicalizes_nonfinite(writer: str) -> None:
head_dim = 512
rope_dim = 64
block_size = 4
device = "cuda"
positions = torch.zeros(1, dtype=torch.int64, device=device)
slot_mapping = torch.zeros(1, dtype=torch.int64, device=device)
rms_weight = torch.ones(head_dim, dtype=torch.bfloat16, device=device)
rms_weight[0] = float("inf")
rms_weight[64] = torch.finfo(torch.bfloat16).max
rms_weight[448] = float("inf")
rms_weight[450] = float("nan")
cos_sin_cache = torch.zeros(1, rope_dim, dtype=torch.float32, device=device)
cos_sin_cache[:, : rope_dim // 2] = 1.0
cache = torch.zeros(1, block_size, 584, dtype=torch.uint8, device=device)
state_cache = torch.zeros(1, 1, 2 * head_dim, dtype=torch.float32, device=device)
state_cache[..., :head_dim] = 1.0
token_to_req = torch.zeros(1, dtype=torch.int32, device=device)
block_table = torch.zeros(1, 1, dtype=torch.int32, device=device)
if writer == "single_pass":
compress_norm_rope_store_triton(
state_cache=state_cache,
num_actual=1,
token_to_req_indices=token_to_req,
positions=positions,
slot_mapping=slot_mapping,
block_table=block_table,
block_size=1,
state_width=head_dim,
cos_sin_cache=cos_sin_cache,
kv_cache=cache,
k_cache_metadata=SimpleNamespace(slot_mapping=slot_mapping),
pdl_kwargs={},
head_dim=head_dim,
rope_head_dim=rope_dim,
compress_ratio=1,
overlap=False,
use_fp4_cache=False,
rms_norm_weight=rms_weight,
rms_norm_eps=1e-6,
quant_block=64,
token_stride=576,
scale_dim=8,
)
else:
_launch_two_stage_sparse_attn_compressor(
state_cache,
token_to_req,
positions,
slot_mapping,
block_table,
1,
head_dim,
1,
cos_sin_cache,
cache,
slot_mapping,
rms_weight,
1e-6,
64,
576,
8,
head_dim,
rope_dim,
1,
torch.empty(1, head_dim, dtype=torch.float32, device=device),
)
_assert_nan_free_cache_matches_legacy_scrub(cache, block_size)
@pytest.mark.parametrize(
("starts", "query_start_loc", "expected"),
[
@@ -402,7 +590,8 @@ def test_indexer_gather_accepts_upper_bound_output():
valid_tokens = 9
upper_bound_tokens = 13
block_size = 16
num_blocks = 2
num_seqs = 3
num_blocks = num_seqs
sentinel = 123
device = "cuda"
@@ -410,13 +599,15 @@ def test_indexer_gather_accepts_upper_bound_output():
kv_cache = torch.zeros(
num_blocks, block_size, cache_stride, dtype=torch.uint8, device=device
)
slot_mapping = torch.arange(valid_tokens, dtype=torch.int64, device=device)
slot_mapping = torch.tensor(
[0, 1, 2, 16, 17, 18, 32, 33, 34], dtype=torch.int64, device=device
)
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, quant_block_size, "ue8m0")
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(
0
1
)
cu_seq_lens = torch.tensor([0, valid_tokens], dtype=torch.int32, device=device)
cu_seq_lens = torch.tensor([0, 3, 6, 9], dtype=torch.int32, device=device)
dst_k = torch.full(
(upper_bound_tokens, head_dim), sentinel, dtype=torch.uint8, device=device
)
@@ -431,8 +622,52 @@ def test_indexer_gather_accepts_upper_bound_output():
ops.cp_gather_indexer_k_quant_cache(
kv_cache, dst_k, dst_scale, block_table, cu_seq_lens
)
if current_platform.is_rocm():
triton_kv_cache = torch.zeros_like(kv_cache)
indexer_k_quant_and_cache_triton(
k,
triton_kv_cache,
slot_mapping,
quant_block_size,
"ue8m0",
)
triton_dst_k = torch.full_like(dst_k, sentinel)
triton_dst_scale = torch.full_like(dst_scale, sentinel)
token_to_seq = torch.cat(
(
torch.repeat_interleave(
torch.arange(num_seqs, dtype=torch.int32, device=device), 3
),
torch.full(
(upper_bound_tokens - valid_tokens,),
-1,
dtype=torch.int32,
device=device,
),
)
)
cp_gather_indexer_k_quant_cache_triton(
triton_kv_cache,
triton_dst_k.view(current_platform.fp8_dtype()),
triton_dst_scale,
block_table,
cu_seq_lens,
token_to_seq,
)
torch.accelerator.synchronize()
if current_platform.is_rocm():
triton_recovered = triton_dst_k[:valid_tokens].view(
current_platform.fp8_dtype()
).float() * triton_dst_scale[:valid_tokens].view(torch.float32)
triton_error = (triton_recovered - k.float()).abs().amax(dim=1)
max_triton_error = (
16.0 * triton_dst_scale[:valid_tokens].view(torch.float32).flatten()
)
assert torch.all(triton_error <= max_triton_error)
assert torch.all(triton_dst_k[valid_tokens:] == sentinel)
assert torch.all(triton_dst_scale[valid_tokens:] == sentinel)
k_recovered = dst_k[:valid_tokens].view(torch.float8_e4m3fn).float() * dst_scale[
:valid_tokens
].view(torch.float32)