[ROCm][DSV4][Perf] Optimize Triton sparse-MLA decode on gfx950 (#52212)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Signed-off-by: fai <fangzhouai@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Cursor Grok 4.6 <cursoragent@cursor.com>
This commit is contained in:
Fangzhou Ai
2026-08-16 12:16:46 -07:00
committed by GitHub
co-authored by Nick Hill Cursor Grok 4.6
parent 1f0e0bf612
commit ef43e3101b
5 changed files with 1550 additions and 94 deletions
+239 -4
View File
@@ -28,16 +28,81 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
_fused_kv_compress_norm_rope_insert_indexer_attn,
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn,
_launch_two_stage_sparse_attn_compressor,
compress_norm_rope_store_triton,
)
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
cp_gather_indexer_k_quant_cache_triton,
indexer_k_quant_and_cache_triton,
)
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
def _on_gfx950() -> bool:
if not current_platform.is_rocm():
return False
try:
from vllm.platforms.rocm import _ON_GFX950
return _ON_GFX950
except Exception:
return False
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
def test_cp_gather_despecialized_kernel_is_gfx950_only(monkeypatch):
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
class FakeKernel:
def __init__(self):
self.calls = []
def __getitem__(self, grid):
def launch(*args):
self.calls.append((grid, args))
return launch
legacy_kernel = FakeKernel()
gfx950_kernel = FakeKernel()
monkeypatch.setattr(mod, "_cp_gather_indexer_quant_cache_kernel", legacy_kernel)
monkeypatch.setattr(
mod,
"_cp_gather_indexer_quant_cache_gfx950_kernel",
gfx950_kernel,
)
k_cache = torch.zeros((4, 1, 132), dtype=torch.uint8)
k_fp8 = torch.empty((5, 128), dtype=current_platform.fp8_dtype())
k_scale = torch.empty((5, 4), dtype=torch.uint8)
block_table = torch.zeros((2, 7), dtype=torch.int32)
cu_seqlen = torch.tensor([0, 2, 5], dtype=torch.int32)
token_to_seq = torch.tensor([0, 0, 1, 1, 1], dtype=torch.int32)
args = (k_cache, k_fp8, k_scale, block_table, cu_seqlen, token_to_seq)
monkeypatch.setattr(mod, "_ON_GFX950", True)
mod.cp_gather_indexer_k_quant_cache_triton(*args)
assert len(gfx950_kernel.calls) == 1
assert not legacy_kernel.calls
gfx950_grid, gfx950_args = gfx950_kernel.calls[0]
assert gfx950_grid == (5,)
assert len(gfx950_args) == 18
assert gfx950_args[-3:] == (2, 7, 4)
monkeypatch.setattr(mod, "_ON_GFX950", False)
mod.cp_gather_indexer_k_quant_cache_triton(*args)
assert len(legacy_kernel.calls) == 1
legacy_grid, legacy_args = legacy_kernel.calls[0]
assert legacy_grid == (5,)
assert len(legacy_args) == 19
assert legacy_args[-4:] == (5, 2, 7, 4)
@pytest.mark.parametrize(
("window_size", "num_speculative_tokens", "expected"),
[(128, 5, 192), (512, 5, 576), (1024, 0, 1024)],
@@ -91,6 +156,129 @@ def _ue8m0_reference(x: torch.Tensor, block_size: int, fp8_max: float):
return x_fp8, scales
def _decode_dsv4_cache_row(
cache: torch.Tensor, block_size: int, scrub_nan: bool
) -> torch.Tensor:
flat = cache.flatten()
nope = flat[:448].view(torch.float8_e4m3fn).to(torch.bfloat16)
encoded = flat[block_size * 576 : block_size * 576 + 7]
scales = torch.exp2(encoded.to(torch.float32) - 127.0).to(torch.bfloat16)
nope = nope * scales.repeat_interleave(64)
rope = flat[448:576].view(torch.bfloat16)
decoded = torch.cat((nope, rope))
if scrub_nan:
decoded = torch.where(decoded == decoded, decoded, 0.0)
return decoded
def _assert_nan_free_cache_matches_legacy_scrub(
cache: torch.Tensor, block_size: int
) -> None:
flat = cache.flatten()
scale_base = block_size * 576
scale_codes = flat[scale_base : scale_base + 8]
assert scale_codes[0].item() == 254
assert scale_codes[1].item() == 247
assert scale_codes[:7].max().item() <= 254
nope_bytes = flat[:448]
assert not ((nope_bytes == 0x7F) | (nope_bytes == 0xFF)).any()
rope = flat[448:576].view(torch.bfloat16)
assert not torch.isnan(rope).any()
assert torch.isposinf(rope[0])
assert torch.equal(rope[1:4], torch.zeros_like(rope[1:4]))
legacy_cache = cache.clone()
legacy_flat = legacy_cache.flatten()
legacy_flat[scale_base] = 255
legacy_rope = legacy_flat[448:576].view(torch.bfloat16)
legacy_rope[1:4] = float("nan")
canonical = _decode_dsv4_cache_row(cache, block_size, scrub_nan=False)
legacy = _decode_dsv4_cache_row(legacy_cache, block_size, scrub_nan=True)
torch.testing.assert_close(canonical, legacy, rtol=0, atol=0)
assert torch.isinf(canonical[0])
assert torch.isposinf(canonical[64])
@pytest.mark.skipif(
not _on_gfx950(),
reason="NaN-free fp8_ds_mla compressed-cache contract is gfx950-only",
)
@pytest.mark.parametrize("writer", ["single_pass", "two_stage_finalizer"])
def test_gfx950_compressed_cache_canonicalizes_nonfinite(writer: str) -> None:
head_dim = 512
rope_dim = 64
block_size = 4
device = "cuda"
positions = torch.zeros(1, dtype=torch.int64, device=device)
slot_mapping = torch.zeros(1, dtype=torch.int64, device=device)
rms_weight = torch.ones(head_dim, dtype=torch.bfloat16, device=device)
rms_weight[0] = float("inf")
rms_weight[64] = torch.finfo(torch.bfloat16).max
rms_weight[448] = float("inf")
rms_weight[450] = float("nan")
cos_sin_cache = torch.zeros(1, rope_dim, dtype=torch.float32, device=device)
cos_sin_cache[:, : rope_dim // 2] = 1.0
cache = torch.zeros(1, block_size, 584, dtype=torch.uint8, device=device)
state_cache = torch.zeros(1, 1, 2 * head_dim, dtype=torch.float32, device=device)
state_cache[..., :head_dim] = 1.0
token_to_req = torch.zeros(1, dtype=torch.int32, device=device)
block_table = torch.zeros(1, 1, dtype=torch.int32, device=device)
if writer == "single_pass":
compress_norm_rope_store_triton(
state_cache=state_cache,
num_actual=1,
token_to_req_indices=token_to_req,
positions=positions,
slot_mapping=slot_mapping,
block_table=block_table,
block_size=1,
state_width=head_dim,
cos_sin_cache=cos_sin_cache,
kv_cache=cache,
k_cache_metadata=SimpleNamespace(slot_mapping=slot_mapping),
pdl_kwargs={},
head_dim=head_dim,
rope_head_dim=rope_dim,
compress_ratio=1,
overlap=False,
use_fp4_cache=False,
rms_norm_weight=rms_weight,
rms_norm_eps=1e-6,
quant_block=64,
token_stride=576,
scale_dim=8,
)
else:
_launch_two_stage_sparse_attn_compressor(
state_cache,
token_to_req,
positions,
slot_mapping,
block_table,
1,
head_dim,
1,
cos_sin_cache,
cache,
slot_mapping,
rms_weight,
1e-6,
64,
576,
8,
head_dim,
rope_dim,
1,
torch.empty(1, head_dim, dtype=torch.float32, device=device),
)
_assert_nan_free_cache_matches_legacy_scrub(cache, block_size)
@pytest.mark.parametrize(
("starts", "query_start_loc", "expected"),
[
@@ -402,7 +590,8 @@ def test_indexer_gather_accepts_upper_bound_output():
valid_tokens = 9
upper_bound_tokens = 13
block_size = 16
num_blocks = 2
num_seqs = 3
num_blocks = num_seqs
sentinel = 123
device = "cuda"
@@ -410,13 +599,15 @@ def test_indexer_gather_accepts_upper_bound_output():
kv_cache = torch.zeros(
num_blocks, block_size, cache_stride, dtype=torch.uint8, device=device
)
slot_mapping = torch.arange(valid_tokens, dtype=torch.int64, device=device)
slot_mapping = torch.tensor(
[0, 1, 2, 16, 17, 18, 32, 33, 34], dtype=torch.int64, device=device
)
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, quant_block_size, "ue8m0")
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(
0
1
)
cu_seq_lens = torch.tensor([0, valid_tokens], dtype=torch.int32, device=device)
cu_seq_lens = torch.tensor([0, 3, 6, 9], dtype=torch.int32, device=device)
dst_k = torch.full(
(upper_bound_tokens, head_dim), sentinel, dtype=torch.uint8, device=device
)
@@ -431,8 +622,52 @@ def test_indexer_gather_accepts_upper_bound_output():
ops.cp_gather_indexer_k_quant_cache(
kv_cache, dst_k, dst_scale, block_table, cu_seq_lens
)
if current_platform.is_rocm():
triton_kv_cache = torch.zeros_like(kv_cache)
indexer_k_quant_and_cache_triton(
k,
triton_kv_cache,
slot_mapping,
quant_block_size,
"ue8m0",
)
triton_dst_k = torch.full_like(dst_k, sentinel)
triton_dst_scale = torch.full_like(dst_scale, sentinel)
token_to_seq = torch.cat(
(
torch.repeat_interleave(
torch.arange(num_seqs, dtype=torch.int32, device=device), 3
),
torch.full(
(upper_bound_tokens - valid_tokens,),
-1,
dtype=torch.int32,
device=device,
),
)
)
cp_gather_indexer_k_quant_cache_triton(
triton_kv_cache,
triton_dst_k.view(current_platform.fp8_dtype()),
triton_dst_scale,
block_table,
cu_seq_lens,
token_to_seq,
)
torch.accelerator.synchronize()
if current_platform.is_rocm():
triton_recovered = triton_dst_k[:valid_tokens].view(
current_platform.fp8_dtype()
).float() * triton_dst_scale[:valid_tokens].view(torch.float32)
triton_error = (triton_recovered - k.float()).abs().amax(dim=1)
max_triton_error = (
16.0 * triton_dst_scale[:valid_tokens].view(torch.float32).flatten()
)
assert torch.all(triton_error <= max_triton_error)
assert torch.all(triton_dst_k[valid_tokens:] == sentinel)
assert torch.all(triton_dst_scale[valid_tokens:] == sentinel)
k_recovered = dst_k[:valid_tokens].view(torch.float8_e4m3fn).float() * dst_scale[
:valid_tokens
].view(torch.float32)