[ROCm][DSV4][Perf] Optimize Triton sparse-MLA decode on gfx950 (#52212)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Signed-off-by: fai <fangzhouai@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Cursor Grok 4.6 <cursoragent@cursor.com>
This commit is contained in:
Fangzhou Ai
2026-08-16 12:16:46 -07:00
committed by GitHub
co-authored by Nick Hill Cursor Grok 4.6
parent 1f0e0bf612
commit ef43e3101b
5 changed files with 1550 additions and 94 deletions
@@ -24,8 +24,14 @@ from typing import Any
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
if current_platform.is_rocm():
from vllm.platforms.rocm import _ON_GFX950
else:
_ON_GFX950 = False
from .fused_indexer_q import _fp32x2_to_fp4x2
@@ -61,12 +67,15 @@ def compress_norm_rope_store_triton(
if head_dim == 512:
kernel = _fused_kv_compress_norm_rope_insert_sparse_attn
num_warps = 4
kernel_kwargs = {"SANITIZE_CACHE_NANS": _ON_GFX950}
elif use_fp4_cache:
kernel = _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn
num_warps = 1
kernel_kwargs = {}
else:
kernel = _fused_kv_compress_norm_rope_insert_indexer_attn
num_warps = 1
kernel_kwargs = {}
kernel[(num_actual,)](
# state cache
@@ -103,6 +112,7 @@ def compress_norm_rope_store_triton(
SCALE_DIM=scale_dim,
KV_BLOCK_STRIDE=kv_cache.stride(0),
num_warps=num_warps,
**kernel_kwargs,
**pdl_kwargs,
)
@@ -145,6 +155,7 @@ def _fused_kv_compress_norm_rope_insert_sparse_attn(
TOKEN_STRIDE: tl.constexpr, # 576 for DeepseekV4
SCALE_DIM: tl.constexpr, # 8 for DeepseekV4 (7 real + 1 pad)
KV_BLOCK_STRIDE: tl.constexpr,
SANITIZE_CACHE_NANS: tl.constexpr,
):
"""Fused compress → RMSNorm → FP8 quant (nope) → RoPE → bf16 store (rope).
@@ -261,7 +272,8 @@ def _fused_kv_compress_norm_rope_insert_sparse_attn(
scale_idx = tl.arange(0, N_QUANT_BLOCKS)
encoded = exponents + 127.0
encoded = tl.maximum(tl.minimum(encoded, 255.0), 0.0)
max_encoded: tl.constexpr = 254.0 if SANITIZE_CACHE_NANS else 255.0
encoded = tl.maximum(tl.minimum(encoded, max_encoded), 0.0)
tl.store(
scale_ptr + scale_idx,
encoded.to(tl.uint8),
@@ -289,6 +301,8 @@ def _fused_kv_compress_norm_rope_insert_sparse_attn(
new_even = even * cos_v - odd * sin_v
new_odd = odd * cos_v + even * sin_v
result = tl.interleave(new_even, new_odd) # [TRITON_BLOCK_SIZE] fp32
if SANITIZE_CACHE_NANS:
result = tl.where(result == result, result, 0.0)
# Store rotated rope portion as bf16 into the cache's bf16 area.
bf16_ptr = (fp8_ptr + NOPE_HEAD_DIM).to(tl.pointer_type(tl.bfloat16))
@@ -417,6 +431,7 @@ def _finalize_norm_rope_quant_store_sparse_attn(
TOKEN_STRIDE: tl.constexpr,
SCALE_DIM: tl.constexpr,
KV_BLOCK_STRIDE: tl.constexpr,
SANITIZE_CACHE_NANS: tl.constexpr,
):
"""Stage 2: read compressed_kv[512] from scratch buffer, then
RMSNorm + FP8 quant (nope) + RoPE + bf16 store
@@ -474,7 +489,8 @@ def _finalize_norm_rope_quant_store_sparse_attn(
tl.store(fp8_ptr + block, x_uint8, mask=block < NOPE_HEAD_DIM)
scale_idx = tl.arange(0, N_QUANT_BLOCKS)
encoded = tl.maximum(tl.minimum(exponents + 127.0, 255.0), 0.0)
max_encoded: tl.constexpr = 254.0 if SANITIZE_CACHE_NANS else 255.0
encoded = tl.maximum(tl.minimum(exponents + 127.0, max_encoded), 0.0)
tl.store(
scale_ptr + scale_idx, encoded.to(tl.uint8), mask=scale_idx < N_NOPE_BLOCKS
)
@@ -494,6 +510,8 @@ def _finalize_norm_rope_quant_store_sparse_attn(
new_even = even * cos_v - odd * sin_v
new_odd = odd * cos_v + even * sin_v
result = tl.interleave(new_even, new_odd)
if SANITIZE_CACHE_NANS:
result = tl.where(result == result, result, 0.0)
bf16_ptr = (fp8_ptr + NOPE_HEAD_DIM).to(tl.pointer_type(tl.bfloat16))
rope_local = block - NOPE_HEAD_DIM
is_rope = (block >= NOPE_HEAD_DIM) & mask
@@ -564,6 +582,7 @@ def _launch_two_stage_sparse_attn_compressor(
TOKEN_STRIDE=token_stride,
SCALE_DIM=scale_dim,
KV_BLOCK_STRIDE=kv_cache.stride(0),
SANITIZE_CACHE_NANS=_ON_GFX950,
)