[ROCm][DSV4][Perf] Optimize Triton sparse-MLA decode on gfx950 (#52212)
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Signed-off-by: fai <fangzhouai@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Cursor Grok 4.6 <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Nick Hill
Cursor Grok 4.6
parent
1f0e0bf612
commit
ef43e3101b
@@ -19,6 +19,7 @@ from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4SparseMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import _ON_GFX950
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
@@ -36,6 +37,19 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
|
||||
def _trust_dsv4_extra_cache_nan_free(
|
||||
kv_cache_dtype: str,
|
||||
has_kv_transfer: bool,
|
||||
has_extra_cache: bool,
|
||||
) -> bool:
|
||||
return (
|
||||
_ON_GFX950
|
||||
and kv_cache_dtype == "fp8_ds_mla"
|
||||
and not has_kv_transfer
|
||||
and has_extra_cache
|
||||
)
|
||||
|
||||
|
||||
def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
|
||||
lengths = lengths.to(dtype=torch.int32).contiguous()
|
||||
indptr = torch.zeros(lengths.shape[0] + 1, dtype=torch.int32, device=lengths.device)
|
||||
@@ -294,9 +308,13 @@ def _copy_ragged_to_graph_buffers(
|
||||
|
||||
max_entries = max(num_rows * max_entries_per_row, 1)
|
||||
ragged_out = ragged_indices_buffer[:max_entries]
|
||||
nnz = ragged_indices.numel()
|
||||
if nnz > 0:
|
||||
ragged_out[:nnz].copy_(ragged_indices, non_blocking=True)
|
||||
source_entries = ragged_indices.numel()
|
||||
if source_entries > 0:
|
||||
ragged_out[:source_entries].copy_(ragged_indices, non_blocking=True)
|
||||
if _ON_GFX950:
|
||||
# Preserve the graph-stable base pointer while exposing source capacity
|
||||
# to the sync-free split selector; indptr still carries the true NNZ.
|
||||
ragged_out = ragged_out[: max(source_entries, 1)]
|
||||
return ragged_out, indptr_out
|
||||
|
||||
|
||||
@@ -306,6 +324,7 @@ class DeepseekV4ROCMAiterMLASparseMetadata(DeepseekV4FlashMLAMetadata):
|
||||
|
||||
c128a_decode_topk_ragged_indices: torch.Tensor | None = None
|
||||
c128a_decode_topk_ragged_indptr: torch.Tensor | None = None
|
||||
for_cudagraph_capture: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -370,6 +389,16 @@ class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBui
|
||||
c128a_decode_topk_ragged_indptr=ragged_indptr,
|
||||
)
|
||||
|
||||
def build_for_cudagraph_capture(
|
||||
self, common_attn_metadata: CommonAttentionMetadata
|
||||
) -> DeepseekV4ROCMAiterMLASparseMetadata:
|
||||
metadata = cast(
|
||||
DeepseekV4ROCMAiterMLASparseMetadata,
|
||||
super().build_for_cudagraph_capture(common_attn_metadata),
|
||||
)
|
||||
metadata.for_cudagraph_capture = _ON_GFX950
|
||||
return metadata
|
||||
|
||||
|
||||
class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
|
||||
# Keep fused multi-step decode disabled until update_draft_decode_metadata()
|
||||
@@ -451,7 +480,9 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
backend_cls = DeepseekV4ROCMAiterMLASparseBackend
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
vllm_config = args[0] if args else kwargs["vllm_config"]
|
||||
super().__init__(*args, **kwargs)
|
||||
self._has_kv_transfer = vllm_config.kv_transfer_config is not None
|
||||
# Block scale for the preshuffled weight; None = not preshuffled.
|
||||
self._wqa_wkv_scale: torch.Tensor | None = None
|
||||
self._wo_b_scale: torch.Tensor | None = None
|
||||
@@ -605,6 +636,13 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
attn_metadata=rocm_metadata,
|
||||
swa_only=swa_only,
|
||||
output=output[:num_decode_tokens],
|
||||
adaptive_splits=(
|
||||
_ON_GFX950
|
||||
and not swa_only
|
||||
and self.compress_ratio == 128
|
||||
and rocm_metadata is not None
|
||||
and rocm_metadata.for_cudagraph_capture
|
||||
),
|
||||
)
|
||||
|
||||
def _forward_decode(
|
||||
@@ -615,6 +653,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None,
|
||||
swa_only: bool,
|
||||
output: torch.Tensor,
|
||||
adaptive_splits: bool,
|
||||
) -> None:
|
||||
num_decodes = swa_metadata.num_decodes
|
||||
num_decode_tokens = swa_metadata.num_decode_tokens
|
||||
@@ -666,6 +705,12 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
nope_head_dim=self.nope_head_dim,
|
||||
rope_head_dim=self.rope_head_dim,
|
||||
output=output,
|
||||
adaptive_splits=adaptive_splits,
|
||||
extra_cache_nan_free=_trust_dsv4_extra_cache_nan_free(
|
||||
self.kv_cache_dtype,
|
||||
self._has_kv_transfer,
|
||||
not swa_only and kv_cache is not None,
|
||||
),
|
||||
)
|
||||
|
||||
def _forward_prefill(
|
||||
|
||||
@@ -24,8 +24,14 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import _ON_GFX950
|
||||
else:
|
||||
_ON_GFX950 = False
|
||||
|
||||
from .fused_indexer_q import _fp32x2_to_fp4x2
|
||||
|
||||
|
||||
@@ -61,12 +67,15 @@ def compress_norm_rope_store_triton(
|
||||
if head_dim == 512:
|
||||
kernel = _fused_kv_compress_norm_rope_insert_sparse_attn
|
||||
num_warps = 4
|
||||
kernel_kwargs = {"SANITIZE_CACHE_NANS": _ON_GFX950}
|
||||
elif use_fp4_cache:
|
||||
kernel = _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn
|
||||
num_warps = 1
|
||||
kernel_kwargs = {}
|
||||
else:
|
||||
kernel = _fused_kv_compress_norm_rope_insert_indexer_attn
|
||||
num_warps = 1
|
||||
kernel_kwargs = {}
|
||||
|
||||
kernel[(num_actual,)](
|
||||
# state cache
|
||||
@@ -103,6 +112,7 @@ def compress_norm_rope_store_triton(
|
||||
SCALE_DIM=scale_dim,
|
||||
KV_BLOCK_STRIDE=kv_cache.stride(0),
|
||||
num_warps=num_warps,
|
||||
**kernel_kwargs,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
|
||||
@@ -145,6 +155,7 @@ def _fused_kv_compress_norm_rope_insert_sparse_attn(
|
||||
TOKEN_STRIDE: tl.constexpr, # 576 for DeepseekV4
|
||||
SCALE_DIM: tl.constexpr, # 8 for DeepseekV4 (7 real + 1 pad)
|
||||
KV_BLOCK_STRIDE: tl.constexpr,
|
||||
SANITIZE_CACHE_NANS: tl.constexpr,
|
||||
):
|
||||
"""Fused compress → RMSNorm → FP8 quant (nope) → RoPE → bf16 store (rope).
|
||||
|
||||
@@ -261,7 +272,8 @@ def _fused_kv_compress_norm_rope_insert_sparse_attn(
|
||||
|
||||
scale_idx = tl.arange(0, N_QUANT_BLOCKS)
|
||||
encoded = exponents + 127.0
|
||||
encoded = tl.maximum(tl.minimum(encoded, 255.0), 0.0)
|
||||
max_encoded: tl.constexpr = 254.0 if SANITIZE_CACHE_NANS else 255.0
|
||||
encoded = tl.maximum(tl.minimum(encoded, max_encoded), 0.0)
|
||||
tl.store(
|
||||
scale_ptr + scale_idx,
|
||||
encoded.to(tl.uint8),
|
||||
@@ -289,6 +301,8 @@ def _fused_kv_compress_norm_rope_insert_sparse_attn(
|
||||
new_even = even * cos_v - odd * sin_v
|
||||
new_odd = odd * cos_v + even * sin_v
|
||||
result = tl.interleave(new_even, new_odd) # [TRITON_BLOCK_SIZE] fp32
|
||||
if SANITIZE_CACHE_NANS:
|
||||
result = tl.where(result == result, result, 0.0)
|
||||
|
||||
# Store rotated rope portion as bf16 into the cache's bf16 area.
|
||||
bf16_ptr = (fp8_ptr + NOPE_HEAD_DIM).to(tl.pointer_type(tl.bfloat16))
|
||||
@@ -417,6 +431,7 @@ def _finalize_norm_rope_quant_store_sparse_attn(
|
||||
TOKEN_STRIDE: tl.constexpr,
|
||||
SCALE_DIM: tl.constexpr,
|
||||
KV_BLOCK_STRIDE: tl.constexpr,
|
||||
SANITIZE_CACHE_NANS: tl.constexpr,
|
||||
):
|
||||
"""Stage 2: read compressed_kv[512] from scratch buffer, then
|
||||
RMSNorm + FP8 quant (nope) + RoPE + bf16 store
|
||||
@@ -474,7 +489,8 @@ def _finalize_norm_rope_quant_store_sparse_attn(
|
||||
tl.store(fp8_ptr + block, x_uint8, mask=block < NOPE_HEAD_DIM)
|
||||
|
||||
scale_idx = tl.arange(0, N_QUANT_BLOCKS)
|
||||
encoded = tl.maximum(tl.minimum(exponents + 127.0, 255.0), 0.0)
|
||||
max_encoded: tl.constexpr = 254.0 if SANITIZE_CACHE_NANS else 255.0
|
||||
encoded = tl.maximum(tl.minimum(exponents + 127.0, max_encoded), 0.0)
|
||||
tl.store(
|
||||
scale_ptr + scale_idx, encoded.to(tl.uint8), mask=scale_idx < N_NOPE_BLOCKS
|
||||
)
|
||||
@@ -494,6 +510,8 @@ def _finalize_norm_rope_quant_store_sparse_attn(
|
||||
new_even = even * cos_v - odd * sin_v
|
||||
new_odd = odd * cos_v + even * sin_v
|
||||
result = tl.interleave(new_even, new_odd)
|
||||
if SANITIZE_CACHE_NANS:
|
||||
result = tl.where(result == result, result, 0.0)
|
||||
bf16_ptr = (fp8_ptr + NOPE_HEAD_DIM).to(tl.pointer_type(tl.bfloat16))
|
||||
rope_local = block - NOPE_HEAD_DIM
|
||||
is_rope = (block >= NOPE_HEAD_DIM) & mask
|
||||
@@ -564,6 +582,7 @@ def _launch_two_stage_sparse_attn_compressor(
|
||||
TOKEN_STRIDE=token_stride,
|
||||
SCALE_DIM=scale_dim,
|
||||
KV_BLOCK_STRIDE=kv_cache.stride(0),
|
||||
SANITIZE_CACHE_NANS=_ON_GFX950,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user