[ROCm][DSV4][Perf] Optimize Triton sparse-MLA decode on gfx950 (#52212)

Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Signed-off-by: fai <fangzhouai@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Cursor Grok 4.6 <cursoragent@cursor.com>
This commit is contained in:
Fangzhou Ai
2026-08-16 12:16:46 -07:00
committed by GitHub
co-authored by Nick Hill Cursor Grok 4.6
parent 1f0e0bf612
commit ef43e3101b
5 changed files with 1550 additions and 94 deletions
+48 -3
View File
@@ -19,6 +19,7 @@ from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4SparseMLAMetadataBuilder,
)
from vllm.platforms import current_platform
from vllm.platforms.rocm import _ON_GFX950
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backend import (
CommonAttentionMetadata,
@@ -36,6 +37,19 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
from vllm.v1.worker.workspace import current_workspace_manager
def _trust_dsv4_extra_cache_nan_free(
kv_cache_dtype: str,
has_kv_transfer: bool,
has_extra_cache: bool,
) -> bool:
return (
_ON_GFX950
and kv_cache_dtype == "fp8_ds_mla"
and not has_kv_transfer
and has_extra_cache
)
def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
lengths = lengths.to(dtype=torch.int32).contiguous()
indptr = torch.zeros(lengths.shape[0] + 1, dtype=torch.int32, device=lengths.device)
@@ -294,9 +308,13 @@ def _copy_ragged_to_graph_buffers(
max_entries = max(num_rows * max_entries_per_row, 1)
ragged_out = ragged_indices_buffer[:max_entries]
nnz = ragged_indices.numel()
if nnz > 0:
ragged_out[:nnz].copy_(ragged_indices, non_blocking=True)
source_entries = ragged_indices.numel()
if source_entries > 0:
ragged_out[:source_entries].copy_(ragged_indices, non_blocking=True)
if _ON_GFX950:
# Preserve the graph-stable base pointer while exposing source capacity
# to the sync-free split selector; indptr still carries the true NNZ.
ragged_out = ragged_out[: max(source_entries, 1)]
return ragged_out, indptr_out
@@ -306,6 +324,7 @@ class DeepseekV4ROCMAiterMLASparseMetadata(DeepseekV4FlashMLAMetadata):
c128a_decode_topk_ragged_indices: torch.Tensor | None = None
c128a_decode_topk_ragged_indptr: torch.Tensor | None = None
for_cudagraph_capture: bool = False
@dataclass
@@ -370,6 +389,16 @@ class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBui
c128a_decode_topk_ragged_indptr=ragged_indptr,
)
def build_for_cudagraph_capture(
self, common_attn_metadata: CommonAttentionMetadata
) -> DeepseekV4ROCMAiterMLASparseMetadata:
metadata = cast(
DeepseekV4ROCMAiterMLASparseMetadata,
super().build_for_cudagraph_capture(common_attn_metadata),
)
metadata.for_cudagraph_capture = _ON_GFX950
return metadata
class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuilder):
# Keep fused multi-step decode disabled until update_draft_decode_metadata()
@@ -451,7 +480,9 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
backend_cls = DeepseekV4ROCMAiterMLASparseBackend
def __init__(self, *args, **kwargs):
vllm_config = args[0] if args else kwargs["vllm_config"]
super().__init__(*args, **kwargs)
self._has_kv_transfer = vllm_config.kv_transfer_config is not None
# Block scale for the preshuffled weight; None = not preshuffled.
self._wqa_wkv_scale: torch.Tensor | None = None
self._wo_b_scale: torch.Tensor | None = None
@@ -605,6 +636,13 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
attn_metadata=rocm_metadata,
swa_only=swa_only,
output=output[:num_decode_tokens],
adaptive_splits=(
_ON_GFX950
and not swa_only
and self.compress_ratio == 128
and rocm_metadata is not None
and rocm_metadata.for_cudagraph_capture
),
)
def _forward_decode(
@@ -615,6 +653,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
attn_metadata: DeepseekV4ROCMAiterMLASparseMetadata | None,
swa_only: bool,
output: torch.Tensor,
adaptive_splits: bool,
) -> None:
num_decodes = swa_metadata.num_decodes
num_decode_tokens = swa_metadata.num_decode_tokens
@@ -666,6 +705,12 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
nope_head_dim=self.nope_head_dim,
rope_head_dim=self.rope_head_dim,
output=output,
adaptive_splits=adaptive_splits,
extra_cache_nan_free=_trust_dsv4_extra_cache_nan_free(
self.kv_cache_dtype,
self._has_kv_transfer,
not swa_only and kv_cache is not None,
),
)
def _forward_prefill(