[ROCm][MLA] Enable sparse MLA Gluon kernel from Aiter (#53492)

Signed-off-by: Mehmet Cagri Kaymak <mehmet.kaymak@amd.com>
Signed-off-by: Mehmet Cagri <mehmet.kaymak@amd.com>
This commit is contained in:
Mehmet Cagri
2026-09-29 09:08:18 +08:00
committed by GitHub
parent 869278cbb0
commit 70ae0b7435
7 changed files with 989 additions and 22 deletions
+73
View File
@@ -1927,6 +1927,7 @@ class rocm_aiter_ops:
VLLM_ROCM_USE_AITER_FP8BMM: Controls FP8 batched matrix multiply.
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: Controls Triton sparse MLA (gfx950).
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation
dtype (a16w4 | a8w4 | a4w4).
@@ -1998,6 +1999,7 @@ class rocm_aiter_ops:
_FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
_TRITON_SPARSE_MLA = envs.VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
_MOE_SITUV2 = _resolve_situv2_activation()
# TODO: Consolidate under _LINEAR_ENABLED
@@ -2027,6 +2029,7 @@ class rocm_aiter_ops:
cls._LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
cls._TRITON_SPARSE_MLA = envs.VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
cls._MOE_SITUV2 = _resolve_situv2_activation()
_sync_aiter_situv2_moe_env()
@@ -2391,6 +2394,25 @@ class rocm_aiter_ops:
def is_triton_rotary_embed_enabled(cls) -> bool:
return cls._AITER_ENABLED and cls._TRITON_ROTARY_EMBED
@classmethod
@if_aiter_supported
def is_triton_sparse_mla_enabled(cls) -> bool:
if not cls._TRITON_SPARSE_MLA:
return False
from vllm.platforms.rocm import on_gfx950
if not cls._AITER_ENABLED:
reason = "VLLM_ROCM_USE_AITER is off"
elif not on_gfx950():
reason = "the kernel is gfx950-only"
else:
return True
logger.warning_once(
"VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA is set, but %s; ignoring it.",
reason,
)
return False
@classmethod
@if_aiter_supported
def is_triton_gemm_enabled(cls) -> bool:
@@ -3137,6 +3159,57 @@ class rocm_aiter_ops:
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops.vllm.rocm_aiter_fused_topk(x, router_logits, top_k, gate_up)
@staticmethod
def triton_sparse_mla_fwd(
q: torch.Tensor,
kv_buffer: torch.Tensor,
o: torch.Tensor,
sm_scale: float,
kv_indptr: torch.Tensor,
kv_indices: torch.Tensor,
kv_lora_rank: int = 512,
qk_rope_head_dim: int = 64,
q_scale: torch.Tensor | None = None,
kv_scale: torch.Tensor | None = None,
attn_sink: torch.Tensor | None = None,
extra_kv_buffer: torch.Tensor | None = None,
extra_kv_indptr: torch.Tensor | None = None,
extra_kv_indices: torch.Tensor | None = None,
has_invalid: bool = True,
) -> None:
"""Sparse MLA read straight from the KV cache, for prefill and decode.
kv_indices are ragged global slot ids. With has_invalid they may hold
-1, which the kernel masks; a caller whose stream never holds a
negative slot passes False, which lets the fp8 kernel stage K straight
into LDS. The cache format (bf16, per-tensor fp8, or DeepSeek V4's
paged fp8_ds_mla) is inferred from kv_buffer and kv_scale. An fp8 q
must come with its q_scale and runs both dots in fp8. The extra segment
is DeepSeek V4's second cache: SWA window in kv_buffer, top-k
compressed tokens here.
"""
from aiter.ops.triton.attention.sparse_mla import sparse_mla_fwd
fp8_q = q.dtype == FP8_DTYPE
sparse_mla_fwd(
q,
kv_buffer,
kv_indptr,
kv_indices,
sm_scale,
kv_scale=kv_scale,
kv_lora_rank=kv_lora_rank,
qk_rope_head_dim=qk_rope_head_dim,
has_invalid=has_invalid,
dot_precision="fp8" if fp8_q else "bf16",
q_scale=q_scale if fp8_q else None,
out=o,
attn_sink=attn_sink,
extra_kv=extra_kv_buffer,
extra_indptr=extra_kv_indptr,
extra_indices=extra_kv_indices,
)
@staticmethod
def mla_decode_fwd(
q: torch.Tensor,