[ROCm] Add MLA dual RMS norm fusion (Q, KV) pass for DeepSeek/Kimi-K2 (#39242)

Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com>
This commit is contained in:
Rita Brugarolas
2026-04-20 14:56:27 +00:00
committed by GitHub
parent b42e878ec0
commit fb5635d3f9
8 changed files with 361 additions and 1 deletions
+42
View File
@@ -962,6 +962,37 @@ def _rocm_aiter_triton_add_rmsnorm_pad_fake(
return out, residual_out
def _fused_mla_dual_rms_norm_impl(
x1: torch.Tensor,
x1_weight: torch.Tensor,
x2: torch.Tensor,
x2_weight: torch.Tensor,
x1_epsilon: float,
x2_epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor]:
from aiter.ops.fused_qk_norm_rope_cache_quant import fused_qk_rmsnorm
return fused_qk_rmsnorm(
q=x1,
q_weight=x1_weight,
q_eps=x1_epsilon,
k=x2,
k_weight=x2_weight,
k_eps=x2_epsilon,
)
def _fused_mla_dual_rms_norm_fake(
x1: torch.Tensor,
x1_weight: torch.Tensor,
x2: torch.Tensor,
x2_weight: torch.Tensor,
x1_epsilon: float,
x2_epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor]:
return (torch.empty_like(x1), torch.empty_like(x2))
def _rocm_aiter_gemm_a8wfp4_impl(
x: torch.Tensor,
w: torch.Tensor,
@@ -1491,6 +1522,13 @@ class rocm_aiter_ops:
fake_impl=_triton_rotary_embedding_fake,
)
direct_register_custom_op(
op_name="fused_mla_dual_rms_norm",
op_func=_fused_mla_dual_rms_norm_impl,
mutates_args=[],
fake_impl=_fused_mla_dual_rms_norm_fake,
)
_OPS_REGISTERED = True
@staticmethod
@@ -1537,6 +1575,10 @@ class rocm_aiter_ops:
def get_triton_rotary_embedding_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default
@staticmethod
def get_fused_mla_dual_rms_norm_op() -> OpOverload:
return torch.ops.vllm.fused_mla_dual_rms_norm.default
@staticmethod
def rms_norm(
x: torch.Tensor, weight: torch.Tensor, variance_epsilon: float