[ROCm] Add MLA dual RMS norm fusion (Q, KV) pass for DeepSeek/Kimi-K2 (#39242)
Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com>
This commit is contained in:
@@ -962,6 +962,37 @@ def _rocm_aiter_triton_add_rmsnorm_pad_fake(
|
||||
return out, residual_out
|
||||
|
||||
|
||||
def _fused_mla_dual_rms_norm_impl(
|
||||
x1: torch.Tensor,
|
||||
x1_weight: torch.Tensor,
|
||||
x2: torch.Tensor,
|
||||
x2_weight: torch.Tensor,
|
||||
x1_epsilon: float,
|
||||
x2_epsilon: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from aiter.ops.fused_qk_norm_rope_cache_quant import fused_qk_rmsnorm
|
||||
|
||||
return fused_qk_rmsnorm(
|
||||
q=x1,
|
||||
q_weight=x1_weight,
|
||||
q_eps=x1_epsilon,
|
||||
k=x2,
|
||||
k_weight=x2_weight,
|
||||
k_eps=x2_epsilon,
|
||||
)
|
||||
|
||||
|
||||
def _fused_mla_dual_rms_norm_fake(
|
||||
x1: torch.Tensor,
|
||||
x1_weight: torch.Tensor,
|
||||
x2: torch.Tensor,
|
||||
x2_weight: torch.Tensor,
|
||||
x1_epsilon: float,
|
||||
x2_epsilon: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return (torch.empty_like(x1), torch.empty_like(x2))
|
||||
|
||||
|
||||
def _rocm_aiter_gemm_a8wfp4_impl(
|
||||
x: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
@@ -1491,6 +1522,13 @@ class rocm_aiter_ops:
|
||||
fake_impl=_triton_rotary_embedding_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_mla_dual_rms_norm",
|
||||
op_func=_fused_mla_dual_rms_norm_impl,
|
||||
mutates_args=[],
|
||||
fake_impl=_fused_mla_dual_rms_norm_fake,
|
||||
)
|
||||
|
||||
_OPS_REGISTERED = True
|
||||
|
||||
@staticmethod
|
||||
@@ -1537,6 +1575,10 @@ class rocm_aiter_ops:
|
||||
def get_triton_rotary_embedding_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default
|
||||
|
||||
@staticmethod
|
||||
def get_fused_mla_dual_rms_norm_op() -> OpOverload:
|
||||
return torch.ops.vllm.fused_mla_dual_rms_norm.default
|
||||
|
||||
@staticmethod
|
||||
def rms_norm(
|
||||
x: torch.Tensor, weight: torch.Tensor, variance_epsilon: float
|
||||
|
||||
Reference in New Issue
Block a user