[ROCm] Add MLA dual RMS norm fusion (Q, KV) pass for DeepSeek/Kimi-K2 (#39242)

Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com>
This commit is contained in:
Rita Brugarolas
2026-04-20 14:56:27 +00:00
committed by GitHub
parent b42e878ec0
commit fb5635d3f9
8 changed files with 361 additions and 1 deletions
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from typing import Any
import torch
@@ -7,6 +8,7 @@ import torch._inductor.pattern_matcher as pm
from torch import fx
from torch._inductor.pattern_matcher import PatternMatcherPass
import vllm.ir.ops
import vllm.model_executor.layers.quantization.utils.fp8_utils # noqa: F401
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
@@ -20,7 +22,12 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.platforms import current_platform
from ..inductor_pass import enable_fake_mode
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
from ..vllm_inductor_pass import (
VllmFusionPatternMatcherPass,
VllmInductorPass,
VllmPatternMatcherPass,
VllmPatternReplacement,
)
from .act_quant_fusion import ActivationQuantPattern
from .matcher_utils import (
MatcherFusedAddRMSNorm,
@@ -512,3 +519,101 @@ class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
def uuid(self) -> str:
return VllmInductorPass.hash_source(self, AddAiterRMSNormPadPattern)
class MLADualRMSNormPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]
):
"""
Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
AITER's ``fused_qk_rmsnorm`` HIP kernel.
Target FX-graph pattern (unfused, ``vllm_ir`` stage)::
gemm -> split_with_sizes([q_dim, kv_dim])
+-- q_c -> vllm_ir.rms_norm(q_c, q_w, eps)
+-- kv_lora -> split_with_sizes([kv_c_dim, k_pe_dim])
+-- kv_c -> vllm_ir.rms_norm(kv_c, kv_w, eps)
+-- k_pe
The pattern covers the connected subgraph rooted at the first
``split_with_sizes`` (which produces ``q_c`` and ``kv_lora``),
through the two ``rms_norm`` calls, and the ``k_pe`` passthrough.
"""
def __init__(self, epsilon: float) -> None:
self._epsilon = epsilon
def get_inputs(self) -> list[torch.Tensor]:
q_dim, kv_c_dim, k_pe_dim = 8, 4, 2
return [
self.empty_bf16(5, q_dim + kv_c_dim + k_pe_dim),
self.empty_bf16(q_dim),
self.empty_bf16(kv_c_dim),
]
@property
def pattern(
self,
) -> Callable[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
eps = self._epsilon
def _pattern(
projected: torch.Tensor,
q_weight: torch.Tensor,
kv_weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
q_dim = q_weight.shape[0]
kv_dim = projected.shape[-1] - q_dim
kv_c_dim = kv_weight.shape[0]
k_pe_dim = kv_dim - kv_c_dim
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
q_normed = vllm.ir.ops.rms_norm(q_c, q_weight, eps)
kv_normed = vllm.ir.ops.rms_norm(kv_c, kv_weight, eps)
return q_normed, kv_normed, k_pe
return _pattern
@property
def replacement(
self,
) -> Callable[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
eps = self._epsilon
def _replacement(
projected: torch.Tensor,
q_weight: torch.Tensor,
kv_weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
q_dim = q_weight.shape[0]
kv_dim = projected.shape[-1] - q_dim
kv_c_dim = kv_weight.shape[0]
k_pe_dim = kv_dim - kv_c_dim
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
q_normed, kv_normed = torch.ops.vllm.fused_mla_dual_rms_norm(
q_c,
q_weight,
kv_c,
kv_weight,
eps,
eps,
)
return q_normed, kv_normed, k_pe
return _replacement
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
"""
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
``fused_qk_rmsnorm`` HIP kernel.
"""
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "mla_dual_rms_norm_fusion_pass")
for epsilon in [1e-5, 1e-6]:
self.register(MLADualRMSNormPattern(epsilon))
+4
View File
@@ -19,6 +19,7 @@ from .vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
if rocm_aiter_ops.is_enabled():
from .fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
RocmAiterRMSNormQuantFusionPass,
RocmAiterSiluMulFp8GroupQuantFusionPass,
RocmAiterTritonAddRMSNormPadFusionPass,
@@ -155,6 +156,9 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
if self.pass_config.fuse_act_padding and rocm_aiter_ops.is_enabled():
self.passes += [RocmAiterTritonAddRMSNormPadFusionPass(config)]
if self.pass_config.fuse_mla_dual_rms_norm and rocm_aiter_ops.is_enabled():
self.passes += [MLADualRMSNormFusionPass(config)]
if self.pass_config.fuse_rope_kvcache:
self.passes += [SplitCoalescingPass(config)]
self.passes += [ScatterSplitReplacementPass(config)]