[ROCm] Add MLA dual RMS norm fusion (Q, KV) pass for DeepSeek/Kimi-K2 (#39242)
Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -7,6 +8,7 @@ import torch._inductor.pattern_matcher as pm
|
||||
from torch import fx
|
||||
from torch._inductor.pattern_matcher import PatternMatcherPass
|
||||
|
||||
import vllm.ir.ops
|
||||
import vllm.model_executor.layers.quantization.utils.fp8_utils # noqa: F401
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
@@ -20,7 +22,12 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..inductor_pass import enable_fake_mode
|
||||
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
from ..vllm_inductor_pass import (
|
||||
VllmFusionPatternMatcherPass,
|
||||
VllmInductorPass,
|
||||
VllmPatternMatcherPass,
|
||||
VllmPatternReplacement,
|
||||
)
|
||||
from .act_quant_fusion import ActivationQuantPattern
|
||||
from .matcher_utils import (
|
||||
MatcherFusedAddRMSNorm,
|
||||
@@ -512,3 +519,101 @@ class RocmAiterTritonAddRMSNormPadFusionPass(VllmPatternMatcherPass):
|
||||
|
||||
def uuid(self) -> str:
|
||||
return VllmInductorPass.hash_source(self, AddAiterRMSNormPadPattern)
|
||||
|
||||
|
||||
class MLADualRMSNormPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Fuse paired q_a_layernorm + kv_a_layernorm in MLA attention into
|
||||
AITER's ``fused_qk_rmsnorm`` HIP kernel.
|
||||
|
||||
Target FX-graph pattern (unfused, ``vllm_ir`` stage)::
|
||||
|
||||
gemm -> split_with_sizes([q_dim, kv_dim])
|
||||
+-- q_c -> vllm_ir.rms_norm(q_c, q_w, eps)
|
||||
+-- kv_lora -> split_with_sizes([kv_c_dim, k_pe_dim])
|
||||
+-- kv_c -> vllm_ir.rms_norm(kv_c, kv_w, eps)
|
||||
+-- k_pe
|
||||
|
||||
The pattern covers the connected subgraph rooted at the first
|
||||
``split_with_sizes`` (which produces ``q_c`` and ``kv_lora``),
|
||||
through the two ``rms_norm`` calls, and the ``k_pe`` passthrough.
|
||||
"""
|
||||
|
||||
def __init__(self, epsilon: float) -> None:
|
||||
self._epsilon = epsilon
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
q_dim, kv_c_dim, k_pe_dim = 8, 4, 2
|
||||
return [
|
||||
self.empty_bf16(5, q_dim + kv_c_dim + k_pe_dim),
|
||||
self.empty_bf16(q_dim),
|
||||
self.empty_bf16(kv_c_dim),
|
||||
]
|
||||
|
||||
@property
|
||||
def pattern(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
|
||||
eps = self._epsilon
|
||||
|
||||
def _pattern(
|
||||
projected: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
kv_weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
q_dim = q_weight.shape[0]
|
||||
kv_dim = projected.shape[-1] - q_dim
|
||||
kv_c_dim = kv_weight.shape[0]
|
||||
k_pe_dim = kv_dim - kv_c_dim
|
||||
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
|
||||
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
|
||||
q_normed = vllm.ir.ops.rms_norm(q_c, q_weight, eps)
|
||||
kv_normed = vllm.ir.ops.rms_norm(kv_c, kv_weight, eps)
|
||||
return q_normed, kv_normed, k_pe
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(
|
||||
self,
|
||||
) -> Callable[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]:
|
||||
eps = self._epsilon
|
||||
|
||||
def _replacement(
|
||||
projected: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
kv_weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
q_dim = q_weight.shape[0]
|
||||
kv_dim = projected.shape[-1] - q_dim
|
||||
kv_c_dim = kv_weight.shape[0]
|
||||
k_pe_dim = kv_dim - kv_c_dim
|
||||
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
|
||||
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
|
||||
q_normed, kv_normed = torch.ops.vllm.fused_mla_dual_rms_norm(
|
||||
q_c,
|
||||
q_weight,
|
||||
kv_c,
|
||||
kv_weight,
|
||||
eps,
|
||||
eps,
|
||||
)
|
||||
return q_normed, kv_normed, k_pe
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
|
||||
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
|
||||
``fused_qk_rmsnorm`` HIP kernel.
|
||||
"""
|
||||
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "mla_dual_rms_norm_fusion_pass")
|
||||
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
self.register(MLADualRMSNormPattern(epsilon))
|
||||
|
||||
@@ -19,6 +19,7 @@ from .vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
from .fusion.rocm_aiter_fusion import (
|
||||
MLADualRMSNormFusionPass,
|
||||
RocmAiterRMSNormQuantFusionPass,
|
||||
RocmAiterSiluMulFp8GroupQuantFusionPass,
|
||||
RocmAiterTritonAddRMSNormPadFusionPass,
|
||||
@@ -155,6 +156,9 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
if self.pass_config.fuse_act_padding and rocm_aiter_ops.is_enabled():
|
||||
self.passes += [RocmAiterTritonAddRMSNormPadFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_mla_dual_rms_norm and rocm_aiter_ops.is_enabled():
|
||||
self.passes += [MLADualRMSNormFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_rope_kvcache:
|
||||
self.passes += [SplitCoalescingPass(config)]
|
||||
self.passes += [ScatterSplitReplacementPass(config)]
|
||||
|
||||
Reference in New Issue
Block a user