[ROCm][MLA] Fuse MLA q/kv RMSNorm + FP8 per-token quant in the FP8 attention path (#44977)

Signed-off-by: Xavier Aguilar <xavier.aguilarfruto@amd.com>
Signed-off-by: Xavier Aguilar <Xavier.AguilarFruto@amd.com>
This commit is contained in:
xaguilar-amd
2026-07-02 13:00:41 +08:00
committed by GitHub
parent d63c8e9444
commit 09663abde0
4 changed files with 371 additions and 5 deletions
+68
View File
@@ -1355,6 +1355,63 @@ def _fused_mla_dual_rms_norm_fake(
return (torch.empty_like(x1), torch.empty_like(x2))
def _fused_mla_dual_rms_norm_per_token_quant_impl(
q: torch.Tensor,
q_weight: torch.Tensor,
kv: torch.Tensor,
kv_weight: torch.Tensor,
q_epsilon: float,
kv_epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fused MLA q/kv RMSNorm (+ FP8 per-token quant on q) via AITER.
Backs the ``fused_mla_dual_rms_norm_per_token_quant`` custom op used by the
MLA FP8 attention fusion when the q latent is quantized *per token* (a single
``(M, 1)`` scale). Only the *q* latent is FP8 quantized (it feeds the
FP8 ``q_b_proj`` GEMM); the *kv* latent is RMS-normed and consumed by attention as bf16.
"""
from aiter.ops.fused_qk_rmsnorm_group_quant import (
fused_qk_rmsnorm_per_token_quant,
)
mq, nq = q.shape
q_out = torch.empty((mq, nq), dtype=FP8_DTYPE, device=q.device)
q_scale = torch.empty((mq, 1), dtype=torch.float32, device=q.device)
kv_normed = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
# q -> RMSNorm + FP8 per-token quant (q slot); kv -> RMSNorm only (k slot).
# `split` views are accepted directly (unit inner stride); the kernel
# handles strided inputs, matching the aiter op-test usage.
fused_qk_rmsnorm_per_token_quant(
q_out_quantized=q_out,
q_out_scale=q_scale,
q=q,
q_weight=q_weight,
q_epsilon=q_epsilon,
k_out=kv_normed,
k=kv,
k_weight=kv_weight,
k_epsilon=kv_epsilon,
gemma_norm=False,
)
return q_out, q_scale, kv_normed
def _fused_mla_dual_rms_norm_per_token_quant_fake(
q: torch.Tensor,
q_weight: torch.Tensor,
kv: torch.Tensor,
kv_weight: torch.Tensor,
q_epsilon: float,
kv_epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
mq, nq = q.shape
q_out = torch.empty((mq, nq), dtype=FP8_DTYPE, device=q.device)
q_scale = torch.empty((mq, 1), dtype=torch.float32, device=q.device)
kv_normed = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
return q_out, q_scale, kv_normed
def _rocm_aiter_gemm_a8wfp4_impl(
x: torch.Tensor,
w: torch.Tensor,
@@ -2046,6 +2103,13 @@ class rocm_aiter_ops:
fake_impl=_fused_mla_dual_rms_norm_fake,
)
direct_register_custom_op(
op_name="fused_mla_dual_rms_norm_per_token_quant",
op_func=_fused_mla_dual_rms_norm_per_token_quant_impl,
mutates_args=[],
fake_impl=_fused_mla_dual_rms_norm_per_token_quant_fake,
)
_OPS_REGISTERED = True
@staticmethod
@@ -2120,6 +2184,10 @@ class rocm_aiter_ops:
def get_fused_mla_dual_rms_norm_op() -> OpOverload:
return torch.ops.vllm.fused_mla_dual_rms_norm.default
@staticmethod
def get_fused_mla_dual_rms_norm_per_token_quant_op() -> OpOverload:
return torch.ops.vllm.fused_mla_dual_rms_norm_per_token_quant.default
@staticmethod
def w8a8_gemm(
A: torch.Tensor,
@@ -922,11 +922,145 @@ class MLADualRMSNormPattern(
return _replacement
class MLADualRMSPerTokenQuantPattern(
VllmPatternReplacement[
...,
tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
],
]
):
"""
Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
plus kv-latent RMSNorm -- into AITER's ``fused_qk_rmsnorm_per_token_quant``.
With a per-token FP8 ``q_b_proj`` (Quark / ModelOpt), the earlier
``RocmAiterRMSNormQuantFusionPass`` folds the q side into
``rocm_aiter_rmsnorm_fused_dynamic_quant`` and leaves the kv side a plain
``vllm_ir.rms_norm``. This pattern matches that asymmetric pair::
gemm -> split_with_sizes([q_dim, kv_dim])
+-- q_c -> rocm_aiter_rmsnorm_fused_dynamic_quant -> (q_fp8, q_scale)
+-- kv_lora -> split_with_sizes([kv_c_dim, k_pe_dim])
+-- kv_c -> vllm_ir.rms_norm -> kv_normed (bf16)
+-- k_pe
"""
DYNAMIC_QUANT_OP = rocm_aiter_ops.get_rmsnorm_fused_dynamic_quant_op()
FUSED_OP = rocm_aiter_ops.get_fused_mla_dual_rms_norm_per_token_quant_op()
def __init__(self, epsilon: float) -> None:
self._epsilon = epsilon
def get_inputs(self) -> list[torch.Tensor]:
q_dim, kv_c_dim, k_pe_dim = 256, 128, 64
return [
self.empty_bf16(5, q_dim + kv_c_dim + k_pe_dim),
self.empty_bf16(q_dim),
self.empty_bf16(kv_c_dim),
]
@property
def pattern(
self,
) -> Callable[
...,
tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
],
]:
eps = self._epsilon
dynamic_quant_op = self.DYNAMIC_QUANT_OP
def _pattern(
projected: torch.Tensor,
q_weight: torch.Tensor,
kv_weight: torch.Tensor,
) -> tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
]:
q_dim = q_weight.shape[0]
kv_dim = projected.shape[-1] - q_dim
kv_c_dim = kv_weight.shape[0]
k_pe_dim = kv_dim - kv_c_dim
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
q_quant = dynamic_quant_op(
x=q_c,
weight=q_weight,
epsilon=eps,
quant_dtype=FP8_DTYPE,
)
kv_normed = vllm.ir.ops.rms_norm(kv_c, kv_weight, eps)
return q_quant[0], q_quant[1], kv_normed, k_pe
return _pattern
@property
def replacement(
self,
) -> Callable[
...,
tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
],
]:
eps = self._epsilon
fused_op = self.FUSED_OP
def _replacement(
projected: torch.Tensor,
q_weight: torch.Tensor,
kv_weight: torch.Tensor,
) -> tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
]:
q_dim = q_weight.shape[0]
kv_dim = projected.shape[-1] - q_dim
kv_c_dim = kv_weight.shape[0]
k_pe_dim = kv_dim - kv_c_dim
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
at = fused_op(
q_c,
q_weight,
kv_c,
kv_weight,
eps,
eps,
)
# q_fp8, q_scale, kv_normed, k_pe
return at[0], at[1], at[2], k_pe
return _replacement
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
"""
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
``fused_qk_rmsnorm`` HIP kernel.
The FP8 attention path is also handled via
:class:`MLADualRMSPerTokenQuantPattern`, which fuses the q-latent RMSNorm +
FP8 per-token quant together with the kv-latent RMSNorm into
``fused_mla_dual_rms_norm_per_token_quant`` backed by aiter's
``fused_qk_rmsnorm_per_token_quant`` HIP kernel.
"""
def __init__(self, config: VllmConfig) -> None:
@@ -934,3 +1068,4 @@ class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
for epsilon in [1e-5, 1e-6]:
self.register(MLADualRMSNormPattern(epsilon))
self.register(MLADualRMSPerTokenQuantPattern(epsilon))