[ROCm] Fuse MLA dual RMSNorm + FP8 group quant for DeepSeek-R1 (#54857)

Signed-off-by: Ethan Ky <ethan.ky@amd.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
eky-amd
2026-10-06 11:26:38 -05:00
committed by GitHub
co-authored by Claude Fable 5
parent 4516896626
commit 29f955cce0
3 changed files with 272 additions and 4 deletions
@@ -285,3 +285,134 @@ def test_fuse_mla_dual_rms_norm_fp8_per_token(
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
class MLADualRMSNormFp8GroupTestModel(torch.nn.Module):
"""Minimal model reproducing the FP8 MLA attention path with *group* quant
(block-scaled fp8 weights, 128-wide activation groups):
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fp8_group_quant -> dequant
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm (bf16)
+-- k_pe
"""
GROUP_SIZE = 128
def __init__(
self,
hidden_size: int,
q_dim: int = Q_DIM,
kv_c_dim: int = KV_C_DIM,
k_pe_dim: int = K_PE_DIM,
eps: float = EPS,
):
super().__init__()
self.q_dim = q_dim
self.kv_dim = kv_c_dim + k_pe_dim
self.kv_c_dim = kv_c_dim
self.k_pe_dim = k_pe_dim
self.eps = eps
self.proj = torch.nn.Linear(hidden_size, q_dim + self.kv_dim, bias=False)
self.q_weight = torch.nn.Parameter(torch.ones(q_dim))
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
def _dequant(self, x_fp8: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
# Group: one (M, N / 128) scale per 128-column group of the row.
m = x_fp8.shape[0]
x = x_fp8.to(torch.float32).view(m, -1, self.GROUP_SIZE)
return (x * scale.view(m, -1, 1)).view(m, -1).to(torch.bfloat16)
def forward(self, x: torch.Tensor):
# Avoid graph input being a direct arg to a matched pattern node
x = torch.relu(x)
projected = self.proj(x)
q_c, kv_lora = projected.split([self.q_dim, self.kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([self.kv_c_dim, self.k_pe_dim], dim=-1)
q_fp8, q_scale = torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant(
q_c, self.q_weight, self.eps, self.GROUP_SIZE
)
kv_normed = self.kv_norm(kv_c)
# Also return the group scales so the test can check them directly:
# equal scales pin any fp8 payload difference to rounding alone.
return self._dequant(q_fp8, q_scale), q_scale, kv_normed, k_pe
def ops_in_model_before(self):
return [
torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant.default,
torch.ops.vllm_ir.rms_norm.default,
]
def ops_in_model_after(self):
return [torch.ops.vllm.fused_mla_dual_rms_norm_group_quant.default]
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [7168])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_fuse_mla_dual_rms_norm_fp8_group(
dtype: torch.dtype,
hidden_size: int,
monkeypatch: pytest.MonkeyPatch,
):
torch._dynamo.reset()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm"],
pass_config=PassConfig(
fuse_mla_dual_rms_norm=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
fusion_pass = MLADualRMSNormFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
model = MLADualRMSNormFp8GroupTestModel(hidden_size)
x = torch.randn(64, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
with torch.inference_mode():
outputs_unfused = model(x)
model_fused = torch.compile(model, backend=backend)
outputs_fused = model_fused(x)
q_deq_u, q_scale_u, kv_normed_u, k_pe_u = outputs_unfused
q_deq_f, q_scale_f, kv_normed_f, k_pe_f = outputs_fused
torch.testing.assert_close(k_pe_u, k_pe_f, atol=0, rtol=0)
torch.testing.assert_close(kv_normed_u, kv_normed_f, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(q_scale_u, q_scale_f, atol=0, rtol=1e-5)
E4M3_STEP = 0.125
exact_frac = (q_deq_u == q_deq_f).float().mean().item()
assert exact_frac > 0.99, f"q: only {exact_frac:.4f} of elements bit-exact"
torch.testing.assert_close(q_deq_u, q_deq_f, atol=1e-2, rtol=E4M3_STEP)
assert fusion_pass.matched_count == 1, (
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
)
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
+4
View File
@@ -2994,6 +2994,10 @@ class rocm_aiter_ops:
transpose_scale,
)
@staticmethod
def get_fused_mla_dual_rms_norm_group_quant_op() -> OpOverload:
return torch.ops.vllm.fused_mla_dual_rms_norm_group_quant.default
@staticmethod
def rms_norm(
x: torch.Tensor,
@@ -1074,16 +1074,148 @@ class MLADualRMSPerTokenQuantPattern(
return _replacement
class MLADualRMSGroupQuantPattern(
VllmPatternReplacement[
...,
tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
],
]
):
"""Fuse the MLA FP8 *block-scale* attention path -- q-latent RMSNorm + FP8
group quant plus kv-latent RMSNorm -- into AITER's
``fused_qk_rmsnorm_group_quant``.
With a block-scale FP8 ``q_b_proj`` (DeepSeek fp8), the earlier
``RocmAiterRMSNormQuantFusionPass`` folds the q side into
``rocm_aiter_rmsnorm_fp8_group_quant`` (the Triton rms+group-quant kernel)
and leaves the kv side a plain ``vllm_ir.rms_norm``. This pattern matches
that asymmetric pair::
gemm -> split_with_sizes([q_dim, kv_dim])
+-- q_c -> rocm_aiter_rmsnorm_fp8_group_quant -> (q_fp8, q_scale)
+-- kv_lora -> split_with_sizes([kv_c_dim, k_pe_dim])
+-- kv_c -> vllm_ir.rms_norm -> kv_normed (bf16)
+-- k_pe
"""
GROUP_QUANT_OP = rocm_aiter_ops.get_rmsnorm_group_fused_quant_op()
FUSED_OP = rocm_aiter_ops.get_fused_mla_dual_rms_norm_group_quant_op()
def __init__(self, epsilon: float) -> None:
self._epsilon = epsilon
def get_inputs(self) -> list[torch.Tensor]:
q_dim, kv_c_dim, k_pe_dim = 256, 128, 64
return [
self.empty_bf16(5, q_dim + kv_c_dim + k_pe_dim),
self.empty_bf16(q_dim),
self.empty_bf16(kv_c_dim),
]
@property
def pattern(
self,
) -> Callable[
...,
tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
],
]:
eps = self._epsilon
group_quant_op = self.GROUP_QUANT_OP
def _pattern(
projected: torch.Tensor,
q_weight: torch.Tensor,
kv_weight: torch.Tensor,
) -> tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
]:
q_dim = q_weight.shape[0]
kv_dim = projected.shape[-1] - q_dim
kv_c_dim = kv_weight.shape[0]
k_pe_dim = kv_dim - kv_c_dim
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
quant_kwargs = dict(
x=q_c,
weight=q_weight,
variance_epsilon=eps,
group_size=128,
)
q_quant = group_quant_op(**quant_kwargs)
kv_normed = vllm.ir.ops.rms_norm(kv_c, kv_weight, eps)
return q_quant[0], q_quant[1], kv_normed, k_pe
return _pattern
@property
def replacement(
self,
) -> Callable[
...,
tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
],
]:
eps = self._epsilon
fused_op = self.FUSED_OP
def _replacement(
projected: torch.Tensor,
q_weight: torch.Tensor,
kv_weight: torch.Tensor,
) -> tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
]:
q_dim = q_weight.shape[0]
kv_dim = projected.shape[-1] - q_dim
kv_c_dim = kv_weight.shape[0]
k_pe_dim = kv_dim - kv_c_dim
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
at = fused_op(
q_c,
q_weight,
kv_c,
kv_weight,
eps,
eps,
128,
False,
)
# q_fp8, q_scale, kv_normed, k_pe
return at[0], at[1], at[2], k_pe
return _replacement
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
"""Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
``fused_qk_rmsnorm`` HIP kernel.
The FP8 attention path is also handled via
:class:`MLADualRMSPerTokenQuantPattern`, which fuses the q-latent RMSNorm +
FP8 per-token quant together with the kv-latent RMSNorm into
``fused_mla_dual_rms_norm_per_token_quant`` backed by aiter's
``fused_qk_rmsnorm_per_token_quant`` HIP kernel.
:class:`MLADualRMSPerTokenQuantPattern` or :class:`MLADualRMSGroupQuantPattern`,
which fuse the q-latent RMSNorm + FP8 per-token/group quant together with
the kv-latent RMSNorm into ``fused_mla_dual_rms_norm_<per_token|group>_quant``
backed by aiter's ``fused_qk_rmsnorm_<per_token|group>_quant`` HIP kernel.
"""
def __init__(self, config: VllmConfig) -> None:
@@ -1092,3 +1224,4 @@ class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
for epsilon in [1e-5, 1e-6]:
self.register(MLADualRMSNormPattern(epsilon))
self.register(MLADualRMSPerTokenQuantPattern(epsilon))
self.register(MLADualRMSGroupQuantPattern(epsilon))