[ROCm][Perf] Fix RMSNorm+Quant fusion for gfx950 (non-fnuz) (#41825)

Signed-off-by: Frida Andersson <fanderss@amd.com>
Signed-off-by: Chuan Li <chuali@amd.com>
Co-authored-by: Markus Hartikainen <markus.hartikainen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Chuan Li <chuali@amd.com>
Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Co-authored-by: Frida Andersson <frida-andersson@users.noreply.github.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
frida-andersson
2026-05-11 15:00:51 -04:00
committed by GitHub
co-authored by Markus Hartikainen Cursor Chuan Li Luka Govedič Frida Andersson TJian
parent 6fdb49392e
commit a721315488
3 changed files with 342 additions and 7 deletions
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for the DoubleQuant fan-out variants registered by
``RocmAiterRMSNormQuantFusionPass``.
Both variants target a 1-to-2 fan-out where one ``rms_norm`` output feeds
two distinct ``rocm_aiter_group_fp8_quant`` consumers and rewrite it into
two independent fused ``rms_norm + group_fp8_quant`` ops:
* ``DoubleAiterRMSFp8GroupQuantPattern`` matches the un-viewed shape
(e.g. Kimi-K2.5 / DSR1).
* ``DoubleAiterRMSFp8GroupQuantViewPattern`` (this PR) is the view-tolerant
sibling that additionally matches the
``rms_norm -> view -> group_fp8_quant`` shape that DSv3.2's MLA indexer
q_c norm exposes through ``Fp8BlockScaledMMLinearKernel.apply_weights``'s
2D-flatten boilerplate.
"""
import pytest
import torch
import vllm.config
from tests.compile.backend import TestBackend
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
EPS = 1e-5
HIDDEN_SIZE = 256
GROUP_SIZE = 128
class _NoViewDoubleQuantModel(torch.nn.Module):
"""``rms_norm -> 2x group_fp8_quant`` fan-out (Kimi-K2.5 / DSR1 shape)."""
def __init__(self) -> None:
super().__init__()
self.weight = torch.nn.Parameter(torch.ones(HIDDEN_SIZE, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# avoid graph input being a direct arg to a matched pattern node
x = torch.relu(x)
rms = torch.ops.vllm_ir.rms_norm(x, self.weight, EPS)
q1, s1 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(rms, GROUP_SIZE)
q2, s2 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(rms, GROUP_SIZE)
return q1, s1, q2, s2
class _ViewDoubleQuantModel(torch.nn.Module):
"""``rms_norm -> view -> 2x group_fp8_quant`` fan-out (DSv3.2 shape).
Reproduces the FX-graph shape produced by ``Fp8BlockScaledMMLinearKernel``'s
2D-flatten before the FP8 group quant op.
"""
def __init__(self) -> None:
super().__init__()
self.weight = torch.nn.Parameter(torch.ones(HIDDEN_SIZE, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
x = torch.relu(x)
rms = torch.ops.vllm_ir.rms_norm(x, self.weight, EPS)
view = rms.view(-1, rms.shape[-1])
q1, s1 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(view, GROUP_SIZE)
q2, s2 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(view, GROUP_SIZE)
return q1, s1, q2, s2
@pytest.mark.parametrize(
"model_cls",
[_NoViewDoubleQuantModel, _ViewDoubleQuantModel],
ids=["no_view", "with_view"],
)
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_double_aiter_rms_fp8_group_quant_fusion(
model_cls: type[torch.nn.Module],
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Both fan-out shapes (with and without an intermediate view) must fuse
into ``rocm_aiter_rmsnorm_fp8_group_quant``: the no-view shape via
``DoubleAiterRMSFp8GroupQuantPattern`` and the viewed shape via the
new ``DoubleAiterRMSFp8GroupQuantViewPattern`` sibling.
A failure on the ``with_view`` parametrization is a regression on the
DSv3.2 q_c norm path that this PR's view-tolerant pattern is intended
to cover.
"""
torch._dynamo.reset()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=torch.bfloat16),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm", "+quant_fp8"],
pass_config=PassConfig(
fuse_norm_quant=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
RocmAiterRMSNormQuantFusionPass,
)
torch.set_default_device("cuda")
torch.set_default_dtype(torch.bfloat16)
torch.manual_seed(0)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
model = model_cls()
x = torch.randn(8, HIDDEN_SIZE)
torch._dynamo.mark_dynamic(x, 0)
outputs_unfused = model(x)
model_fused = torch.compile(model, backend=backend)
outputs_fused = model_fused(x)
# Both consumers must be rewritten into the fused op (one
# ``register_replacement`` rewrite covers the whole 1-to-2 fan-out).
assert fusion_pass.matched_count == 1, (
f"Expected the {model_cls.__name__} fan-out to fuse via the "
f"DoubleQuant pattern (matched_count == 1), got "
f"{fusion_pass.matched_count}"
)
fused_op = rocm_aiter_ops.get_rmsnorm_group_fused_quant_op()
backend.check_after_ops([fused_op])
# Numerical parity sanity-check: the fused pair must match the
# unfused pair on FP8 outputs (exact byte-equality is the goal,
# but allow a tiny tolerance for any residual numeric noise).
for fused_t, unfused_t in zip(outputs_fused, outputs_unfused):
torch.testing.assert_close(
fused_t.to(torch.float32),
unfused_t.to(torch.float32),
atol=1e-2,
rtol=1e-2,
)
@@ -273,12 +273,7 @@ class MatcherQuantFP8(MatcherCustomOp):
"ROCm aiter fusion pass currently supports "
"quantization operation with group_size 128"
)
if current_platform.is_fp8_fnuz():
self.QUANT_OP = rocm_aiter_ops.get_group_quant_op()
else:
self.QUANT_OP = (
torch.ops.vllm.triton_per_token_group_quant_fp8.default
)
self.QUANT_OP = rocm_aiter_ops.get_group_quant_op()
else:
assert quant_key in QUANT_OPS, (
@@ -6,6 +6,7 @@ from typing import Any
import torch
import torch._inductor.pattern_matcher as pm
from torch import fx
from torch._inductor.fx_passes.post_grad import view_to_reshape
from torch._inductor.pattern_matcher import PatternMatcherPass
import vllm.ir.ops
@@ -293,6 +294,161 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
pm.register_replacement(pattern, replacement, inputs, pm.fwd_only, pm_pass)
class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""
Pattern matching ``rms_norm`` whose output feeds *two* distinct
``rocm_aiter_group_fp8_quant`` consumers, replacing it with two
independent fused ``rms_norm_group_fp8_quant`` ops.
Repeating the rms_norm in the replacement is preferable to leaving
the fused 16-bit rms output materialized for two unfused quant
consumers, and matches what the previous manual graph surgery
achieved by cloning the rms_norm node.
"""
FUSED_OP = rocm_aiter_ops.get_rmsnorm_group_fused_quant_op()
def __init__(
self,
epsilon: float,
quant_dtype: torch.dtype,
group_shape: GroupShape,
match_aiter_quant: bool = True,
symmetric: bool = True,
) -> None:
scale = ScaleDesc(torch.float32, False, group_shape)
key = FusedRMSQuantKey(
fused_add=False,
quant=QuantKey(dtype=quant_dtype, scale=scale, symmetric=symmetric),
)
super().__init__(epsilon, key, match_aiter_quant)
def register(self, pm_pass: PatternMatcherPass) -> None:
def pattern(
input: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms = torch.ops.vllm_ir.rms_norm(input, weight, self.epsilon)
result1, scale1 = self.quant_matcher(result_rms)
result2, scale2 = self.quant_matcher(result_rms)
return result1, scale1, result2, scale2
def replacement(
input: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
at1 = self.FUSED_OP(
x=input,
weight=weight,
variance_epsilon=self.epsilon,
group_size=128,
)
at2 = self.FUSED_OP(
x=input,
weight=weight,
variance_epsilon=self.epsilon,
group_size=128,
)
return at1[0], at1[1], at2[0], at2[1]
pm.register_replacement(
pattern,
replacement,
# input, weight
[self.empty(5, 16), self.empty(16)],
pm.fwd_only,
pm_pass,
)
class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
"""
View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
Matches the same 1-to-2 fan-out, but with a ``view``/``reshape`` between
the ``rms_norm`` output and the two ``rocm_aiter_group_fp8_quant``
consumers::
rms_norm -> view -> rocm_aiter_group_fp8_quant
\\-> view -> rocm_aiter_group_fp8_quant
This shape arises in DeepSeek-V3.2's MLA indexer q_c norm, where the
FP8 linear path's 2D-flatten boilerplate
(``Fp8BlockScaledMMLinearKernel.apply_weights``) inserts a view between
the rms_norm output and each FP8 group quant op. The non-view sibling
pattern silently no-ops on this graph because the pattern matcher
requires the in-graph and in-pattern node shapes to align.
The trace_fn runs Inductor's ``view_to_reshape`` post-grad pass to
normalize ``view`` to ``reshape`` in both the pattern and the input
graph, widening the match without touching the no-view sibling.
"""
FUSED_OP = rocm_aiter_ops.get_rmsnorm_group_fused_quant_op()
def __init__(
self,
epsilon: float,
quant_dtype: torch.dtype,
group_shape: GroupShape,
match_aiter_quant: bool = True,
symmetric: bool = True,
) -> None:
scale = ScaleDesc(torch.float32, False, group_shape)
key = FusedRMSQuantKey(
fused_add=False,
quant=QuantKey(dtype=quant_dtype, scale=scale, symmetric=symmetric),
)
super().__init__(epsilon, key, match_aiter_quant)
def register(self, pm_pass: PatternMatcherPass) -> None:
def pattern(
input: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms = torch.ops.vllm_ir.rms_norm(input, weight, self.epsilon)
view_rms = result_rms.view(-1, result_rms.shape[-1])
result1, scale1 = self.quant_matcher(view_rms)
result2, scale2 = self.quant_matcher(view_rms)
return result1, scale1, result2, scale2
def replacement(
input: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
at1 = self.FUSED_OP(
x=input,
weight=weight,
variance_epsilon=self.epsilon,
group_size=128,
)
at2 = self.FUSED_OP(
x=input,
weight=weight,
variance_epsilon=self.epsilon,
group_size=128,
)
return at1[0], at1[1], at2[0], at2[1]
def trace_with_view_to_reshape(*args: Any, **kwargs: Any) -> fx.GraphModule:
gm = pm.fwd_only(*args, **kwargs)
view_to_reshape(gm)
return gm
pm.register_replacement(
pattern,
replacement,
# input, weight
[self.empty(5, 16), self.empty(16)],
trace_with_view_to_reshape,
pm_pass,
)
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
"""
This pass fuses aiter rms_norm & vllm/aiter quant custom ops
@@ -309,8 +465,24 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
)
# Make sure fused add patterns are before simple rms norm,
# as the latter is a subset of the former in torch ops
# as the latter is a subset of the former in torch ops.
# The DoubleQuant patterns handle 1 rms_norm -> 2 group_fp8_quant
# fan-out (e.g. DSv3.2) and must be registered before the single
# group-quant pattern so they match first. The view-tolerant variant
# additionally covers the rms_norm -> view -> 2x quant shape that
# appears when the FP8 linear path inserts a 2D-flatten boilerplate
# (DSv3.2 MLA indexer q_c norm).
for epsilon in [1e-5, 1e-6]:
# Fuse aiter rms_norm + 2x aiter group fp8 quant
DoubleAiterRMSFp8GroupQuantPattern(
epsilon, FP8_DTYPE, GroupShape(1, 128)
).register(self.patterns)
# View-tolerant sibling for DSv3.2 q_c norm fan-out
DoubleAiterRMSFp8GroupQuantViewPattern(
epsilon, FP8_DTYPE, GroupShape(1, 128)
).register(self.patterns)
# Fuse aiter rms_norm + aiter dynamic group fp8 quant
AiterRMSFp8GroupQuantPattern(
epsilon, FP8_DTYPE, GroupShape(1, 128)
@@ -360,6 +532,8 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
AiterFusedAddRMSNormDynamicQuantPattern,
AiterRMSFp8GroupQuantPattern,
AiterFusedAddRMSFp8GroupQuantPattern,
DoubleAiterRMSFp8GroupQuantPattern,
DoubleAiterRMSFp8GroupQuantViewPattern,
]
return self.hash_source(self, *fusion_patterns)