[ROCm][Perf] Fix RMSNorm+Quant fusion for gfx950 (non-fnuz) (#41825)
Signed-off-by: Frida Andersson <fanderss@amd.com> Signed-off-by: Chuan Li <chuali@amd.com> Co-authored-by: Markus Hartikainen <markus.hartikainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Chuan Li <chuali@amd.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Frida Andersson <frida-andersson@users.noreply.github.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
Markus Hartikainen
Cursor
Chuan Li
Luka Govedič
Frida Andersson
TJian
parent
6fdb49392e
commit
a721315488
@@ -0,0 +1,166 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for the DoubleQuant fan-out variants registered by
|
||||
``RocmAiterRMSNormQuantFusionPass``.
|
||||
|
||||
Both variants target a 1-to-2 fan-out where one ``rms_norm`` output feeds
|
||||
two distinct ``rocm_aiter_group_fp8_quant`` consumers and rewrite it into
|
||||
two independent fused ``rms_norm + group_fp8_quant`` ops:
|
||||
|
||||
* ``DoubleAiterRMSFp8GroupQuantPattern`` matches the un-viewed shape
|
||||
(e.g. Kimi-K2.5 / DSR1).
|
||||
* ``DoubleAiterRMSFp8GroupQuantViewPattern`` (this PR) is the view-tolerant
|
||||
sibling that additionally matches the
|
||||
``rms_norm -> view -> group_fp8_quant`` shape that DSv3.2's MLA indexer
|
||||
q_c norm exposes through ``Fp8BlockScaledMMLinearKernel.apply_weights``'s
|
||||
2D-flatten boilerplate.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.config
|
||||
from tests.compile.backend import TestBackend
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
|
||||
EPS = 1e-5
|
||||
HIDDEN_SIZE = 256
|
||||
GROUP_SIZE = 128
|
||||
|
||||
|
||||
class _NoViewDoubleQuantModel(torch.nn.Module):
|
||||
"""``rms_norm -> 2x group_fp8_quant`` fan-out (Kimi-K2.5 / DSR1 shape)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.ones(HIDDEN_SIZE, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# avoid graph input being a direct arg to a matched pattern node
|
||||
x = torch.relu(x)
|
||||
rms = torch.ops.vllm_ir.rms_norm(x, self.weight, EPS)
|
||||
q1, s1 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(rms, GROUP_SIZE)
|
||||
q2, s2 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(rms, GROUP_SIZE)
|
||||
return q1, s1, q2, s2
|
||||
|
||||
|
||||
class _ViewDoubleQuantModel(torch.nn.Module):
|
||||
"""``rms_norm -> view -> 2x group_fp8_quant`` fan-out (DSv3.2 shape).
|
||||
|
||||
Reproduces the FX-graph shape produced by ``Fp8BlockScaledMMLinearKernel``'s
|
||||
2D-flatten before the FP8 group quant op.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.ones(HIDDEN_SIZE, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
x = torch.relu(x)
|
||||
rms = torch.ops.vllm_ir.rms_norm(x, self.weight, EPS)
|
||||
view = rms.view(-1, rms.shape[-1])
|
||||
q1, s1 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(view, GROUP_SIZE)
|
||||
q2, s2 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(view, GROUP_SIZE)
|
||||
return q1, s1, q2, s2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_cls",
|
||||
[_NoViewDoubleQuantModel, _ViewDoubleQuantModel],
|
||||
ids=["no_view", "with_view"],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="Only test on ROCm with AITER installed and supported",
|
||||
)
|
||||
def test_double_aiter_rms_fp8_group_quant_fusion(
|
||||
model_cls: type[torch.nn.Module],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""
|
||||
Both fan-out shapes (with and without an intermediate view) must fuse
|
||||
into ``rocm_aiter_rmsnorm_fp8_group_quant``: the no-view shape via
|
||||
``DoubleAiterRMSFp8GroupQuantPattern`` and the viewed shape via the
|
||||
new ``DoubleAiterRMSFp8GroupQuantViewPattern`` sibling.
|
||||
|
||||
A failure on the ``with_view`` parametrization is a regression on the
|
||||
DSv3.2 q_c norm path that this PR's view-tolerant pattern is intended
|
||||
to cover.
|
||||
"""
|
||||
torch._dynamo.reset()
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=torch.bfloat16),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["+rms_norm", "+quant_fp8"],
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
RocmAiterRMSNormQuantFusionPass,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
torch.manual_seed(0)
|
||||
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
|
||||
passes = [
|
||||
NoOpEliminationPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
]
|
||||
backend = TestBackend(*passes)
|
||||
model = model_cls()
|
||||
|
||||
x = torch.randn(8, HIDDEN_SIZE)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
|
||||
outputs_unfused = model(x)
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
outputs_fused = model_fused(x)
|
||||
|
||||
# Both consumers must be rewritten into the fused op (one
|
||||
# ``register_replacement`` rewrite covers the whole 1-to-2 fan-out).
|
||||
assert fusion_pass.matched_count == 1, (
|
||||
f"Expected the {model_cls.__name__} fan-out to fuse via the "
|
||||
f"DoubleQuant pattern (matched_count == 1), got "
|
||||
f"{fusion_pass.matched_count}"
|
||||
)
|
||||
|
||||
fused_op = rocm_aiter_ops.get_rmsnorm_group_fused_quant_op()
|
||||
backend.check_after_ops([fused_op])
|
||||
|
||||
# Numerical parity sanity-check: the fused pair must match the
|
||||
# unfused pair on FP8 outputs (exact byte-equality is the goal,
|
||||
# but allow a tiny tolerance for any residual numeric noise).
|
||||
for fused_t, unfused_t in zip(outputs_fused, outputs_unfused):
|
||||
torch.testing.assert_close(
|
||||
fused_t.to(torch.float32),
|
||||
unfused_t.to(torch.float32),
|
||||
atol=1e-2,
|
||||
rtol=1e-2,
|
||||
)
|
||||
@@ -273,12 +273,7 @@ class MatcherQuantFP8(MatcherCustomOp):
|
||||
"ROCm aiter fusion pass currently supports "
|
||||
"quantization operation with group_size 128"
|
||||
)
|
||||
if current_platform.is_fp8_fnuz():
|
||||
self.QUANT_OP = rocm_aiter_ops.get_group_quant_op()
|
||||
else:
|
||||
self.QUANT_OP = (
|
||||
torch.ops.vllm.triton_per_token_group_quant_fp8.default
|
||||
)
|
||||
self.QUANT_OP = rocm_aiter_ops.get_group_quant_op()
|
||||
|
||||
else:
|
||||
assert quant_key in QUANT_OPS, (
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any
|
||||
import torch
|
||||
import torch._inductor.pattern_matcher as pm
|
||||
from torch import fx
|
||||
from torch._inductor.fx_passes.post_grad import view_to_reshape
|
||||
from torch._inductor.pattern_matcher import PatternMatcherPass
|
||||
|
||||
import vllm.ir.ops
|
||||
@@ -293,6 +294,161 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
pm.register_replacement(pattern, replacement, inputs, pm.fwd_only, pm_pass)
|
||||
|
||||
|
||||
class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
Pattern matching ``rms_norm`` whose output feeds *two* distinct
|
||||
``rocm_aiter_group_fp8_quant`` consumers, replacing it with two
|
||||
independent fused ``rms_norm_group_fp8_quant`` ops.
|
||||
|
||||
Repeating the rms_norm in the replacement is preferable to leaving
|
||||
the fused 16-bit rms output materialized for two unfused quant
|
||||
consumers, and matches what the previous manual graph surgery
|
||||
achieved by cloning the rms_norm node.
|
||||
"""
|
||||
|
||||
FUSED_OP = rocm_aiter_ops.get_rmsnorm_group_fused_quant_op()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
quant_dtype: torch.dtype,
|
||||
group_shape: GroupShape,
|
||||
match_aiter_quant: bool = True,
|
||||
symmetric: bool = True,
|
||||
) -> None:
|
||||
scale = ScaleDesc(torch.float32, False, group_shape)
|
||||
key = FusedRMSQuantKey(
|
||||
fused_add=False,
|
||||
quant=QuantKey(dtype=quant_dtype, scale=scale, symmetric=symmetric),
|
||||
)
|
||||
|
||||
super().__init__(epsilon, key, match_aiter_quant)
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
def pattern(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
result_rms = torch.ops.vllm_ir.rms_norm(input, weight, self.epsilon)
|
||||
result1, scale1 = self.quant_matcher(result_rms)
|
||||
result2, scale2 = self.quant_matcher(result_rms)
|
||||
return result1, scale1, result2, scale2
|
||||
|
||||
def replacement(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
at1 = self.FUSED_OP(
|
||||
x=input,
|
||||
weight=weight,
|
||||
variance_epsilon=self.epsilon,
|
||||
group_size=128,
|
||||
)
|
||||
at2 = self.FUSED_OP(
|
||||
x=input,
|
||||
weight=weight,
|
||||
variance_epsilon=self.epsilon,
|
||||
group_size=128,
|
||||
)
|
||||
|
||||
return at1[0], at1[1], at2[0], at2[1]
|
||||
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
replacement,
|
||||
# input, weight
|
||||
[self.empty(5, 16), self.empty(16)],
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
)
|
||||
|
||||
|
||||
class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
|
||||
"""
|
||||
View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``.
|
||||
|
||||
Matches the same 1-to-2 fan-out, but with a ``view``/``reshape`` between
|
||||
the ``rms_norm`` output and the two ``rocm_aiter_group_fp8_quant``
|
||||
consumers::
|
||||
|
||||
rms_norm -> view -> rocm_aiter_group_fp8_quant
|
||||
\\-> view -> rocm_aiter_group_fp8_quant
|
||||
|
||||
This shape arises in DeepSeek-V3.2's MLA indexer q_c norm, where the
|
||||
FP8 linear path's 2D-flatten boilerplate
|
||||
(``Fp8BlockScaledMMLinearKernel.apply_weights``) inserts a view between
|
||||
the rms_norm output and each FP8 group quant op. The non-view sibling
|
||||
pattern silently no-ops on this graph because the pattern matcher
|
||||
requires the in-graph and in-pattern node shapes to align.
|
||||
|
||||
The trace_fn runs Inductor's ``view_to_reshape`` post-grad pass to
|
||||
normalize ``view`` to ``reshape`` in both the pattern and the input
|
||||
graph, widening the match without touching the no-view sibling.
|
||||
"""
|
||||
|
||||
FUSED_OP = rocm_aiter_ops.get_rmsnorm_group_fused_quant_op()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
quant_dtype: torch.dtype,
|
||||
group_shape: GroupShape,
|
||||
match_aiter_quant: bool = True,
|
||||
symmetric: bool = True,
|
||||
) -> None:
|
||||
scale = ScaleDesc(torch.float32, False, group_shape)
|
||||
key = FusedRMSQuantKey(
|
||||
fused_add=False,
|
||||
quant=QuantKey(dtype=quant_dtype, scale=scale, symmetric=symmetric),
|
||||
)
|
||||
|
||||
super().__init__(epsilon, key, match_aiter_quant)
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
def pattern(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
result_rms = torch.ops.vllm_ir.rms_norm(input, weight, self.epsilon)
|
||||
view_rms = result_rms.view(-1, result_rms.shape[-1])
|
||||
result1, scale1 = self.quant_matcher(view_rms)
|
||||
result2, scale2 = self.quant_matcher(view_rms)
|
||||
return result1, scale1, result2, scale2
|
||||
|
||||
def replacement(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
at1 = self.FUSED_OP(
|
||||
x=input,
|
||||
weight=weight,
|
||||
variance_epsilon=self.epsilon,
|
||||
group_size=128,
|
||||
)
|
||||
at2 = self.FUSED_OP(
|
||||
x=input,
|
||||
weight=weight,
|
||||
variance_epsilon=self.epsilon,
|
||||
group_size=128,
|
||||
)
|
||||
|
||||
return at1[0], at1[1], at2[0], at2[1]
|
||||
|
||||
def trace_with_view_to_reshape(*args: Any, **kwargs: Any) -> fx.GraphModule:
|
||||
gm = pm.fwd_only(*args, **kwargs)
|
||||
view_to_reshape(gm)
|
||||
return gm
|
||||
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
replacement,
|
||||
# input, weight
|
||||
[self.empty(5, 16), self.empty(16)],
|
||||
trace_with_view_to_reshape,
|
||||
pm_pass,
|
||||
)
|
||||
|
||||
|
||||
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses aiter rms_norm & vllm/aiter quant custom ops
|
||||
@@ -309,8 +465,24 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
)
|
||||
|
||||
# Make sure fused add patterns are before simple rms norm,
|
||||
# as the latter is a subset of the former in torch ops
|
||||
# as the latter is a subset of the former in torch ops.
|
||||
# The DoubleQuant patterns handle 1 rms_norm -> 2 group_fp8_quant
|
||||
# fan-out (e.g. DSv3.2) and must be registered before the single
|
||||
# group-quant pattern so they match first. The view-tolerant variant
|
||||
# additionally covers the rms_norm -> view -> 2x quant shape that
|
||||
# appears when the FP8 linear path inserts a 2D-flatten boilerplate
|
||||
# (DSv3.2 MLA indexer q_c norm).
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
# Fuse aiter rms_norm + 2x aiter group fp8 quant
|
||||
DoubleAiterRMSFp8GroupQuantPattern(
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128)
|
||||
).register(self.patterns)
|
||||
|
||||
# View-tolerant sibling for DSv3.2 q_c norm fan-out
|
||||
DoubleAiterRMSFp8GroupQuantViewPattern(
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128)
|
||||
).register(self.patterns)
|
||||
|
||||
# Fuse aiter rms_norm + aiter dynamic group fp8 quant
|
||||
AiterRMSFp8GroupQuantPattern(
|
||||
epsilon, FP8_DTYPE, GroupShape(1, 128)
|
||||
@@ -360,6 +532,8 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
AiterFusedAddRMSNormDynamicQuantPattern,
|
||||
AiterRMSFp8GroupQuantPattern,
|
||||
AiterFusedAddRMSFp8GroupQuantPattern,
|
||||
DoubleAiterRMSFp8GroupQuantPattern,
|
||||
DoubleAiterRMSFp8GroupQuantViewPattern,
|
||||
]
|
||||
return self.hash_source(self, *fusion_patterns)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user