diff --git a/tests/compile/passes/test_double_aiter_rms_quant_fusion.py b/tests/compile/passes/test_double_aiter_rms_quant_fusion.py new file mode 100644 index 0000000000..161c956548 --- /dev/null +++ b/tests/compile/passes/test_double_aiter_rms_quant_fusion.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Unit tests for the DoubleQuant fan-out variants registered by +``RocmAiterRMSNormQuantFusionPass``. + +Both variants target a 1-to-2 fan-out where one ``rms_norm`` output feeds +two distinct ``rocm_aiter_group_fp8_quant`` consumers and rewrite it into +two independent fused ``rms_norm + group_fp8_quant`` ops: + +* ``DoubleAiterRMSFp8GroupQuantPattern`` matches the un-viewed shape + (e.g. Kimi-K2.5 / DSR1). +* ``DoubleAiterRMSFp8GroupQuantViewPattern`` (this PR) is the view-tolerant + sibling that additionally matches the + ``rms_norm -> view -> group_fp8_quant`` shape that DSv3.2's MLA indexer + q_c norm exposes through ``Fp8BlockScaledMMLinearKernel.apply_weights``'s + 2D-flatten boilerplate. +""" + +import pytest +import torch + +import vllm.config +from tests.compile.backend import TestBackend +from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops +from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass +from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass +from vllm.config import ( + CompilationConfig, + CompilationMode, + ModelConfig, + PassConfig, + VllmConfig, +) + +EPS = 1e-5 +HIDDEN_SIZE = 256 +GROUP_SIZE = 128 + + +class _NoViewDoubleQuantModel(torch.nn.Module): + """``rms_norm -> 2x group_fp8_quant`` fan-out (Kimi-K2.5 / DSR1 shape).""" + + def __init__(self) -> None: + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(HIDDEN_SIZE, dtype=torch.bfloat16)) + + def forward( + self, x: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + # avoid graph input being a direct arg to a matched pattern node + x = torch.relu(x) + rms = torch.ops.vllm_ir.rms_norm(x, self.weight, EPS) + q1, s1 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(rms, GROUP_SIZE) + q2, s2 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(rms, GROUP_SIZE) + return q1, s1, q2, s2 + + +class _ViewDoubleQuantModel(torch.nn.Module): + """``rms_norm -> view -> 2x group_fp8_quant`` fan-out (DSv3.2 shape). + + Reproduces the FX-graph shape produced by ``Fp8BlockScaledMMLinearKernel``'s + 2D-flatten before the FP8 group quant op. + """ + + def __init__(self) -> None: + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(HIDDEN_SIZE, dtype=torch.bfloat16)) + + def forward( + self, x: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + x = torch.relu(x) + rms = torch.ops.vllm_ir.rms_norm(x, self.weight, EPS) + view = rms.view(-1, rms.shape[-1]) + q1, s1 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(view, GROUP_SIZE) + q2, s2 = torch.ops.vllm.rocm_aiter_group_fp8_quant.default(view, GROUP_SIZE) + return q1, s1, q2, s2 + + +@pytest.mark.parametrize( + "model_cls", + [_NoViewDoubleQuantModel, _ViewDoubleQuantModel], + ids=["no_view", "with_view"], +) +@pytest.mark.skipif( + not is_aiter_found_and_supported(), + reason="Only test on ROCm with AITER installed and supported", +) +def test_double_aiter_rms_fp8_group_quant_fusion( + model_cls: type[torch.nn.Module], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ + Both fan-out shapes (with and without an intermediate view) must fuse + into ``rocm_aiter_rmsnorm_fp8_group_quant``: the no-view shape via + ``DoubleAiterRMSFp8GroupQuantPattern`` and the viewed shape via the + new ``DoubleAiterRMSFp8GroupQuantViewPattern`` sibling. + + A failure on the ``with_view`` parametrization is a regression on the + DSv3.2 q_c norm path that this PR's view-tolerant pattern is intended + to cover. + """ + torch._dynamo.reset() + + vllm_config = VllmConfig( + model_config=ModelConfig(dtype=torch.bfloat16), + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + custom_ops=["+rms_norm", "+quant_fp8"], + pass_config=PassConfig( + fuse_norm_quant=True, + eliminate_noops=True, + ), + ), + ) + + with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m: + from vllm.compilation.passes.fusion.rocm_aiter_fusion import ( + RocmAiterRMSNormQuantFusionPass, + ) + + torch.set_default_device("cuda") + torch.set_default_dtype(torch.bfloat16) + torch.manual_seed(0) + + m.setenv("VLLM_ROCM_USE_AITER", "1") + rocm_aiter_ops.refresh_env_variables() + + fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config) + passes = [ + NoOpEliminationPass(vllm_config), + fusion_pass, + PostCleanupPass(vllm_config), + ] + backend = TestBackend(*passes) + model = model_cls() + + x = torch.randn(8, HIDDEN_SIZE) + torch._dynamo.mark_dynamic(x, 0) + + outputs_unfused = model(x) + model_fused = torch.compile(model, backend=backend) + outputs_fused = model_fused(x) + + # Both consumers must be rewritten into the fused op (one + # ``register_replacement`` rewrite covers the whole 1-to-2 fan-out). + assert fusion_pass.matched_count == 1, ( + f"Expected the {model_cls.__name__} fan-out to fuse via the " + f"DoubleQuant pattern (matched_count == 1), got " + f"{fusion_pass.matched_count}" + ) + + fused_op = rocm_aiter_ops.get_rmsnorm_group_fused_quant_op() + backend.check_after_ops([fused_op]) + + # Numerical parity sanity-check: the fused pair must match the + # unfused pair on FP8 outputs (exact byte-equality is the goal, + # but allow a tiny tolerance for any residual numeric noise). + for fused_t, unfused_t in zip(outputs_fused, outputs_unfused): + torch.testing.assert_close( + fused_t.to(torch.float32), + unfused_t.to(torch.float32), + atol=1e-2, + rtol=1e-2, + ) diff --git a/vllm/compilation/passes/fusion/matcher_utils.py b/vllm/compilation/passes/fusion/matcher_utils.py index d861c83496..94f0e62204 100644 --- a/vllm/compilation/passes/fusion/matcher_utils.py +++ b/vllm/compilation/passes/fusion/matcher_utils.py @@ -273,12 +273,7 @@ class MatcherQuantFP8(MatcherCustomOp): "ROCm aiter fusion pass currently supports " "quantization operation with group_size 128" ) - if current_platform.is_fp8_fnuz(): - self.QUANT_OP = rocm_aiter_ops.get_group_quant_op() - else: - self.QUANT_OP = ( - torch.ops.vllm.triton_per_token_group_quant_fp8.default - ) + self.QUANT_OP = rocm_aiter_ops.get_group_quant_op() else: assert quant_key in QUANT_OPS, ( diff --git a/vllm/compilation/passes/fusion/rocm_aiter_fusion.py b/vllm/compilation/passes/fusion/rocm_aiter_fusion.py index 83abcdeb5a..9a975c5fed 100644 --- a/vllm/compilation/passes/fusion/rocm_aiter_fusion.py +++ b/vllm/compilation/passes/fusion/rocm_aiter_fusion.py @@ -6,6 +6,7 @@ from typing import Any import torch import torch._inductor.pattern_matcher as pm from torch import fx +from torch._inductor.fx_passes.post_grad import view_to_reshape from torch._inductor.pattern_matcher import PatternMatcherPass import vllm.ir.ops @@ -293,6 +294,161 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern): pm.register_replacement(pattern, replacement, inputs, pm.fwd_only, pm_pass) +class DoubleAiterRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern): + """ + Pattern matching ``rms_norm`` whose output feeds *two* distinct + ``rocm_aiter_group_fp8_quant`` consumers, replacing it with two + independent fused ``rms_norm_group_fp8_quant`` ops. + + Repeating the rms_norm in the replacement is preferable to leaving + the fused 16-bit rms output materialized for two unfused quant + consumers, and matches what the previous manual graph surgery + achieved by cloning the rms_norm node. + """ + + FUSED_OP = rocm_aiter_ops.get_rmsnorm_group_fused_quant_op() + + def __init__( + self, + epsilon: float, + quant_dtype: torch.dtype, + group_shape: GroupShape, + match_aiter_quant: bool = True, + symmetric: bool = True, + ) -> None: + scale = ScaleDesc(torch.float32, False, group_shape) + key = FusedRMSQuantKey( + fused_add=False, + quant=QuantKey(dtype=quant_dtype, scale=scale, symmetric=symmetric), + ) + + super().__init__(epsilon, key, match_aiter_quant) + + def register(self, pm_pass: PatternMatcherPass) -> None: + def pattern( + input: torch.Tensor, + weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + result_rms = torch.ops.vllm_ir.rms_norm(input, weight, self.epsilon) + result1, scale1 = self.quant_matcher(result_rms) + result2, scale2 = self.quant_matcher(result_rms) + return result1, scale1, result2, scale2 + + def replacement( + input: torch.Tensor, + weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + at1 = self.FUSED_OP( + x=input, + weight=weight, + variance_epsilon=self.epsilon, + group_size=128, + ) + at2 = self.FUSED_OP( + x=input, + weight=weight, + variance_epsilon=self.epsilon, + group_size=128, + ) + + return at1[0], at1[1], at2[0], at2[1] + + pm.register_replacement( + pattern, + replacement, + # input, weight + [self.empty(5, 16), self.empty(16)], + pm.fwd_only, + pm_pass, + ) + + +class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern): + """ + View-tolerant variant of ``DoubleAiterRMSFp8GroupQuantPattern``. + + Matches the same 1-to-2 fan-out, but with a ``view``/``reshape`` between + the ``rms_norm`` output and the two ``rocm_aiter_group_fp8_quant`` + consumers:: + + rms_norm -> view -> rocm_aiter_group_fp8_quant + \\-> view -> rocm_aiter_group_fp8_quant + + This shape arises in DeepSeek-V3.2's MLA indexer q_c norm, where the + FP8 linear path's 2D-flatten boilerplate + (``Fp8BlockScaledMMLinearKernel.apply_weights``) inserts a view between + the rms_norm output and each FP8 group quant op. The non-view sibling + pattern silently no-ops on this graph because the pattern matcher + requires the in-graph and in-pattern node shapes to align. + + The trace_fn runs Inductor's ``view_to_reshape`` post-grad pass to + normalize ``view`` to ``reshape`` in both the pattern and the input + graph, widening the match without touching the no-view sibling. + """ + + FUSED_OP = rocm_aiter_ops.get_rmsnorm_group_fused_quant_op() + + def __init__( + self, + epsilon: float, + quant_dtype: torch.dtype, + group_shape: GroupShape, + match_aiter_quant: bool = True, + symmetric: bool = True, + ) -> None: + scale = ScaleDesc(torch.float32, False, group_shape) + key = FusedRMSQuantKey( + fused_add=False, + quant=QuantKey(dtype=quant_dtype, scale=scale, symmetric=symmetric), + ) + + super().__init__(epsilon, key, match_aiter_quant) + + def register(self, pm_pass: PatternMatcherPass) -> None: + def pattern( + input: torch.Tensor, + weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + result_rms = torch.ops.vllm_ir.rms_norm(input, weight, self.epsilon) + view_rms = result_rms.view(-1, result_rms.shape[-1]) + result1, scale1 = self.quant_matcher(view_rms) + result2, scale2 = self.quant_matcher(view_rms) + return result1, scale1, result2, scale2 + + def replacement( + input: torch.Tensor, + weight: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + at1 = self.FUSED_OP( + x=input, + weight=weight, + variance_epsilon=self.epsilon, + group_size=128, + ) + at2 = self.FUSED_OP( + x=input, + weight=weight, + variance_epsilon=self.epsilon, + group_size=128, + ) + + return at1[0], at1[1], at2[0], at2[1] + + def trace_with_view_to_reshape(*args: Any, **kwargs: Any) -> fx.GraphModule: + gm = pm.fwd_only(*args, **kwargs) + view_to_reshape(gm) + return gm + + pm.register_replacement( + pattern, + replacement, + # input, weight + [self.empty(5, 16), self.empty(16)], + trace_with_view_to_reshape, + pm_pass, + ) + + class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass): """ This pass fuses aiter rms_norm & vllm/aiter quant custom ops @@ -309,8 +465,24 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass): ) # Make sure fused add patterns are before simple rms norm, - # as the latter is a subset of the former in torch ops + # as the latter is a subset of the former in torch ops. + # The DoubleQuant patterns handle 1 rms_norm -> 2 group_fp8_quant + # fan-out (e.g. DSv3.2) and must be registered before the single + # group-quant pattern so they match first. The view-tolerant variant + # additionally covers the rms_norm -> view -> 2x quant shape that + # appears when the FP8 linear path inserts a 2D-flatten boilerplate + # (DSv3.2 MLA indexer q_c norm). for epsilon in [1e-5, 1e-6]: + # Fuse aiter rms_norm + 2x aiter group fp8 quant + DoubleAiterRMSFp8GroupQuantPattern( + epsilon, FP8_DTYPE, GroupShape(1, 128) + ).register(self.patterns) + + # View-tolerant sibling for DSv3.2 q_c norm fan-out + DoubleAiterRMSFp8GroupQuantViewPattern( + epsilon, FP8_DTYPE, GroupShape(1, 128) + ).register(self.patterns) + # Fuse aiter rms_norm + aiter dynamic group fp8 quant AiterRMSFp8GroupQuantPattern( epsilon, FP8_DTYPE, GroupShape(1, 128) @@ -360,6 +532,8 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass): AiterFusedAddRMSNormDynamicQuantPattern, AiterRMSFp8GroupQuantPattern, AiterFusedAddRMSFp8GroupQuantPattern, + DoubleAiterRMSFp8GroupQuantPattern, + DoubleAiterRMSFp8GroupQuantViewPattern, ] return self.hash_source(self, *fusion_patterns)