[Aiter][ROCm] RMSNormGated+GroupedQuantFP8 fusion (#40710)

Signed-off-by: Tres Popp <tres.popp@amd.com>
Signed-off-by: Tres Popp <trespopp@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Tres
2026-05-14 15:37:09 -04:00
committed by GitHub
co-authored by Cursor
parent 9898f94abe
commit f887aa1a53
6 changed files with 557 additions and 34 deletions
@@ -10,6 +10,7 @@ from torch._ops import OpOverload
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.layernorm import RMSNormGated
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
@@ -28,7 +29,6 @@ from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
)
from vllm.platforms import current_platform
RMS_ADD_OP = torch.ops._C.fused_add_rms_norm.default
ROTARY_OP = torch.ops._C.rotary_embedding.default
FLASHINFER_ROTARY_OP = torch.ops.vllm.flashinfer_rotary_embedding.default
@@ -161,6 +161,67 @@ class MatcherRotaryEmbedding(MatcherCustomOp):
return result
class MatcherRMSNormGated(MatcherCustomOp):
"""Matches RMSNormGated with norm_before_gate=True and group_size=None."""
def __init__(
self,
epsilon: float,
enabled: bool | None = None,
norm_before_gate: bool = True,
group_size: int | None = None,
) -> None:
if enabled is None:
enabled = RMSNormGated.enabled()
super().__init__(enabled)
self.epsilon = epsilon
self.norm_before_gate = norm_before_gate
self.group_size = group_size
def inputs(self) -> list[torch.Tensor]:
x = self.empty(5, 16)
z = self.empty(5, 16)
weight = self.empty(16)
return [x, z, weight]
def forward_custom(
self,
x: torch.Tensor,
z: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
from vllm.model_executor.layers.fla.ops.layernorm_guard import (
rmsnorm_fn,
)
return rmsnorm_fn(
x,
weight,
bias=None,
z=z,
eps=self.epsilon,
group_size=self.group_size,
norm_before_gate=self.norm_before_gate,
)
def forward_native(
self,
x: torch.Tensor,
z: torch.Tensor,
weight: torch.Tensor,
) -> torch.Tensor:
return RMSNormGated.forward_static(
x,
z,
weight,
self.epsilon,
self.model_dtype,
group_size=self.group_size,
norm_before_gate=self.norm_before_gate,
)
class MatcherDeepseekScalingRotaryEmbedding(MatcherCustomOp):
def __init__(
self,
@@ -12,7 +12,7 @@ from torch._inductor.pattern_matcher import PatternMatcherPass
import vllm.ir.ops
import vllm.model_executor.layers.quantization.utils.fp8_utils # noqa: F401
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
@@ -28,9 +28,12 @@ from ..vllm_inductor_pass import (
VllmInductorPass,
VllmPatternMatcherPass,
VllmPatternReplacement,
_fx_view_to_reshape,
fold_consecutive_reshapes,
)
from .matcher_utils import (
MatcherQuantFP8,
MatcherRMSNormGated,
MatcherSiluAndMul,
)
from .rms_quant_fusion import (
@@ -449,6 +452,97 @@ class DoubleAiterRMSFp8GroupQuantViewPattern(AiterRMSNormQuantPattern):
)
class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""
Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
with rocm_aiter_fused_rms_gated_fp8_group_quant.
The norm operates per-head on (N*H, D) tensors. The compiler folds the
reshape chain so after norm the result goes through reshape->merge->quant.
The pattern reshapes from (N*H, D) to (N, H*D) before calling
MatcherQuantFP8 so that _quantize_group_native sees the full hidden dim
and computes the correct num_groups.
"""
FUSED_OP = rocm_aiter_ops.get_fused_rms_gated_fp8_group_quant_op()
def __init__(
self,
epsilon: float,
quant_dtype: torch.dtype,
group_shape: GroupShape,
num_heads: int,
head_dim: int,
match_aiter_quant: bool = True,
symmetric: bool = True,
) -> None:
scale = ScaleDesc(torch.float32, False, group_shape)
key = FusedRMSQuantKey(
fused_add=False,
quant=QuantKey(dtype=quant_dtype, scale=scale, symmetric=symmetric),
)
super().__init__(epsilon, key, match_aiter_quant)
self.rmsnorm_gated_matcher = MatcherRMSNormGated(epsilon)
self.num_heads = num_heads
self.head_dim = head_dim
def register(self, pm_pass: PatternMatcherPass) -> None:
num_heads = self.num_heads
head_dim = self.head_dim
hidden_dim = num_heads * head_dim
quant_matcher = self.quant_matcher
def pattern(
x: torch.Tensor,
z: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
normed = self.rmsnorm_gated_matcher(x, z, weight)
merged = normed.reshape(-1, hidden_dim)
quant_out, scales_out = quant_matcher(merged)
return quant_out, scales_out
def replacement(
x: torch.Tensor,
z: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
fused = self.FUSED_OP(
x=x,
weight=weight,
bias=None,
z=z,
eps=self.epsilon,
norm_before_gate=True,
activation="silu",
group_size=head_dim,
)
fp8_out = fused[0]
scales_out = fused[1]
fp8_reshaped = fp8_out.reshape(-1, hidden_dim)
scales_reshaped = scales_out.reshape(-1, num_heads)
return fp8_reshaped, scales_reshaped
n_tokens = 2
x = self.empty(n_tokens * num_heads, head_dim)
z = self.empty(n_tokens * num_heads, head_dim)
w = self.empty(head_dim)
def trace_fn(*args, **kwargs):
gm = pm.fwd_only(*args, **kwargs)
_fx_view_to_reshape(gm)
fold_consecutive_reshapes(gm)
return gm
pm.register_replacement(
pattern,
replacement,
[x, z, w],
trace_fn,
pm_pass,
)
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
"""
This pass fuses aiter rms_norm & vllm/aiter quant custom ops
@@ -464,6 +558,19 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
pass_name="rocm_aiter_rms_norm_quant_fusion_pass"
)
# Discover (num_heads, head_dim) pairs for gated RMSNorm patterns
# from GatedDeltaNetAttention layers in static_forward_context.
from vllm.model_executor.layers.mamba.gdn_linear_attn import (
GatedDeltaNetAttention,
)
gdn_layers = get_layers_from_vllm_config(config, GatedDeltaNetAttention)
gated_norm_shapes: set[tuple[int, int]] = set()
for layer in gdn_layers.values():
gated_norm_shapes.add(
(layer.num_v_heads // layer.tp_size, layer.head_v_dim)
)
# Make sure fused add patterns are before simple rms norm,
# as the latter is a subset of the former in torch ops.
# The DoubleQuant patterns handle 1 rms_norm -> 2 group_fp8_quant
@@ -517,6 +624,21 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
epsilon, FP8_DTYPE, match_aiter_quant=match_aiter_quant
).register(self.patterns)
# Fuse decomposed RMSNormGated + group fp8 quant.
# The replacement op (fused_rms_gated_fp8_group_quant) requires
# an aiter version that includes the GDN triton kernel renames.
if gated_norm_shapes and rocm_aiter_ops.are_gdn_triton_kernels_available():
for num_heads, head_dim in gated_norm_shapes:
if head_dim != 128:
continue
AiterRMSNormGatedFp8GroupQuantPattern(
epsilon,
FP8_DTYPE,
GroupShape(1, 128),
num_heads=num_heads,
head_dim=head_dim,
).register(self.patterns)
self.dump_patterns(config, self.patterns)
@VllmInductorPass.time_and_log
@@ -534,6 +656,7 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
AiterFusedAddRMSFp8GroupQuantPattern,
DoubleAiterRMSFp8GroupQuantPattern,
DoubleAiterRMSFp8GroupQuantViewPattern,
AiterRMSNormGatedFp8GroupQuantPattern,
]
return self.hash_source(self, *fusion_patterns)
@@ -252,6 +252,33 @@ def _fx_view_to_reshape(gm: fx.GraphModule) -> None:
view_to_reshape(gm)
def fold_consecutive_reshapes(gm: fx.GraphModule) -> None:
"""Fold consecutive reshape ops into a single reshape.
``make_fx`` faithfully records every view/reshape the Python code performs,
so patterns like ``x.reshape(a, b).reshape(c, d)`` produce two reshape
nodes. Inductor's own optimisation would fold these, but
``pm.register_replacement``'s ``trace_fn`` runs before Inductor, so we
must fold them ourselves for the pattern to match the compiled graph.
When reshape(A, shape1) feeds only into reshape(result, shape2),
the first reshape is redundant -- replace with reshape(A, shape2).
"""
aten_reshape = torch.ops.aten.reshape.default
for node in list(gm.graph.nodes):
if not is_func(node, aten_reshape):
continue
inp = node.args[0]
if not isinstance(inp, fx.Node) or not is_func(inp, aten_reshape):
continue
if len(inp.users) != 1:
continue
original_input = inp.args[0]
node.args = (original_input, node.args[1])
inp.replace_all_uses_with(original_input)
gm.graph.erase_node(inp)
def _remove_noop_permutes(gm: fx.GraphModule) -> None:
for node in gm.graph.nodes:
if not is_func(node, torch.ops.aten.permute.default):