[ROCm][Perf] Avoid extra reshape kernel in Qwen GDN output norm (#47842)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Mikko Tukiainen
2026-09-22 14:38:16 +08:00
committed by GitHub
co-authored by Cursor
parent 496c6472cb
commit 68363d64ec
6 changed files with 170 additions and 33 deletions
@@ -30,6 +30,7 @@ from ..vllm_inductor_pass import (
VllmPatternReplacement,
_fx_view_to_reshape,
fold_consecutive_reshapes,
remove_noop_reshapes,
)
from .matcher_utils import (
MatcherQuantFP8,
@@ -452,9 +453,9 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
"""Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
with rocm_aiter_fused_rms_gated_fp8_group_quant.
The norm operates per-head on (N*H, D) tensors. The compiler folds the
reshape chain so after norm the result goes through reshape->merge->quant.
The pattern reshapes from (N*H, D) to (N, H*D) before calling
The norm operates per-head, on either (N*H, D) or (N, H, D). The compiler
folds the reshape chain so after norm the result goes through
reshape->merge->quant. The pattern reshapes to (N, H*D) before calling
MatcherQuantFP8 so that _quantize_group_native sees the full hidden dim
and computes the correct num_groups.
"""
@@ -482,6 +483,16 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
self.head_dim = head_dim
def register(self, pm_pass: PatternMatcherPass) -> None:
self._register_layout(pm_pass, flatten_heads=True)
# Both layouts reach the norm. With an opaque quant op they trace to
# the same pattern; with a native quant they differ, because its group
# reshape back to (N, H, D) is a no-op the compiler already dropped.
if not self.quant_matcher.enabled:
self._register_layout(pm_pass, flatten_heads=False)
def _register_layout(
self, pm_pass: PatternMatcherPass, *, flatten_heads: bool
) -> None:
num_heads = self.num_heads
head_dim = self.head_dim
hidden_dim = num_heads * head_dim
@@ -502,6 +513,11 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
z: torch.Tensor,
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
# The pattern matches on op structure rather than rank, so either
# per-head layout can land here. The fused op takes (M, D), and
# both operands are contiguous, so collapsing the heads is a view.
x = x.reshape(-1, head_dim)
z = z.reshape(-1, head_dim)
fused = self.FUSED_OP(
x=x,
weight=weight,
@@ -519,14 +535,21 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
return fp8_reshaped, scales_reshaped
n_tokens = 2
x = self.empty(n_tokens * num_heads, head_dim)
z = self.empty(n_tokens * num_heads, head_dim)
shape = (
(n_tokens * num_heads, head_dim)
if flatten_heads
else (n_tokens, num_heads, head_dim)
)
x = self.empty(*shape)
z = self.empty(*shape)
w = self.empty(head_dim)
def trace_fn(*args, **kwargs):
gm = pm.fwd_only(*args, **kwargs)
_fx_view_to_reshape(gm)
fold_consecutive_reshapes(gm)
if not flatten_heads:
remove_noop_reshapes(gm)
return gm
pm.register_replacement(
@@ -277,6 +277,27 @@ def fold_consecutive_reshapes(gm: fx.GraphModule) -> None:
gm.graph.erase_node(inp)
def remove_noop_reshapes(gm: fx.GraphModule) -> None:
"""Drop reshape ops whose output shape equals their input shape.
Companion to :func:`fold_consecutive_reshapes`. ``make_fx`` records a
reshape to the shape the input already has; the compiled graph has already
dropped it, so the pattern only matches once we drop it too.
"""
aten_reshape = torch.ops.aten.reshape.default
for node in list(gm.graph.nodes):
if not is_func(node, aten_reshape):
continue
inp = node.args[0]
if not isinstance(inp, fx.Node):
continue
val, inp_val = node.meta.get("val"), inp.meta.get("val")
if val is None or inp_val is None or val.shape != inp_val.shape:
continue
node.replace_all_uses_with(inp)
gm.graph.erase_node(node)
def _remove_noop_permutes(gm: fx.GraphModule) -> None:
for node in gm.graph.nodes:
if not is_func(node, torch.ops.aten.permute.default):