[ROCm][Perf] Avoid extra reshape kernel in Qwen GDN output norm (#47842)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
496c6472cb
commit
68363d64ec
@@ -30,6 +30,7 @@ from ..vllm_inductor_pass import (
|
||||
VllmPatternReplacement,
|
||||
_fx_view_to_reshape,
|
||||
fold_consecutive_reshapes,
|
||||
remove_noop_reshapes,
|
||||
)
|
||||
from .matcher_utils import (
|
||||
MatcherQuantFP8,
|
||||
@@ -452,9 +453,9 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
"""Matches decomposed RMSNormGated + reshape + group FP8 quant and replaces
|
||||
with rocm_aiter_fused_rms_gated_fp8_group_quant.
|
||||
|
||||
The norm operates per-head on (N*H, D) tensors. The compiler folds the
|
||||
reshape chain so after norm the result goes through reshape->merge->quant.
|
||||
The pattern reshapes from (N*H, D) to (N, H*D) before calling
|
||||
The norm operates per-head, on either (N*H, D) or (N, H, D). The compiler
|
||||
folds the reshape chain so after norm the result goes through
|
||||
reshape->merge->quant. The pattern reshapes to (N, H*D) before calling
|
||||
MatcherQuantFP8 so that _quantize_group_native sees the full hidden dim
|
||||
and computes the correct num_groups.
|
||||
"""
|
||||
@@ -482,6 +483,16 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
self.head_dim = head_dim
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
self._register_layout(pm_pass, flatten_heads=True)
|
||||
# Both layouts reach the norm. With an opaque quant op they trace to
|
||||
# the same pattern; with a native quant they differ, because its group
|
||||
# reshape back to (N, H, D) is a no-op the compiler already dropped.
|
||||
if not self.quant_matcher.enabled:
|
||||
self._register_layout(pm_pass, flatten_heads=False)
|
||||
|
||||
def _register_layout(
|
||||
self, pm_pass: PatternMatcherPass, *, flatten_heads: bool
|
||||
) -> None:
|
||||
num_heads = self.num_heads
|
||||
head_dim = self.head_dim
|
||||
hidden_dim = num_heads * head_dim
|
||||
@@ -502,6 +513,11 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
z: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# The pattern matches on op structure rather than rank, so either
|
||||
# per-head layout can land here. The fused op takes (M, D), and
|
||||
# both operands are contiguous, so collapsing the heads is a view.
|
||||
x = x.reshape(-1, head_dim)
|
||||
z = z.reshape(-1, head_dim)
|
||||
fused = self.FUSED_OP(
|
||||
x=x,
|
||||
weight=weight,
|
||||
@@ -519,14 +535,21 @@ class AiterRMSNormGatedFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
return fp8_reshaped, scales_reshaped
|
||||
|
||||
n_tokens = 2
|
||||
x = self.empty(n_tokens * num_heads, head_dim)
|
||||
z = self.empty(n_tokens * num_heads, head_dim)
|
||||
shape = (
|
||||
(n_tokens * num_heads, head_dim)
|
||||
if flatten_heads
|
||||
else (n_tokens, num_heads, head_dim)
|
||||
)
|
||||
x = self.empty(*shape)
|
||||
z = self.empty(*shape)
|
||||
w = self.empty(head_dim)
|
||||
|
||||
def trace_fn(*args, **kwargs):
|
||||
gm = pm.fwd_only(*args, **kwargs)
|
||||
_fx_view_to_reshape(gm)
|
||||
fold_consecutive_reshapes(gm)
|
||||
if not flatten_heads:
|
||||
remove_noop_reshapes(gm)
|
||||
return gm
|
||||
|
||||
pm.register_replacement(
|
||||
|
||||
@@ -277,6 +277,27 @@ def fold_consecutive_reshapes(gm: fx.GraphModule) -> None:
|
||||
gm.graph.erase_node(inp)
|
||||
|
||||
|
||||
def remove_noop_reshapes(gm: fx.GraphModule) -> None:
|
||||
"""Drop reshape ops whose output shape equals their input shape.
|
||||
|
||||
Companion to :func:`fold_consecutive_reshapes`. ``make_fx`` records a
|
||||
reshape to the shape the input already has; the compiled graph has already
|
||||
dropped it, so the pattern only matches once we drop it too.
|
||||
"""
|
||||
aten_reshape = torch.ops.aten.reshape.default
|
||||
for node in list(gm.graph.nodes):
|
||||
if not is_func(node, aten_reshape):
|
||||
continue
|
||||
inp = node.args[0]
|
||||
if not isinstance(inp, fx.Node):
|
||||
continue
|
||||
val, inp_val = node.meta.get("val"), inp.meta.get("val")
|
||||
if val is None or inp_val is None or val.shape != inp_val.shape:
|
||||
continue
|
||||
node.replace_all_uses_with(inp)
|
||||
gm.graph.erase_node(node)
|
||||
|
||||
|
||||
def _remove_noop_permutes(gm: fx.GraphModule) -> None:
|
||||
for node in gm.graph.nodes:
|
||||
if not is_func(node, torch.ops.aten.permute.default):
|
||||
|
||||
Reference in New Issue
Block a user