[ROCm] [fusion] support ROCm aiter fusion of per tensor quant + rmsnorm (#52831)

Signed-off-by: Andy Friedrich <afriedri@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
afriedri
2026-10-16 15:19:07 -05:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Claude Opus 5.5
parent 9eb7fb9157
commit cc2e4fecf2
4 changed files with 274 additions and 6 deletions
@@ -394,10 +394,13 @@ class MatcherQuantFP8(MatcherCustomOp):
self.is_tma_aligned = is_tma_aligned
if match_rocm_aiter:
assert not quant_key.scale.group_shape.is_per_tensor(), (
"ROCm aiter fusion pass does not support per tensor quantization"
)
if quant_key.scale.group_shape.is_per_token():
if quant_key.scale.group_shape.is_per_tensor():
assert quant_key.scale.static, (
"ROCm aiter fusion pass only supports static per-tensor "
"quantization"
)
self.QUANT_OP = rocm_aiter_ops.get_per_tensor_quant_op()
elif quant_key.scale.group_shape.is_per_token():
self.QUANT_OP = rocm_aiter_ops.get_per_token_quant_op()
else:
assert quant_key.scale.group_shape.col == 128, (
@@ -432,14 +435,24 @@ class MatcherQuantFP8(MatcherCustomOp):
scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
quant_key_group_shape = self.quant_key.scale.group_shape
if quant_key_group_shape == GroupShape.PER_TENSOR:
# rocm_aiter_per_tensor_quant mutates `out` and `scale` and returns
# None, so it has to be matched through auto_functionalized.
assert scale is not None
out = torch.empty_like(input, dtype=self.quant_key.dtype)
result = auto_functionalized(
self.QUANT_OP, out=out, x=input, scale=scale, is_dynamic=False
)
# `scale` is mutable, so its functionalized output is a distinct
# value; return it to keep the quant node inside the match.
return result[1], result[2]
if quant_key_group_shape == GroupShape.PER_TOKEN:
return self.QUANT_OP( # type: ignore[no-any-return]
x=input,
quant_dtype=self.quant_key.dtype,
scale=scale,
)
else:
return self.QUANT_OP(input, quant_key_group_shape.col) # type: ignore[no-any-return]
return self.QUANT_OP(input, quant_key_group_shape.col) # type: ignore[no-any-return]
def forward_custom(
self,