[ROCm] [fusion] support ROCm aiter fusion of per tensor quant + rmsnorm (#52831)

Signed-off-by: Andy Friedrich <afriedri@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
afriedri
2026-10-16 15:19:07 -05:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Claude Opus 5.5
parent 9eb7fb9157
commit cc2e4fecf2
4 changed files with 274 additions and 6 deletions
+16
View File
@@ -107,6 +107,8 @@ AITER_KERNEL_GROUPSHAPE_COMBINATIONS = [
# Per-token with ChannelWiseTorchFP8ScaledMMLinearKernel
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, True),
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, False),
# Static per-tensor: only the aiter quant op is matched by the pass
(ROCmFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR, True),
# Blockwise
(AiterFp8BlockScaledMMKernel, GroupShape(1, 128), True),
]
@@ -197,6 +199,9 @@ class TestModel(torch.nn.Module):
# Blockwise path
if self.use_aiter_fusion and self.use_aiter_quant_op:
return [rocm_aiter_ops.get_group_quant_op()]
elif self.group_shape.is_per_tensor():
if self.use_aiter_quant_op:
return [rocm_aiter_ops.get_per_tensor_quant_op()]
else:
if self.use_aiter_quant_op:
return [rocm_aiter_ops.get_per_token_quant_op()]
@@ -221,6 +226,17 @@ class TestModel(torch.nn.Module):
AiterFusedAddRMSFp8GroupQuantPattern.FUSED_OP,
AiterRMSFp8GroupQuantPattern.FUSED_OP,
]
elif self.group_shape.is_per_tensor():
# Static per-tensor aiter fusion
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
AiterFusedAddRMSNormStaticQuantPattern,
AiterRMSNormStaticQuantPattern,
)
return [
AiterFusedAddRMSNormStaticQuantPattern.FUSED_OP,
AiterRMSNormStaticQuantPattern.FUSED_OP,
]
else:
# Per-token aiter fusion
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (