[ROCm] [fusion] support ROCm aiter fusion of per tensor quant + rmsnorm (#52831)
Signed-off-by: Andy Friedrich <afriedri@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Claude Opus 5.5
parent
9eb7fb9157
commit
cc2e4fecf2
@@ -107,6 +107,8 @@ AITER_KERNEL_GROUPSHAPE_COMBINATIONS = [
|
||||
# Per-token with ChannelWiseTorchFP8ScaledMMLinearKernel
|
||||
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, True),
|
||||
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, False),
|
||||
# Static per-tensor: only the aiter quant op is matched by the pass
|
||||
(ROCmFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR, True),
|
||||
# Blockwise
|
||||
(AiterFp8BlockScaledMMKernel, GroupShape(1, 128), True),
|
||||
]
|
||||
@@ -197,6 +199,9 @@ class TestModel(torch.nn.Module):
|
||||
# Blockwise path
|
||||
if self.use_aiter_fusion and self.use_aiter_quant_op:
|
||||
return [rocm_aiter_ops.get_group_quant_op()]
|
||||
elif self.group_shape.is_per_tensor():
|
||||
if self.use_aiter_quant_op:
|
||||
return [rocm_aiter_ops.get_per_tensor_quant_op()]
|
||||
else:
|
||||
if self.use_aiter_quant_op:
|
||||
return [rocm_aiter_ops.get_per_token_quant_op()]
|
||||
@@ -221,6 +226,17 @@ class TestModel(torch.nn.Module):
|
||||
AiterFusedAddRMSFp8GroupQuantPattern.FUSED_OP,
|
||||
AiterRMSFp8GroupQuantPattern.FUSED_OP,
|
||||
]
|
||||
elif self.group_shape.is_per_tensor():
|
||||
# Static per-tensor aiter fusion
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
AiterFusedAddRMSNormStaticQuantPattern,
|
||||
AiterRMSNormStaticQuantPattern,
|
||||
)
|
||||
|
||||
return [
|
||||
AiterFusedAddRMSNormStaticQuantPattern.FUSED_OP,
|
||||
AiterRMSNormStaticQuantPattern.FUSED_OP,
|
||||
]
|
||||
else:
|
||||
# Per-token aiter fusion
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
|
||||
Reference in New Issue
Block a user