[ROCm] [fusion] support ROCm aiter fusion of per tensor quant + rmsnorm (#52831)
Signed-off-by: Andy Friedrich <afriedri@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Claude Opus 5.5
parent
9eb7fb9157
commit
cc2e4fecf2
@@ -394,10 +394,13 @@ class MatcherQuantFP8(MatcherCustomOp):
|
||||
self.is_tma_aligned = is_tma_aligned
|
||||
|
||||
if match_rocm_aiter:
|
||||
assert not quant_key.scale.group_shape.is_per_tensor(), (
|
||||
"ROCm aiter fusion pass does not support per tensor quantization"
|
||||
)
|
||||
if quant_key.scale.group_shape.is_per_token():
|
||||
if quant_key.scale.group_shape.is_per_tensor():
|
||||
assert quant_key.scale.static, (
|
||||
"ROCm aiter fusion pass only supports static per-tensor "
|
||||
"quantization"
|
||||
)
|
||||
self.QUANT_OP = rocm_aiter_ops.get_per_tensor_quant_op()
|
||||
elif quant_key.scale.group_shape.is_per_token():
|
||||
self.QUANT_OP = rocm_aiter_ops.get_per_token_quant_op()
|
||||
else:
|
||||
assert quant_key.scale.group_shape.col == 128, (
|
||||
@@ -432,14 +435,24 @@ class MatcherQuantFP8(MatcherCustomOp):
|
||||
scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
quant_key_group_shape = self.quant_key.scale.group_shape
|
||||
if quant_key_group_shape == GroupShape.PER_TENSOR:
|
||||
# rocm_aiter_per_tensor_quant mutates `out` and `scale` and returns
|
||||
# None, so it has to be matched through auto_functionalized.
|
||||
assert scale is not None
|
||||
out = torch.empty_like(input, dtype=self.quant_key.dtype)
|
||||
result = auto_functionalized(
|
||||
self.QUANT_OP, out=out, x=input, scale=scale, is_dynamic=False
|
||||
)
|
||||
# `scale` is mutable, so its functionalized output is a distinct
|
||||
# value; return it to keep the quant node inside the match.
|
||||
return result[1], result[2]
|
||||
if quant_key_group_shape == GroupShape.PER_TOKEN:
|
||||
return self.QUANT_OP( # type: ignore[no-any-return]
|
||||
x=input,
|
||||
quant_dtype=self.quant_key.dtype,
|
||||
scale=scale,
|
||||
)
|
||||
else:
|
||||
return self.QUANT_OP(input, quant_key_group_shape.col) # type: ignore[no-any-return]
|
||||
return self.QUANT_OP(input, quant_key_group_shape.col) # type: ignore[no-any-return]
|
||||
|
||||
def forward_custom(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user