[ROCm] [fusion] support ROCm aiter fusion of per tensor quant + rmsnorm (#52831)

Signed-off-by: Andy Friedrich <afriedri@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
afriedri
2026-10-16 15:19:07 -05:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Claude Opus 5.5
parent 9eb7fb9157
commit cc2e4fecf2
4 changed files with 274 additions and 6 deletions
+94
View File
@@ -1492,6 +1492,74 @@ def _rocm_aiter_rmsnorm_fp8_group_quant_fake(
)
def _rocm_aiter_fused_rms_fp8_per_tensor_static_quant_impl(
x: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
scale: torch.Tensor,
quant_dtype: torch.dtype,
) -> torch.Tensor:
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_per_tensor_static_quant
x_quant, _, _, _ = fused_rms_fp8_per_tensor_static_quant(
x,
weight,
epsilon,
scale,
None,
None,
None,
dtype_quant=quant_dtype,
res1=None,
)
return x_quant
def _rocm_aiter_fused_rms_fp8_per_tensor_static_quant_fake(
x: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
scale: torch.Tensor,
quant_dtype: torch.dtype,
) -> torch.Tensor:
return torch.empty_like(x, dtype=quant_dtype)
def _rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_impl(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
scale: torch.Tensor,
quant_dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_per_tensor_static_quant
x_quant, _, _, residual_out = fused_rms_fp8_per_tensor_static_quant(
x,
weight,
epsilon,
scale,
None,
None,
None,
dtype_quant=quant_dtype,
res1=residual,
)
return x_quant, residual_out
def _rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_fake(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
epsilon: float,
scale: torch.Tensor,
quant_dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
return torch.empty_like(x, dtype=quant_dtype), torch.empty_like(residual)
def _rocm_aiter_fused_rms_gated_fp8_group_quant_impl(
x: torch.Tensor,
weight: torch.Tensor,
@@ -3050,6 +3118,18 @@ class rocm_aiter_ops:
fake_impl=_rocm_aiter_rmsnorm_fp8_group_quant_fake,
)
direct_register_custom_op(
op_name="rocm_aiter_fused_rms_fp8_per_tensor_static_quant",
op_func=_rocm_aiter_fused_rms_fp8_per_tensor_static_quant_impl,
fake_impl=_rocm_aiter_fused_rms_fp8_per_tensor_static_quant_fake,
)
direct_register_custom_op(
op_name="rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant",
op_func=_rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_impl,
fake_impl=_rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_fake,
)
direct_register_custom_op(
op_name="rocm_aiter_fused_rms_gated_fp8_group_quant",
op_func=_rocm_aiter_fused_rms_gated_fp8_group_quant_impl,
@@ -3179,6 +3259,16 @@ class rocm_aiter_ops:
def get_rmsnorm_group_fused_quant_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant.default
@staticmethod
def get_fused_rms_fp8_per_tensor_static_quant_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_fused_rms_fp8_per_tensor_static_quant.default
@staticmethod
def get_fused_add_rms_fp8_per_tensor_static_quant_op() -> OpOverload:
return (
torch.ops.vllm.rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant.default
)
@staticmethod
def get_fused_rms_gated_fp8_group_quant_op() -> OpOverload:
"""Return the fused gated-RMSNorm + FP8 group quant custom op."""
@@ -3192,6 +3282,10 @@ class rocm_aiter_ops:
def get_per_token_quant_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_per_token_quant.default
@staticmethod
def get_per_tensor_quant_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_per_tensor_quant.default
@staticmethod
def get_group_quant_op() -> OpOverload:
return torch.ops.vllm.rocm_aiter_group_fp8_quant.default