[ROCm] [fusion] support ROCm aiter fusion of per tensor quant + rmsnorm (#52831)
Signed-off-by: Andy Friedrich <afriedri@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Claude Opus 5.5
parent
9eb7fb9157
commit
cc2e4fecf2
@@ -1492,6 +1492,74 @@ def _rocm_aiter_rmsnorm_fp8_group_quant_fake(
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_fused_rms_fp8_per_tensor_static_quant_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
scale: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_per_tensor_static_quant
|
||||
|
||||
x_quant, _, _, _ = fused_rms_fp8_per_tensor_static_quant(
|
||||
x,
|
||||
weight,
|
||||
epsilon,
|
||||
scale,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
dtype_quant=quant_dtype,
|
||||
res1=None,
|
||||
)
|
||||
return x_quant
|
||||
|
||||
|
||||
def _rocm_aiter_fused_rms_fp8_per_tensor_static_quant_fake(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
scale: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(x, dtype=quant_dtype)
|
||||
|
||||
|
||||
def _rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_impl(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
scale: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from aiter.ops.triton.fused_fp8_quant import fused_rms_fp8_per_tensor_static_quant
|
||||
|
||||
x_quant, _, _, residual_out = fused_rms_fp8_per_tensor_static_quant(
|
||||
x,
|
||||
weight,
|
||||
epsilon,
|
||||
scale,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
dtype_quant=quant_dtype,
|
||||
res1=residual,
|
||||
)
|
||||
return x_quant, residual_out
|
||||
|
||||
|
||||
def _rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
scale: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.empty_like(x, dtype=quant_dtype), torch.empty_like(residual)
|
||||
|
||||
|
||||
def _rocm_aiter_fused_rms_gated_fp8_group_quant_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
@@ -3050,6 +3118,18 @@ class rocm_aiter_ops:
|
||||
fake_impl=_rocm_aiter_rmsnorm_fp8_group_quant_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_fused_rms_fp8_per_tensor_static_quant",
|
||||
op_func=_rocm_aiter_fused_rms_fp8_per_tensor_static_quant_impl,
|
||||
fake_impl=_rocm_aiter_fused_rms_fp8_per_tensor_static_quant_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant",
|
||||
op_func=_rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_impl,
|
||||
fake_impl=_rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_fused_rms_gated_fp8_group_quant",
|
||||
op_func=_rocm_aiter_fused_rms_gated_fp8_group_quant_impl,
|
||||
@@ -3179,6 +3259,16 @@ class rocm_aiter_ops:
|
||||
def get_rmsnorm_group_fused_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant.default
|
||||
|
||||
@staticmethod
|
||||
def get_fused_rms_fp8_per_tensor_static_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_fused_rms_fp8_per_tensor_static_quant.default
|
||||
|
||||
@staticmethod
|
||||
def get_fused_add_rms_fp8_per_tensor_static_quant_op() -> OpOverload:
|
||||
return (
|
||||
torch.ops.vllm.rocm_aiter_fused_add_rms_fp8_per_tensor_static_quant.default
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_fused_rms_gated_fp8_group_quant_op() -> OpOverload:
|
||||
"""Return the fused gated-RMSNorm + FP8 group quant custom op."""
|
||||
@@ -3192,6 +3282,10 @@ class rocm_aiter_ops:
|
||||
def get_per_token_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_per_token_quant.default
|
||||
|
||||
@staticmethod
|
||||
def get_per_tensor_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_per_tensor_quant.default
|
||||
|
||||
@staticmethod
|
||||
def get_group_quant_op() -> OpOverload:
|
||||
return torch.ops.vllm.rocm_aiter_group_fp8_quant.default
|
||||
|
||||
Reference in New Issue
Block a user