[ROCm][DSv4.1][Perf] Run the delayed mHC seams through aiter's fused Triton kernel (#58655)
Signed-off-by: Ahmed <Muhammad.Ahmed@amd.com>
This commit is contained in:
@@ -13,6 +13,7 @@ __all__ = [
|
||||
"hc_head_fused_kernel_cuda",
|
||||
"mhc_pre_aiter",
|
||||
"mhc_pre_delayed_aiter",
|
||||
"mhc_fused_post_pre_delayed_rms_norm_aiter",
|
||||
"mhc_post_aiter",
|
||||
"mhc_fused_post_pre_aiter",
|
||||
"hc_head_fused_aiter",
|
||||
|
||||
@@ -207,6 +207,79 @@ def _mhc_pre_delayed_aiter_fake(
|
||||
)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_delayed_rms_norm_aiter(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None,
|
||||
sublayer_out: torch.Tensor | None,
|
||||
post_layer_mix: torch.Tensor | None,
|
||||
comb_res_mix: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
residual_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
""":func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in:
|
||||
``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
sublayer_out,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
residual_out,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None,
|
||||
sublayer_out: torch.Tensor | None,
|
||||
post_layer_mix: torch.Tensor | None,
|
||||
comb_res_mix: torch.Tensor | None,
|
||||
norm_weight: torch.Tensor,
|
||||
norm_eps: float,
|
||||
residual_out: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return _mhc_pre_delayed_aiter_fake(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
|
||||
def mhc_post_aiter(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -335,6 +408,12 @@ direct_register_custom_op(
|
||||
mutates_args=["residual_out"],
|
||||
fake_impl=_mhc_pre_delayed_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_fused_post_pre_delayed_rms_norm_aiter",
|
||||
op_func=mhc_fused_post_pre_delayed_rms_norm_aiter,
|
||||
mutates_args=["residual_out"],
|
||||
fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_post_aiter",
|
||||
op_func=mhc_post_aiter,
|
||||
|
||||
@@ -59,9 +59,21 @@ def _aiter_mhc_op_accepts_norm(op_name: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _has_aiter_mhc_fused_post_pre_delayed_rms_norm() -> bool:
|
||||
if not HAS_AITER_MHC:
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
# The fused delayed seam kernel is only validated on gfx950.
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
HAS_AITER_MHC_FUSED = _has_aiter_mhc_fused()
|
||||
HAS_AITER_MHC_PRE_NORM = _aiter_mhc_op_accepts_norm("mhc_pre")
|
||||
HAS_AITER_MHC_FUSED_NORM = _aiter_mhc_op_accepts_norm("mhc_fused_post_pre")
|
||||
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM = (
|
||||
_has_aiter_mhc_fused_post_pre_delayed_rms_norm()
|
||||
)
|
||||
|
||||
|
||||
def _aiter_mhc_supported(
|
||||
@@ -388,10 +400,42 @@ class MHCPreDelayedOp(CustomOp):
|
||||
post_layer_mix: torch.Tensor | None = None,
|
||||
comb_res_mix: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# The AITER delayed path drives mhc_pre_gemm_sqrsum against `residual`
|
||||
# and folds no RMSNorm, so it cannot serve the model-entry broadcast
|
||||
# (which projects a narrower `x`) or a fused norm. Both are handled by
|
||||
# TileLang, or by the reference when TileLang is unavailable.
|
||||
# aiter's fused delayed seam: post-mix, gate projection, and the
|
||||
# collapse with the carried pre-mix and its RMSNorm. It always applies
|
||||
# the norm and projects `residual` itself.
|
||||
if (
|
||||
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
|
||||
and x is None
|
||||
and norm_weight is not None
|
||||
and _aiter_mhc_supported(residual, norm_weight, supports_norm=True)
|
||||
):
|
||||
residual_out = (
|
||||
torch.empty_like(residual) if sublayer_out is not None else None
|
||||
)
|
||||
rest = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
sublayer_out,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
residual_out,
|
||||
)
|
||||
return (residual if residual_out is None else residual_out), *rest
|
||||
# The unfused AITER delayed path drives mhc_pre_gemm_sqrsum against
|
||||
# `residual` and folds no RMSNorm, so it cannot serve the model-entry
|
||||
# broadcast (which projects a narrower `x`) or a fused norm the branch
|
||||
# above did not take. Both are handled by TileLang, or by the reference
|
||||
# when TileLang is unavailable.
|
||||
if (
|
||||
x is None
|
||||
and norm_weight is None
|
||||
|
||||
Reference in New Issue
Block a user