[ROCm][DSV4.1][Perf] Fold the mHC post step into the delayed pre projection (#56513)

Signed-off-by: Fangzhou Ai <fangzhou@semianalysis.ai>
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Fangzhou Ai
2026-09-14 16:44:56 -05:00
committed by GitHub
co-authored by Cursor
parent bdad63c90a
commit 00972dfd72
5 changed files with 346 additions and 64 deletions
+20 -1
View File
@@ -115,6 +115,10 @@ def mhc_pre_delayed_aiter(
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""mHC pre with the pre-mix carried in from the previous sublayer.
@@ -134,6 +138,13 @@ def mhc_pre_delayed_aiter(
sinkhorn_repeat: number of sinkhorn iterations
pre_mix: shape (..., hc_mult) from the previous sublayer, or None at
model entry to select residual stream zero.
sublayer_out: attention or FFN output, shape (..., hidden_size). When
given with the two mixes below, the preceding post block is
applied here so AITER can fold it into the pre projection.
post_layer_mix: shape (..., hc_mult, 1), post gate for that block.
comb_res_mix: shape (..., hc_mult, hc_mult), residual comb for it.
residual_out: shape (..., hc_mult, hidden_size), written with the post
block's new residual. Required exactly when the post is folded in.
Returns:
post_mix: shape (..., hc_mult, 1), dtype torch.float32
@@ -156,6 +167,10 @@ def mhc_pre_delayed_aiter(
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
residual_out,
)
@@ -170,6 +185,10 @@ def _mhc_pre_delayed_aiter_fake(
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
@@ -313,7 +332,7 @@ direct_register_custom_op(
direct_register_custom_op(
op_name="mhc_pre_delayed_aiter",
op_func=mhc_pre_delayed_aiter,
mutates_args=[],
mutates_args=["residual_out"],
fake_impl=_mhc_pre_delayed_aiter_fake,
)
direct_register_custom_op(