[ROCm][DSV4.1][Perf] Fold the mHC post step into the delayed pre projection (#56513)
Signed-off-by: Fangzhou Ai <fangzhou@semianalysis.ai> Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -115,6 +115,10 @@ def mhc_pre_delayed_aiter(
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
sublayer_out: torch.Tensor | None = None,
|
||||
post_layer_mix: torch.Tensor | None = None,
|
||||
comb_res_mix: torch.Tensor | None = None,
|
||||
residual_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""mHC pre with the pre-mix carried in from the previous sublayer.
|
||||
|
||||
@@ -134,6 +138,13 @@ def mhc_pre_delayed_aiter(
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
pre_mix: shape (..., hc_mult) from the previous sublayer, or None at
|
||||
model entry to select residual stream zero.
|
||||
sublayer_out: attention or FFN output, shape (..., hidden_size). When
|
||||
given with the two mixes below, the preceding post block is
|
||||
applied here so AITER can fold it into the pre projection.
|
||||
post_layer_mix: shape (..., hc_mult, 1), post gate for that block.
|
||||
comb_res_mix: shape (..., hc_mult, hc_mult), residual comb for it.
|
||||
residual_out: shape (..., hc_mult, hidden_size), written with the post
|
||||
block's new residual. Required exactly when the post is folded in.
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult, 1), dtype torch.float32
|
||||
@@ -156,6 +167,10 @@ def mhc_pre_delayed_aiter(
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
sublayer_out,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
residual_out,
|
||||
)
|
||||
|
||||
|
||||
@@ -170,6 +185,10 @@ def _mhc_pre_delayed_aiter_fake(
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
sublayer_out: torch.Tensor | None = None,
|
||||
post_layer_mix: torch.Tensor | None = None,
|
||||
comb_res_mix: torch.Tensor | None = None,
|
||||
residual_out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
@@ -313,7 +332,7 @@ direct_register_custom_op(
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_delayed_aiter",
|
||||
op_func=mhc_pre_delayed_aiter,
|
||||
mutates_args=[],
|
||||
mutates_args=["residual_out"],
|
||||
fake_impl=_mhc_pre_delayed_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
|
||||
Reference in New Issue
Block a user