[ROCm][DSV4.1][Perf] Fold the mHC post step into the delayed pre projection (#56513)

Signed-off-by: Fangzhou Ai <fangzhou@semianalysis.ai>
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Fangzhou Ai
2026-09-14 16:44:56 -05:00
committed by GitHub
co-authored by Cursor
parent bdad63c90a
commit 00972dfd72
5 changed files with 346 additions and 64 deletions
+20 -1
View File
@@ -115,6 +115,10 @@ def mhc_pre_delayed_aiter(
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""mHC pre with the pre-mix carried in from the previous sublayer.
@@ -134,6 +138,13 @@ def mhc_pre_delayed_aiter(
sinkhorn_repeat: number of sinkhorn iterations
pre_mix: shape (..., hc_mult) from the previous sublayer, or None at
model entry to select residual stream zero.
sublayer_out: attention or FFN output, shape (..., hidden_size). When
given with the two mixes below, the preceding post block is
applied here so AITER can fold it into the pre projection.
post_layer_mix: shape (..., hc_mult, 1), post gate for that block.
comb_res_mix: shape (..., hc_mult, hc_mult), residual comb for it.
residual_out: shape (..., hc_mult, hidden_size), written with the post
block's new residual. Required exactly when the post is folded in.
Returns:
post_mix: shape (..., hc_mult, 1), dtype torch.float32
@@ -156,6 +167,10 @@ def mhc_pre_delayed_aiter(
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
residual_out,
)
@@ -170,6 +185,10 @@ def _mhc_pre_delayed_aiter_fake(
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
@@ -313,7 +332,7 @@ direct_register_custom_op(
direct_register_custom_op(
op_name="mhc_pre_delayed_aiter",
op_func=mhc_pre_delayed_aiter,
mutates_args=[],
mutates_args=["residual_out"],
fake_impl=_mhc_pre_delayed_aiter_fake,
)
direct_register_custom_op(
+75 -7
View File
@@ -299,7 +299,13 @@ class MHCPreDelayedOp(CustomOp):
Same gates as :class:`MHCPreOp`, but the stream collapse applies the
caller's ``pre_mix`` and this sublayer's pre-mix is returned for the next
sublayer seam. Returns post_mix, comb_mix, layer_input, next_pre_mix.
sublayer seam.
Passing ``sublayer_out`` / ``post_layer_mix`` / ``comb_res_mix`` also
applies the preceding post block, which lets AITER fold it into the pre
projection and saves a launch. Returns residual, post_mix, comb_mix,
layer_input, next_pre_mix; ``residual`` is the caller's own tensor when
no post is requested.
"""
# --8<-- [end:mhc_pre_delayed]
@@ -307,6 +313,24 @@ class MHCPreDelayedOp(CustomOp):
def enabled(cls) -> bool:
return True
def __init__(self) -> None:
super().__init__()
# Built here, not lazily: CustomOp resolves dispatch against the
# current vLLM config, which is only set during model construction.
self._post = MHCPostOp()
def _maybe_post(
self,
residual: torch.Tensor,
sublayer_out: torch.Tensor | None,
post_layer_mix: torch.Tensor | None,
comb_res_mix: torch.Tensor | None,
) -> torch.Tensor:
if sublayer_out is None:
return residual
assert post_layer_mix is not None and comb_res_mix is not None
return self._post(sublayer_out, residual, post_layer_mix, comb_res_mix)
def forward_cuda(
self,
residual: torch.Tensor,
@@ -322,8 +346,14 @@ class MHCPreDelayedOp(CustomOp):
x: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
return torch.ops.vllm.mhc_pre_delayed_tilelang(
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
residual = self._maybe_post(
residual, sublayer_out, post_layer_mix, comb_res_mix
)
return residual, *torch.ops.vllm.mhc_pre_delayed_tilelang(
residual,
fn,
hc_scale,
@@ -354,7 +384,10 @@ class MHCPreDelayedOp(CustomOp):
x: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# The AITER delayed path drives mhc_pre_gemm_sqrsum against `residual`
# and folds no RMSNorm, so it cannot serve the model-entry broadcast
# (which projects a narrower `x`) or a fused norm. Both are handled by
@@ -364,7 +397,26 @@ class MHCPreDelayedOp(CustomOp):
and norm_weight is None
and _aiter_mhc_supported(residual, None, supports_norm=False)
):
return torch.ops.vllm.mhc_pre_delayed_aiter(
from vllm._aiter_ops import rocm_aiter_ops
num_tokens = residual.numel() // (residual.shape[-1] * residual.shape[-2])
# Folding the post in only pays while the residual still fits in
# cache; past that AITER's heuristic wants the separate kernels,
# which can ask for a non-temporal store and a large-m split-k.
if sublayer_out is not None and (
rocm_aiter_ops.mhc_fused_post_pre_delayed_prefers_unfused(num_tokens)
):
residual = self._maybe_post(
residual, sublayer_out, post_layer_mix, comb_res_mix
)
sublayer_out = None
# The op writes the folded post's residual into this buffer rather
# than returning it, since on the unfused path it has none of its
# own and an op must not hand back one of its inputs.
residual_out = (
torch.empty_like(residual) if sublayer_out is not None else None
)
rest = torch.ops.vllm.mhc_pre_delayed_aiter(
residual,
fn,
hc_scale,
@@ -375,9 +427,17 @@ class MHCPreDelayedOp(CustomOp):
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
residual_out,
)
return (residual if residual_out is None else residual_out), *rest
residual = self._maybe_post(
residual, sublayer_out, post_layer_mix, comb_res_mix
)
if HAS_TILELANG_MHC:
return torch.ops.vllm.mhc_pre_delayed_tilelang(
return residual, *torch.ops.vllm.mhc_pre_delayed_tilelang(
residual,
fn,
hc_scale,
@@ -392,6 +452,7 @@ class MHCPreDelayedOp(CustomOp):
norm_weight,
norm_eps,
)
# The post is already applied, so it must not be requested again.
return self.forward_native(
residual,
fn,
@@ -423,7 +484,13 @@ class MHCPreDelayedOp(CustomOp):
x: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
residual = self._maybe_post(
residual, sublayer_out, post_layer_mix, comb_res_mix
)
post_mix, comb_mix, layer_input, next_pre_mix = (
mhc_kernels.mhc_pre_delayed_torch(
residual,
@@ -440,6 +507,7 @@ class MHCPreDelayedOp(CustomOp):
)
)
return (
residual,
post_mix,
comb_mix,
_apply_mhc_norm(layer_input, norm_weight, norm_eps),