[ROCm][DSV4.1][Perf] Fold the mHC post step into the delayed pre projection (#56513)

Signed-off-by: Fangzhou Ai <fangzhou@semianalysis.ai>
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Fangzhou Ai
2026-09-14 16:44:56 -05:00
committed by GitHub
co-authored by Cursor
parent bdad63c90a
commit 00972dfd72
5 changed files with 346 additions and 64 deletions
+138 -37
View File
@@ -1676,6 +1676,75 @@ def _rocm_aiter_fp8_attn_fake(
)
def _mhc_delayed_pre_tail(
gemm_out: torch.Tensor,
sqrsum: torch.Tensor,
residual: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
pre_mix: torch.Tensor | None,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Finish a delayed mHC pre once AITER has produced the projection.
Shared by the plain and the post-fused entry points, which differ only in
which kernel computes ``gemm_out`` / ``sqrsum`` and the residual they
project. ``mhc_pre_big_fuse`` also writes a collapse against its own
pre-mix, which the delayed formulation cannot use and so discards.
Returns ``(post_mix, comb_mix, layer_input, next_pre_mix)``.
"""
from aiter.ops.mhc import mhc_pre_big_fuse
num_tokens, hc_mult, hidden_size = residual.shape
device = residual.device
with torch.device(device):
post_mix = torch.empty(
num_tokens, hc_mult, 1, dtype=torch.float32, device=device
)
comb_mix = torch.empty(
num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=device
)
unused_collapse = torch.empty(
num_tokens, hidden_size, dtype=torch.bfloat16, device=device
)
mhc_pre_big_fuse(
post_mix,
comb_mix,
unused_collapse,
gemm_out,
sqrsum,
hc_scale,
hc_base,
residual,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
next_pre_mix = torch.ops.vllm.mhc_pre_mix_triton(
gemm_out,
sqrsum,
hc_scale,
hc_base,
hc_mult,
hc_mult * hidden_size,
rms_eps,
hc_pre_eps,
)
if pre_mix is None:
# Model entry selects residual stream zero.
layer_input = residual[:, 0]
else:
layer_input = torch.ops.vllm.hc_collapse_triton(residual, pre_mix)
return post_mix, comb_mix, layer_input, next_pre_mix
# Global flag to ensure ops are registered only once
_OPS_REGISTERED = False
@@ -3611,6 +3680,10 @@ class rocm_aiter_ops:
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
sublayer_out: torch.Tensor | None = None,
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
residual_out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""mHC pre using the pre-mix carried from the previous sublayer.
@@ -3625,6 +3698,15 @@ class rocm_aiter_ops:
that redundant store is the price of not having a native delayed
kernel, and is small next to the ~140 launches it replaces.
Passing ``sublayer_out`` / ``post_layer_mix`` / ``comb_res_mix``
also applies the preceding post block. AITER then projects the new
residual in the same kernel that computes it
(``mhc_fused_post_pre_gemm_sqrsum``), which saves one launch; every
later stage is identical either way. That new residual is written to
``residual_out``, which the caller must supply in that case; it is an
output buffer rather than a return value so the op never has to hand
back one of its own inputs on the unfused path.
Returns:
post_mix: shape (..., hc_mult, 1), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
@@ -3632,8 +3714,9 @@ class rocm_aiter_ops:
next_pre_mix: shape (..., hc_mult), dtype torch.float32
"""
from aiter.ops.mhc import (
get_mhc_fused_post_pre_config,
get_mhc_pre_splitk,
mhc_pre_big_fuse,
mhc_fused_post_pre_gemm_sqrsum,
mhc_pre_gemm_sqrsum,
)
@@ -3666,9 +3749,16 @@ class rocm_aiter_ops:
pre_mix_flat = None if pre_mix is None else pre_mix.view(-1, hc_mult)
fuse_post = sublayer_out is not None
# AITER's Python wrappers allocate without explicit device arguments.
with torch.device(device):
splitk, tile_k = get_mhc_pre_splitk(num_tokens, hc_hidden_size)
if fuse_post:
# Keyed on the per-stream K, i.e. hidden_size, not hc_mult * it.
splitk, tile_m, tile_n, tile_k = get_mhc_fused_post_pre_config(
num_tokens, hidden_size
)
else:
splitk, tile_k = get_mhc_pre_splitk(num_tokens, hc_hidden_size)
# AITER pads the GEMM output to a multiple of 32 columns.
gemm_pad = torch.empty(
splitk,
@@ -3679,50 +3769,46 @@ class rocm_aiter_ops:
)
gemm_out = gemm_pad[:, :, :hc_mult3]
sqrsum = torch.empty(splitk, num_tokens, dtype=torch.float32, device=device)
mhc_pre_gemm_sqrsum(gemm_out, sqrsum, residual_flat, fn, tile_k, 0)
if fuse_post:
# Restated for the type checker, which cannot narrow the
# optional arguments through the fuse_post flag.
assert sublayer_out is not None
assert post_layer_mix is not None and comb_res_mix is not None
assert residual_out is not None
projected = residual_out.view(-1, hc_mult, hidden_size)
mhc_fused_post_pre_gemm_sqrsum(
gemm_out,
sqrsum,
projected,
sublayer_out.view(-1, hidden_size),
residual_flat,
# The kernel wants the post mix squeezed to (tokens, hc_mult).
post_layer_mix.view(-1, hc_mult, 1).squeeze(-1),
comb_res_mix.view(-1, hc_mult, hc_mult),
fn,
tile_m,
tile_n,
tile_k,
0,
)
else:
projected = residual_flat
mhc_pre_gemm_sqrsum(gemm_out, sqrsum, projected, fn, tile_k, 0)
post_mix = torch.empty(
num_tokens, hc_mult, 1, dtype=torch.float32, device=device
)
comb_mix = torch.empty(
num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=device
)
unused_collapse = torch.empty(
num_tokens, hidden_size, dtype=torch.bfloat16, device=device
)
mhc_pre_big_fuse(
post_mix,
comb_mix,
unused_collapse,
gemm_out,
sqrsum,
hc_scale,
hc_base,
residual_flat,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
next_pre_mix = torch.ops.vllm.mhc_pre_mix_triton(
post_mix, comb_mix, layer_input, next_pre_mix = _mhc_delayed_pre_tail(
gemm_out,
sqrsum,
projected,
hc_scale,
hc_base,
hc_mult,
hc_hidden_size,
pre_mix_flat,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
if pre_mix_flat is None:
# Model entry selects residual stream zero.
layer_input = residual_flat[:, 0]
else:
layer_input = torch.ops.vllm.hc_collapse_triton(residual_flat, pre_mix_flat)
return (
post_mix.view(*outer_shape, hc_mult, 1),
comb_mix.view(*outer_shape, hc_mult, hc_mult),
@@ -3730,6 +3816,21 @@ class rocm_aiter_ops:
next_pre_mix.view(*outer_shape, hc_mult),
)
@staticmethod
def mhc_fused_post_pre_delayed_prefers_unfused(num_tokens: int) -> bool:
"""True when AITER's own heuristic favours separate post and pre.
Mirrors the ``fused_m_upper_bound`` table in ``aiter.ops.mhc``: the
fused GEMM wins for decode-sized batches and loses for large ones.
"""
from aiter.jit.utils.chip_info import get_gfx_runtime
return num_tokens >= {
"gfx950": 1024,
"gfx942": 128,
"gfx1250": 1024,
}.get(get_gfx_runtime(), 1024)
@staticmethod
def hc_head(
hs_flat: torch.Tensor,