[ROCm][DSV4.1][Perf] Fold the mHC post step into the delayed pre projection (#56513)

Signed-off-by: Fangzhou Ai <fangzhou@semianalysis.ai>
Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Fangzhou Ai
2026-09-14 16:44:56 -05:00
committed by GitHub
co-authored by Cursor
parent bdad63c90a
commit 00972dfd72
5 changed files with 346 additions and 64 deletions
+76 -2
View File
@@ -983,8 +983,12 @@ def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried):
)
expected = mhc_pre_delayed_torch(*args, pre_mix=pre_mix)
actual = object.__new__(MHCPreDelayedOp).forward_hip(*args, pre_mix=pre_mix)
residual_out, *actual = object.__new__(MHCPreDelayedOp).forward_hip(
*args, pre_mix=pre_mix
)
# No post was requested, so the residual comes straight back.
assert residual_out is residual
for i in (0, 1, 3):
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
# The collapse is the same FP32 multiply-and-sum in both paths.
@@ -1026,7 +1030,12 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1)
broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
expected = mhc_pre_delayed_torch(broadcast_residual, broadcast_fn, *args[2:], x=x)
actual = op.forward_hip(broadcast_residual, broadcast_fn, *args[2:], x=x)
# forward_hip leads with the residual; no post was requested here, so it is
# the caller's own tensor and the reference's four outputs follow it.
residual_out, *actual = op.forward_hip(
broadcast_residual, broadcast_fn, *args[2:], x=x
)
assert residual_out is broadcast_residual
assert not took_aiter, "the broadcast seam must not reach AITER"
for i in range(4):
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
@@ -1268,3 +1277,68 @@ def test_deepseek_v4_mhc_broadcast_refit_refreshes_in_place(monkeypatch):
assert layer.hc_attn_fn_broadcast is buffer
expected = layer.hc_attn_fn.detach().view(-1, 2, 8).sum(dim=1)
assert torch.equal(layer.hc_attn_fn_broadcast, expected)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC),
reason="AITER mHC required",
)
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128])
@pytest.mark.parametrize("carried", [False, True])
def test_mhc_fused_post_pre_delayed_rocm_aiter(
num_tokens, carried, default_vllm_config
):
"""Folding the post into the pre projection must match doing them apart.
The residual is bfloat16, so the two accumulation orders are allowed to
land a rounding step apart; the tolerance here is one bfloat16 ULP.
"""
set_random_seed(0)
hc_mult, hidden_size = 4, 5120
residual, fn, hc_scale, hc_base, _ = _rocm_mhc_inputs(
num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult
)
x = torch.randn((num_tokens, hidden_size), dtype=torch.bfloat16, device=DEVICE)
post_layer_mix = (
torch.rand(num_tokens, hc_mult, 1, dtype=torch.float32, device=DEVICE) + 0.5
)
comb_res_mix = (
torch.rand(num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=DEVICE)
+ 0.2
)
pre_mix = (
torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5
if carried
else None
)
rms_eps = hc_pre_eps = hc_sinkhorn_eps = 1e-6
pre_args = (fn, hc_scale, hc_base, rms_eps, hc_pre_eps, hc_sinkhorn_eps, 1.0, 20)
next_residual = mhc_post_torch(x, residual, post_layer_mix, comb_res_mix)
expected = (
next_residual,
*mhc_pre_delayed_torch(next_residual, *pre_args, pre_mix=pre_mix),
)
actual = MHCPreDelayedOp().forward_hip(
residual,
*pre_args,
pre_mix=pre_mix,
sublayer_out=x,
post_layer_mix=post_layer_mix,
comb_res_mix=comb_res_mix,
)
for got, want in zip(actual, expected, strict=True):
torch.testing.assert_close(got, want.reshape(got.shape), atol=2e-2, rtol=8e-3)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC),
reason="AITER mHC required",
)
def test_mhc_fused_post_pre_delayed_falls_back_for_large_batches():
"""AITER's heuristic hands large batches back to the unfused path."""
from vllm._aiter_ops import rocm_aiter_ops
assert not rocm_aiter_ops.mhc_fused_post_pre_delayed_prefers_unfused(1)
assert rocm_aiter_ops.mhc_fused_post_pre_delayed_prefers_unfused(1 << 20)