[ROCm][DSV4.1][Perf] Fold the mHC post step into the delayed pre projection (#56513)
Signed-off-by: Fangzhou Ai <fangzhou@semianalysis.ai> Signed-off-by: Fangzhou Ai <fangzhou.ai@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -983,8 +983,12 @@ def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried):
|
||||
)
|
||||
|
||||
expected = mhc_pre_delayed_torch(*args, pre_mix=pre_mix)
|
||||
actual = object.__new__(MHCPreDelayedOp).forward_hip(*args, pre_mix=pre_mix)
|
||||
residual_out, *actual = object.__new__(MHCPreDelayedOp).forward_hip(
|
||||
*args, pre_mix=pre_mix
|
||||
)
|
||||
|
||||
# No post was requested, so the residual comes straight back.
|
||||
assert residual_out is residual
|
||||
for i in (0, 1, 3):
|
||||
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
|
||||
# The collapse is the same FP32 multiply-and-sum in both paths.
|
||||
@@ -1026,7 +1030,12 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
|
||||
broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1)
|
||||
broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
|
||||
expected = mhc_pre_delayed_torch(broadcast_residual, broadcast_fn, *args[2:], x=x)
|
||||
actual = op.forward_hip(broadcast_residual, broadcast_fn, *args[2:], x=x)
|
||||
# forward_hip leads with the residual; no post was requested here, so it is
|
||||
# the caller's own tensor and the reference's four outputs follow it.
|
||||
residual_out, *actual = op.forward_hip(
|
||||
broadcast_residual, broadcast_fn, *args[2:], x=x
|
||||
)
|
||||
assert residual_out is broadcast_residual
|
||||
assert not took_aiter, "the broadcast seam must not reach AITER"
|
||||
for i in range(4):
|
||||
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
|
||||
@@ -1268,3 +1277,68 @@ def test_deepseek_v4_mhc_broadcast_refit_refreshes_in_place(monkeypatch):
|
||||
assert layer.hc_attn_fn_broadcast is buffer
|
||||
expected = layer.hc_attn_fn.detach().view(-1, 2, 8).sum(dim=1)
|
||||
assert torch.equal(layer.hc_attn_fn_broadcast, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC),
|
||||
reason="AITER mHC required",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128])
|
||||
@pytest.mark.parametrize("carried", [False, True])
|
||||
def test_mhc_fused_post_pre_delayed_rocm_aiter(
|
||||
num_tokens, carried, default_vllm_config
|
||||
):
|
||||
"""Folding the post into the pre projection must match doing them apart.
|
||||
|
||||
The residual is bfloat16, so the two accumulation orders are allowed to
|
||||
land a rounding step apart; the tolerance here is one bfloat16 ULP.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
hc_mult, hidden_size = 4, 5120
|
||||
residual, fn, hc_scale, hc_base, _ = _rocm_mhc_inputs(
|
||||
num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult
|
||||
)
|
||||
x = torch.randn((num_tokens, hidden_size), dtype=torch.bfloat16, device=DEVICE)
|
||||
post_layer_mix = (
|
||||
torch.rand(num_tokens, hc_mult, 1, dtype=torch.float32, device=DEVICE) + 0.5
|
||||
)
|
||||
comb_res_mix = (
|
||||
torch.rand(num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=DEVICE)
|
||||
+ 0.2
|
||||
)
|
||||
pre_mix = (
|
||||
torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5
|
||||
if carried
|
||||
else None
|
||||
)
|
||||
rms_eps = hc_pre_eps = hc_sinkhorn_eps = 1e-6
|
||||
pre_args = (fn, hc_scale, hc_base, rms_eps, hc_pre_eps, hc_sinkhorn_eps, 1.0, 20)
|
||||
|
||||
next_residual = mhc_post_torch(x, residual, post_layer_mix, comb_res_mix)
|
||||
expected = (
|
||||
next_residual,
|
||||
*mhc_pre_delayed_torch(next_residual, *pre_args, pre_mix=pre_mix),
|
||||
)
|
||||
actual = MHCPreDelayedOp().forward_hip(
|
||||
residual,
|
||||
*pre_args,
|
||||
pre_mix=pre_mix,
|
||||
sublayer_out=x,
|
||||
post_layer_mix=post_layer_mix,
|
||||
comb_res_mix=comb_res_mix,
|
||||
)
|
||||
|
||||
for got, want in zip(actual, expected, strict=True):
|
||||
torch.testing.assert_close(got, want.reshape(got.shape), atol=2e-2, rtol=8e-3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC),
|
||||
reason="AITER mHC required",
|
||||
)
|
||||
def test_mhc_fused_post_pre_delayed_falls_back_for_large_batches():
|
||||
"""AITER's heuristic hands large batches back to the unfused path."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
assert not rocm_aiter_ops.mhc_fused_post_pre_delayed_prefers_unfused(1)
|
||||
assert rocm_aiter_ops.mhc_fused_post_pre_delayed_prefers_unfused(1 << 20)
|
||||
|
||||
Reference in New Issue
Block a user