[ROCm][DSv4.1][Perf] Run the delayed mHC seams through aiter's fused Triton kernel (#58655)
Signed-off-by: Ahmed <Muhammad.Ahmed@amd.com>
This commit is contained in:
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.mhc import (
|
||||
HAS_AITER_MHC,
|
||||
HAS_AITER_MHC_FUSED,
|
||||
HAS_AITER_MHC_FUSED_NORM,
|
||||
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM,
|
||||
HAS_AITER_MHC_PRE_NORM,
|
||||
HAS_TILELANG_MHC,
|
||||
MHCFusedPostPreOp,
|
||||
@@ -1300,18 +1301,109 @@ def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried):
|
||||
torch.testing.assert_close(actual[2], expected[2], atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM),
|
||||
reason="AITER fused delayed-seam kernel required (gfx950)",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128, 1024])
|
||||
@pytest.mark.parametrize("seam", ["post", "no_post", "identity_pre"])
|
||||
def test_mhc_pre_delayed_rocm_aiter_fused_rms_norm(num_tokens, seam):
|
||||
"""One fused kernel must match the whole reference seam: the post block
|
||||
(the previous seam's post/comb gates fold ``sublayer_out`` into the
|
||||
streams), the delayed pre block (projection + Sinkhorn producing the next
|
||||
seam's post/comb/pre gates, plus the collapse with the carried pre-mix),
|
||||
and the RMSNorm folded onto the collapse.
|
||||
|
||||
``num_tokens`` spans the kernel's split-K config buckets. ``seam`` covers
|
||||
the three call shapes the model makes: a regular seam (post folded in), an
|
||||
Engram seam (post already applied), and the draft entry seam
|
||||
(``pre_mix=None`` = identity pre-mix).
|
||||
"""
|
||||
set_random_seed(0)
|
||||
hc_mult, hidden_size = 4, 5120
|
||||
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs(
|
||||
num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult
|
||||
)
|
||||
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
|
||||
args = (
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
1.0,
|
||||
20,
|
||||
)
|
||||
|
||||
pre_mix = (
|
||||
None
|
||||
if seam == "identity_pre"
|
||||
else torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5
|
||||
)
|
||||
post_kwargs = {}
|
||||
residual_ref = residual
|
||||
if seam == "post":
|
||||
sublayer_out = torch.randn(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
post_layer_mix = 2 * torch.rand(
|
||||
num_tokens, hc_mult, 1, dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
comb_res_mix = torch.softmax(
|
||||
torch.randn(num_tokens, hc_mult, hc_mult, device=DEVICE), dim=-1
|
||||
)
|
||||
post_kwargs = dict(
|
||||
sublayer_out=sublayer_out,
|
||||
post_layer_mix=post_layer_mix,
|
||||
comb_res_mix=comb_res_mix,
|
||||
)
|
||||
residual_ref = mhc_post_torch(
|
||||
sublayer_out, residual, post_layer_mix, comb_res_mix
|
||||
)
|
||||
|
||||
expected = mhc_pre_delayed_torch(residual_ref, *args[1:], pre_mix=pre_mix)
|
||||
expected_layer_input = F.rms_norm(
|
||||
expected[2], (hidden_size,), norm_weight, norm_eps
|
||||
)
|
||||
|
||||
residual_out, *actual = object.__new__(MHCPreDelayedOp).forward_hip(
|
||||
*args,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
**post_kwargs,
|
||||
)
|
||||
|
||||
if seam == "post":
|
||||
torch.testing.assert_close(residual_out, residual_ref)
|
||||
else:
|
||||
# No post was requested, so the residual comes straight back.
|
||||
assert residual_out is residual
|
||||
# Gates: post_mix, comb_mix, next_pre_mix (bf16 MFMA vs the fp32 reference).
|
||||
for i in (0, 1, 3):
|
||||
torch.testing.assert_close(actual[i], expected[i], atol=5e-4, rtol=1e-3)
|
||||
# The collapse is bf16 and the kernel projects the once-rounded R', so
|
||||
# near-zero values need an absolute tolerance (same bound as aiter's UT).
|
||||
torch.testing.assert_close(actual[2], expected_layer_input, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC),
|
||||
reason="AITER mHC required",
|
||||
)
|
||||
def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
|
||||
"""The broadcast seam and a fused norm must not take the AITER path.
|
||||
"""The broadcast seam must not take the AITER path; a fused norm must not
|
||||
take the unfused AITER op.
|
||||
|
||||
Neither is expressible with AITER's pre kernels: the broadcast projects a
|
||||
narrower ``x``, and ``mhc_pre_gemm_sqrsum`` folds no RMSNorm. What is
|
||||
asserted here is the routing decision, which is what this gate owns; the
|
||||
numerics of whichever fallback it lands on are covered by
|
||||
``test_deepseek_v41_mhc_pre_delayed``.
|
||||
The broadcast projects a narrower ``x``, which no AITER pre kernel takes;
|
||||
``mhc_pre_gemm_sqrsum`` folds no RMSNorm, so a fused norm routes to the
|
||||
fused seam kernel where it exists (gfx950) and away from AITER otherwise.
|
||||
What is asserted here is the routing decision, which is what this gate
|
||||
owns; the numerics of the paths are covered by
|
||||
``test_deepseek_v41_mhc_pre_delayed`` and
|
||||
``test_mhc_pre_delayed_rocm_aiter_fused_rms_norm``.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
hc_mult, hidden_size = 4, 5120
|
||||
@@ -1331,6 +1423,18 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(torch.ops.vllm, "mhc_pre_delayed_aiter", spy)
|
||||
|
||||
fused_op = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter
|
||||
took_fused = False
|
||||
|
||||
def fused_spy(*spy_args, **spy_kwargs):
|
||||
nonlocal took_fused
|
||||
took_fused = True
|
||||
return fused_op(*spy_args, **spy_kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
torch.ops.vllm, "mhc_fused_post_pre_delayed_rms_norm_aiter", fused_spy
|
||||
)
|
||||
|
||||
x = residual[:, 0].contiguous()
|
||||
broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1)
|
||||
broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
|
||||
@@ -1342,11 +1446,15 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
|
||||
)
|
||||
assert residual_out is broadcast_residual
|
||||
assert not took_aiter, "the broadcast seam must not reach AITER"
|
||||
assert not took_fused, "the broadcast seam must not reach AITER"
|
||||
for i in range(4):
|
||||
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
|
||||
|
||||
# A fused norm must not reach the unfused AITER op; where the fused seam
|
||||
# kernel exists (gfx950) it routes there instead.
|
||||
op.forward_hip(*args, norm_weight=norm_weight, norm_eps=1e-6)
|
||||
assert not took_aiter, "a fused norm must not reach AITER"
|
||||
assert not took_aiter, "a fused norm must not reach the unfused AITER op"
|
||||
assert took_fused == mhc_layers.HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
|
||||
|
||||
# Positive control: the same inputs without a norm do take the AITER path,
|
||||
# so the two declines above are the gate discriminating rather than the
|
||||
|
||||
Reference in New Issue
Block a user