[ROCm][DSv4.1][Perf] Run the delayed mHC seams through aiter's fused Triton kernel (#58655)

Signed-off-by: Ahmed <Muhammad.Ahmed@amd.com>
This commit is contained in:
Muhammad Ahmed
2026-09-29 10:19:10 +08:00
committed by GitHub
parent 7230dfea50
commit 0af34418e9
6 changed files with 323 additions and 15 deletions
+115 -7
View File
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.mhc import (
HAS_AITER_MHC,
HAS_AITER_MHC_FUSED,
HAS_AITER_MHC_FUSED_NORM,
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM,
HAS_AITER_MHC_PRE_NORM,
HAS_TILELANG_MHC,
MHCFusedPostPreOp,
@@ -1300,18 +1301,109 @@ def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried):
torch.testing.assert_close(actual[2], expected[2], atol=0, rtol=0)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM),
reason="AITER fused delayed-seam kernel required (gfx950)",
)
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128, 1024])
@pytest.mark.parametrize("seam", ["post", "no_post", "identity_pre"])
def test_mhc_pre_delayed_rocm_aiter_fused_rms_norm(num_tokens, seam):
"""One fused kernel must match the whole reference seam: the post block
(the previous seam's post/comb gates fold ``sublayer_out`` into the
streams), the delayed pre block (projection + Sinkhorn producing the next
seam's post/comb/pre gates, plus the collapse with the carried pre-mix),
and the RMSNorm folded onto the collapse.
``num_tokens`` spans the kernel's split-K config buckets. ``seam`` covers
the three call shapes the model makes: a regular seam (post folded in), an
Engram seam (post already applied), and the draft entry seam
(``pre_mix=None`` = identity pre-mix).
"""
set_random_seed(0)
hc_mult, hidden_size = 4, 5120
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs(
num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult
)
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
args = (
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
1.0,
20,
)
pre_mix = (
None
if seam == "identity_pre"
else torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5
)
post_kwargs = {}
residual_ref = residual
if seam == "post":
sublayer_out = torch.randn(
num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE
)
post_layer_mix = 2 * torch.rand(
num_tokens, hc_mult, 1, dtype=torch.float32, device=DEVICE
)
comb_res_mix = torch.softmax(
torch.randn(num_tokens, hc_mult, hc_mult, device=DEVICE), dim=-1
)
post_kwargs = dict(
sublayer_out=sublayer_out,
post_layer_mix=post_layer_mix,
comb_res_mix=comb_res_mix,
)
residual_ref = mhc_post_torch(
sublayer_out, residual, post_layer_mix, comb_res_mix
)
expected = mhc_pre_delayed_torch(residual_ref, *args[1:], pre_mix=pre_mix)
expected_layer_input = F.rms_norm(
expected[2], (hidden_size,), norm_weight, norm_eps
)
residual_out, *actual = object.__new__(MHCPreDelayedOp).forward_hip(
*args,
pre_mix=pre_mix,
norm_weight=norm_weight,
norm_eps=norm_eps,
**post_kwargs,
)
if seam == "post":
torch.testing.assert_close(residual_out, residual_ref)
else:
# No post was requested, so the residual comes straight back.
assert residual_out is residual
# Gates: post_mix, comb_mix, next_pre_mix (bf16 MFMA vs the fp32 reference).
for i in (0, 1, 3):
torch.testing.assert_close(actual[i], expected[i], atol=5e-4, rtol=1e-3)
# The collapse is bf16 and the kernel projects the once-rounded R', so
# near-zero values need an absolute tolerance (same bound as aiter's UT).
torch.testing.assert_close(actual[2], expected_layer_input, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC),
reason="AITER mHC required",
)
def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
"""The broadcast seam and a fused norm must not take the AITER path.
"""The broadcast seam must not take the AITER path; a fused norm must not
take the unfused AITER op.
Neither is expressible with AITER's pre kernels: the broadcast projects a
narrower ``x``, and ``mhc_pre_gemm_sqrsum`` folds no RMSNorm. What is
asserted here is the routing decision, which is what this gate owns; the
numerics of whichever fallback it lands on are covered by
``test_deepseek_v41_mhc_pre_delayed``.
The broadcast projects a narrower ``x``, which no AITER pre kernel takes;
``mhc_pre_gemm_sqrsum`` folds no RMSNorm, so a fused norm routes to the
fused seam kernel where it exists (gfx950) and away from AITER otherwise.
What is asserted here is the routing decision, which is what this gate
owns; the numerics of the paths are covered by
``test_deepseek_v41_mhc_pre_delayed`` and
``test_mhc_pre_delayed_rocm_aiter_fused_rms_norm``.
"""
set_random_seed(0)
hc_mult, hidden_size = 4, 5120
@@ -1331,6 +1423,18 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
monkeypatch.setattr(torch.ops.vllm, "mhc_pre_delayed_aiter", spy)
fused_op = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter
took_fused = False
def fused_spy(*spy_args, **spy_kwargs):
nonlocal took_fused
took_fused = True
return fused_op(*spy_args, **spy_kwargs)
monkeypatch.setattr(
torch.ops.vllm, "mhc_fused_post_pre_delayed_rms_norm_aiter", fused_spy
)
x = residual[:, 0].contiguous()
broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1)
broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
@@ -1342,11 +1446,15 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
)
assert residual_out is broadcast_residual
assert not took_aiter, "the broadcast seam must not reach AITER"
assert not took_fused, "the broadcast seam must not reach AITER"
for i in range(4):
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
# A fused norm must not reach the unfused AITER op; where the fused seam
# kernel exists (gfx950) it routes there instead.
op.forward_hip(*args, norm_weight=norm_weight, norm_eps=1e-6)
assert not took_aiter, "a fused norm must not reach AITER"
assert not took_aiter, "a fused norm must not reach the unfused AITER op"
assert took_fused == mhc_layers.HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
# Positive control: the same inputs without a norm do take the AITER path,
# so the two declines above are the gate discriminating rather than the
+51
View File
@@ -4235,6 +4235,57 @@ class rocm_aiter_ops:
next_pre_mix.view(*outer_shape, hc_mult),
)
@staticmethod
def mhc_fused_post_pre_delayed_rms_norm(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None,
sublayer_out: torch.Tensor | None,
post_layer_mix: torch.Tensor | None,
comb_res_mix: torch.Tensor | None,
norm_weight: torch.Tensor,
norm_eps: float,
residual_out: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
""":meth:`mhc_pre_delayed` with the RMSNorm of the collapse folded in.
Runs aiter's fused Triton kernel ``mhc_fused_post_pre_delayed_rmsnorm``;
the returned ``layer_input`` is already normalised with ``norm_weight`` /
``norm_eps``.
"""
from aiter.ops.triton.fusions.mhc_fused_post_pre_delayed_rmsnorm import (
mhc_fused_post_pre_delayed_rmsnorm,
)
_, post_mix, comb_mix, layer_input, next_pre_mix = (
mhc_fused_post_pre_delayed_rmsnorm(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
norm_weight,
norm_eps,
residual_out=residual_out,
)
)
return post_mix, comb_mix, layer_input, next_pre_mix
@staticmethod
def mhc_fused_post_pre_delayed_prefers_unfused(num_tokens: int) -> bool:
"""True when AITER's own heuristic favours separate post and pre.
@@ -13,6 +13,7 @@ __all__ = [
"hc_head_fused_kernel_cuda",
"mhc_pre_aiter",
"mhc_pre_delayed_aiter",
"mhc_fused_post_pre_delayed_rms_norm_aiter",
"mhc_post_aiter",
"mhc_fused_post_pre_aiter",
"hc_head_fused_aiter",
+79
View File
@@ -207,6 +207,79 @@ def _mhc_pre_delayed_aiter_fake(
)
def mhc_fused_post_pre_delayed_rms_norm_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None,
sublayer_out: torch.Tensor | None,
post_layer_mix: torch.Tensor | None,
comb_res_mix: torch.Tensor | None,
norm_weight: torch.Tensor,
norm_eps: float,
residual_out: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
""":func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in:
``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``."""
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
norm_weight,
norm_eps,
residual_out,
)
def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None,
sublayer_out: torch.Tensor | None,
post_layer_mix: torch.Tensor | None,
comb_res_mix: torch.Tensor | None,
norm_weight: torch.Tensor,
norm_eps: float,
residual_out: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
return _mhc_pre_delayed_aiter_fake(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
def mhc_post_aiter(
x: torch.Tensor,
residual: torch.Tensor,
@@ -335,6 +408,12 @@ direct_register_custom_op(
mutates_args=["residual_out"],
fake_impl=_mhc_pre_delayed_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_fused_post_pre_delayed_rms_norm_aiter",
op_func=mhc_fused_post_pre_delayed_rms_norm_aiter,
mutates_args=["residual_out"],
fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_post_aiter",
op_func=mhc_post_aiter,
+48 -4
View File
@@ -59,9 +59,21 @@ def _aiter_mhc_op_accepts_norm(op_name: str) -> bool:
return False
def _has_aiter_mhc_fused_post_pre_delayed_rms_norm() -> bool:
if not HAS_AITER_MHC:
return False
from vllm.platforms.rocm import on_gfx950
# The fused delayed seam kernel is only validated on gfx950.
return on_gfx950()
HAS_AITER_MHC_FUSED = _has_aiter_mhc_fused()
HAS_AITER_MHC_PRE_NORM = _aiter_mhc_op_accepts_norm("mhc_pre")
HAS_AITER_MHC_FUSED_NORM = _aiter_mhc_op_accepts_norm("mhc_fused_post_pre")
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM = (
_has_aiter_mhc_fused_post_pre_delayed_rms_norm()
)
def _aiter_mhc_supported(
@@ -388,10 +400,42 @@ class MHCPreDelayedOp(CustomOp):
post_layer_mix: torch.Tensor | None = None,
comb_res_mix: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# The AITER delayed path drives mhc_pre_gemm_sqrsum against `residual`
# and folds no RMSNorm, so it cannot serve the model-entry broadcast
# (which projects a narrower `x`) or a fused norm. Both are handled by
# TileLang, or by the reference when TileLang is unavailable.
# aiter's fused delayed seam: post-mix, gate projection, and the
# collapse with the carried pre-mix and its RMSNorm. It always applies
# the norm and projects `residual` itself.
if (
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
and x is None
and norm_weight is not None
and _aiter_mhc_supported(residual, norm_weight, supports_norm=True)
):
residual_out = (
torch.empty_like(residual) if sublayer_out is not None else None
)
rest = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
sublayer_out,
post_layer_mix,
comb_res_mix,
norm_weight,
norm_eps,
residual_out,
)
return (residual if residual_out is None else residual_out), *rest
# The unfused AITER delayed path drives mhc_pre_gemm_sqrsum against
# `residual` and folds no RMSNorm, so it cannot serve the model-entry
# broadcast (which projects a narrower `x`) or a fused norm the branch
# above did not take. Both are handled by TileLang, or by the reference
# when TileLang is unavailable.
if (
x is None
and norm_weight is None
+29 -4
View File
@@ -22,7 +22,11 @@ from vllm.model_executor.layers.fused_moe import (
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import MHCPostOp, MHCPreDelayedOp
from vllm.model_executor.layers.mhc import (
HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM,
MHCPostOp,
MHCPreDelayedOp,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -251,6 +255,10 @@ class DeepseekV4DecoderLayer(nn.Module):
)
self.mhc_pre_delayed = MHCPreDelayedOp()
self.mhc_post = MHCPostOp()
# Where aiter's fused seam kernel runs (gfx950), it folds the following
# attn_norm / ffn_norm into its collapse, so the separate norms are
# skipped for the seams it takes.
self.fuse_seam_norm = HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM
@staticmethod
def _hc_collapse(x: torch.Tensor, pre_mix: torch.Tensor) -> torch.Tensor:
@@ -269,6 +277,11 @@ class DeepseekV4DecoderLayer(nn.Module):
engram_hashes: torch.Tensor | None = None,
engram_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# Layer 0's attention seam projects the 2-D embedding with the folded
# hc_attn_fn_broadcast instead of the 4-stream residual with hc_attn_fn.
# The fused kernel only takes the latter, so that seam keeps the
# separate attn_norm.
fuse_attn_norm = self.fuse_seam_norm and not (residual is None and x.dim() == 2)
# The reference collapses each sublayer's input with the *previous*
# sublayer's pre-mix: attention uses the pre-mix carried in (identity
# for the first layer), the FFN uses this layer's attention pre-mix.
@@ -303,6 +316,8 @@ class DeepseekV4DecoderLayer(nn.Module):
self.hc_post_alpha,
self.hc_sinkhorn_iters,
pre_mix=pre_mix,
norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None,
norm_eps=self.attn_norm.variance_epsilon,
)
else:
pre_args = (
@@ -328,7 +343,11 @@ class DeepseekV4DecoderLayer(nn.Module):
engram_mask,
)
residual, post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed(
residual, *pre_args, pre_mix=pre_mix
residual,
*pre_args,
pre_mix=pre_mix,
norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None,
norm_eps=self.attn_norm.variance_epsilon,
)
else:
(
@@ -344,8 +363,11 @@ class DeepseekV4DecoderLayer(nn.Module):
sublayer_out=x,
post_layer_mix=post_mix,
comb_res_mix=res_mix,
norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None,
norm_eps=self.attn_norm.variance_epsilon,
)
x = self.attn_norm(x)
if not fuse_attn_norm:
x = self.attn_norm(x)
if self.use_sequence_parallel:
x = sp_all_gather(x)[: positions.shape[0]]
@@ -368,8 +390,11 @@ class DeepseekV4DecoderLayer(nn.Module):
sublayer_out=x,
post_layer_mix=post_mix,
comb_res_mix=res_mix,
norm_weight=self.ffn_norm.weight if self.fuse_seam_norm else None,
norm_eps=self.ffn_norm.variance_epsilon,
)
x = self.ffn_norm(x)
if not self.fuse_seam_norm:
x = self.ffn_norm(x)
x = self.ffn(x, input_ids)
return x, residual, post_mix, res_mix, ffn_pre