From 0af34418e99b972029b53130c9a9665b4e692536 Mon Sep 17 00:00:00 2001 From: Muhammad Ahmed Date: Mon, 28 Sep 2026 22:19:10 -0400 Subject: [PATCH] [ROCm][DSv4.1][Perf] Run the delayed mHC seams through aiter's fused Triton kernel (#58655) Signed-off-by: Ahmed --- tests/kernels/mhc/test_mhc_kernels.py | 122 ++++++++++++++++++-- vllm/_aiter_ops.py | 51 ++++++++ vllm/model_executor/kernels/mhc/__init__.py | 1 + vllm/model_executor/kernels/mhc/aiter.py | 79 +++++++++++++ vllm/model_executor/layers/mhc.py | 52 ++++++++- vllm/models/deepseek_v41/amd/model.py | 33 +++++- 6 files changed, 323 insertions(+), 15 deletions(-) diff --git a/tests/kernels/mhc/test_mhc_kernels.py b/tests/kernels/mhc/test_mhc_kernels.py index 4735fb96d1..0554e9ca66 100644 --- a/tests/kernels/mhc/test_mhc_kernels.py +++ b/tests/kernels/mhc/test_mhc_kernels.py @@ -27,6 +27,7 @@ from vllm.model_executor.layers.mhc import ( HAS_AITER_MHC, HAS_AITER_MHC_FUSED, HAS_AITER_MHC_FUSED_NORM, + HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM, HAS_AITER_MHC_PRE_NORM, HAS_TILELANG_MHC, MHCFusedPostPreOp, @@ -1300,18 +1301,109 @@ def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried): torch.testing.assert_close(actual[2], expected[2], atol=0, rtol=0) +@pytest.mark.skipif( + not (current_platform.is_rocm() and HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM), + reason="AITER fused delayed-seam kernel required (gfx950)", +) +@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128, 1024]) +@pytest.mark.parametrize("seam", ["post", "no_post", "identity_pre"]) +def test_mhc_pre_delayed_rocm_aiter_fused_rms_norm(num_tokens, seam): + """One fused kernel must match the whole reference seam: the post block + (the previous seam's post/comb gates fold ``sublayer_out`` into the + streams), the delayed pre block (projection + Sinkhorn producing the next + seam's post/comb/pre gates, plus the collapse with the carried pre-mix), + and the RMSNorm folded onto the collapse. + + ``num_tokens`` spans the kernel's split-K config buckets. ``seam`` covers + the three call shapes the model makes: a regular seam (post folded in), an + Engram seam (post already applied), and the draft entry seam + (``pre_mix=None`` = identity pre-mix). + """ + set_random_seed(0) + hc_mult, hidden_size = 4, 5120 + residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs( + num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult + ) + rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6 + args = ( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + 1.0, + 20, + ) + + pre_mix = ( + None + if seam == "identity_pre" + else torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5 + ) + post_kwargs = {} + residual_ref = residual + if seam == "post": + sublayer_out = torch.randn( + num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE + ) + post_layer_mix = 2 * torch.rand( + num_tokens, hc_mult, 1, dtype=torch.float32, device=DEVICE + ) + comb_res_mix = torch.softmax( + torch.randn(num_tokens, hc_mult, hc_mult, device=DEVICE), dim=-1 + ) + post_kwargs = dict( + sublayer_out=sublayer_out, + post_layer_mix=post_layer_mix, + comb_res_mix=comb_res_mix, + ) + residual_ref = mhc_post_torch( + sublayer_out, residual, post_layer_mix, comb_res_mix + ) + + expected = mhc_pre_delayed_torch(residual_ref, *args[1:], pre_mix=pre_mix) + expected_layer_input = F.rms_norm( + expected[2], (hidden_size,), norm_weight, norm_eps + ) + + residual_out, *actual = object.__new__(MHCPreDelayedOp).forward_hip( + *args, + pre_mix=pre_mix, + norm_weight=norm_weight, + norm_eps=norm_eps, + **post_kwargs, + ) + + if seam == "post": + torch.testing.assert_close(residual_out, residual_ref) + else: + # No post was requested, so the residual comes straight back. + assert residual_out is residual + # Gates: post_mix, comb_mix, next_pre_mix (bf16 MFMA vs the fp32 reference). + for i in (0, 1, 3): + torch.testing.assert_close(actual[i], expected[i], atol=5e-4, rtol=1e-3) + # The collapse is bf16 and the kernel projects the once-rounded R', so + # near-zero values need an absolute tolerance (same bound as aiter's UT). + torch.testing.assert_close(actual[2], expected_layer_input, atol=2e-2, rtol=1e-2) + + @pytest.mark.skipif( not (current_platform.is_rocm() and HAS_AITER_MHC), reason="AITER mHC required", ) def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch): - """The broadcast seam and a fused norm must not take the AITER path. + """The broadcast seam must not take the AITER path; a fused norm must not + take the unfused AITER op. - Neither is expressible with AITER's pre kernels: the broadcast projects a - narrower ``x``, and ``mhc_pre_gemm_sqrsum`` folds no RMSNorm. What is - asserted here is the routing decision, which is what this gate owns; the - numerics of whichever fallback it lands on are covered by - ``test_deepseek_v41_mhc_pre_delayed``. + The broadcast projects a narrower ``x``, which no AITER pre kernel takes; + ``mhc_pre_gemm_sqrsum`` folds no RMSNorm, so a fused norm routes to the + fused seam kernel where it exists (gfx950) and away from AITER otherwise. + What is asserted here is the routing decision, which is what this gate + owns; the numerics of the paths are covered by + ``test_deepseek_v41_mhc_pre_delayed`` and + ``test_mhc_pre_delayed_rocm_aiter_fused_rms_norm``. """ set_random_seed(0) hc_mult, hidden_size = 4, 5120 @@ -1331,6 +1423,18 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch): monkeypatch.setattr(torch.ops.vllm, "mhc_pre_delayed_aiter", spy) + fused_op = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter + took_fused = False + + def fused_spy(*spy_args, **spy_kwargs): + nonlocal took_fused + took_fused = True + return fused_op(*spy_args, **spy_kwargs) + + monkeypatch.setattr( + torch.ops.vllm, "mhc_fused_post_pre_delayed_rms_norm_aiter", fused_spy + ) + x = residual[:, 0].contiguous() broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1) broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous() @@ -1342,11 +1446,15 @@ def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch): ) assert residual_out is broadcast_residual assert not took_aiter, "the broadcast seam must not reach AITER" + assert not took_fused, "the broadcast seam must not reach AITER" for i in range(4): torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3) + # A fused norm must not reach the unfused AITER op; where the fused seam + # kernel exists (gfx950) it routes there instead. op.forward_hip(*args, norm_weight=norm_weight, norm_eps=1e-6) - assert not took_aiter, "a fused norm must not reach AITER" + assert not took_aiter, "a fused norm must not reach the unfused AITER op" + assert took_fused == mhc_layers.HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM # Positive control: the same inputs without a norm do take the AITER path, # so the two declines above are the gate discriminating rather than the diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index ec249b1eca..906349af9a 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -4235,6 +4235,57 @@ class rocm_aiter_ops: next_pre_mix.view(*outer_shape, hc_mult), ) + @staticmethod + def mhc_fused_post_pre_delayed_rms_norm( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None, + sublayer_out: torch.Tensor | None, + post_layer_mix: torch.Tensor | None, + comb_res_mix: torch.Tensor | None, + norm_weight: torch.Tensor, + norm_eps: float, + residual_out: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """:meth:`mhc_pre_delayed` with the RMSNorm of the collapse folded in. + + Runs aiter's fused Triton kernel ``mhc_fused_post_pre_delayed_rmsnorm``; + the returned ``layer_input`` is already normalised with ``norm_weight`` / + ``norm_eps``. + """ + from aiter.ops.triton.fusions.mhc_fused_post_pre_delayed_rmsnorm import ( + mhc_fused_post_pre_delayed_rmsnorm, + ) + + _, post_mix, comb_mix, layer_input, next_pre_mix = ( + mhc_fused_post_pre_delayed_rmsnorm( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + sublayer_out, + post_layer_mix, + comb_res_mix, + norm_weight, + norm_eps, + residual_out=residual_out, + ) + ) + return post_mix, comb_mix, layer_input, next_pre_mix + @staticmethod def mhc_fused_post_pre_delayed_prefers_unfused(num_tokens: int) -> bool: """True when AITER's own heuristic favours separate post and pre. diff --git a/vllm/model_executor/kernels/mhc/__init__.py b/vllm/model_executor/kernels/mhc/__init__.py index f9df7f6f2c..6fc4919989 100644 --- a/vllm/model_executor/kernels/mhc/__init__.py +++ b/vllm/model_executor/kernels/mhc/__init__.py @@ -13,6 +13,7 @@ __all__ = [ "hc_head_fused_kernel_cuda", "mhc_pre_aiter", "mhc_pre_delayed_aiter", + "mhc_fused_post_pre_delayed_rms_norm_aiter", "mhc_post_aiter", "mhc_fused_post_pre_aiter", "hc_head_fused_aiter", diff --git a/vllm/model_executor/kernels/mhc/aiter.py b/vllm/model_executor/kernels/mhc/aiter.py index 06b4f14984..f102143ab0 100644 --- a/vllm/model_executor/kernels/mhc/aiter.py +++ b/vllm/model_executor/kernels/mhc/aiter.py @@ -207,6 +207,79 @@ def _mhc_pre_delayed_aiter_fake( ) +def mhc_fused_post_pre_delayed_rms_norm_aiter( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None, + sublayer_out: torch.Tensor | None, + post_layer_mix: torch.Tensor | None, + comb_res_mix: torch.Tensor | None, + norm_weight: torch.Tensor, + norm_eps: float, + residual_out: torch.Tensor | None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """:func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in: + ``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``.""" + from vllm._aiter_ops import rocm_aiter_ops + + return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + sublayer_out, + post_layer_mix, + comb_res_mix, + norm_weight, + norm_eps, + residual_out, + ) + + +def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None, + sublayer_out: torch.Tensor | None, + post_layer_mix: torch.Tensor | None, + comb_res_mix: torch.Tensor | None, + norm_weight: torch.Tensor, + norm_eps: float, + residual_out: torch.Tensor | None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + return _mhc_pre_delayed_aiter_fake( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + ) + + def mhc_post_aiter( x: torch.Tensor, residual: torch.Tensor, @@ -335,6 +408,12 @@ direct_register_custom_op( mutates_args=["residual_out"], fake_impl=_mhc_pre_delayed_aiter_fake, ) +direct_register_custom_op( + op_name="mhc_fused_post_pre_delayed_rms_norm_aiter", + op_func=mhc_fused_post_pre_delayed_rms_norm_aiter, + mutates_args=["residual_out"], + fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake, +) direct_register_custom_op( op_name="mhc_post_aiter", op_func=mhc_post_aiter, diff --git a/vllm/model_executor/layers/mhc.py b/vllm/model_executor/layers/mhc.py index 73993fbc98..19e8b427b4 100644 --- a/vllm/model_executor/layers/mhc.py +++ b/vllm/model_executor/layers/mhc.py @@ -59,9 +59,21 @@ def _aiter_mhc_op_accepts_norm(op_name: str) -> bool: return False +def _has_aiter_mhc_fused_post_pre_delayed_rms_norm() -> bool: + if not HAS_AITER_MHC: + return False + from vllm.platforms.rocm import on_gfx950 + + # The fused delayed seam kernel is only validated on gfx950. + return on_gfx950() + + HAS_AITER_MHC_FUSED = _has_aiter_mhc_fused() HAS_AITER_MHC_PRE_NORM = _aiter_mhc_op_accepts_norm("mhc_pre") HAS_AITER_MHC_FUSED_NORM = _aiter_mhc_op_accepts_norm("mhc_fused_post_pre") +HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM = ( + _has_aiter_mhc_fused_post_pre_delayed_rms_norm() +) def _aiter_mhc_supported( @@ -388,10 +400,42 @@ class MHCPreDelayedOp(CustomOp): post_layer_mix: torch.Tensor | None = None, comb_res_mix: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - # The AITER delayed path drives mhc_pre_gemm_sqrsum against `residual` - # and folds no RMSNorm, so it cannot serve the model-entry broadcast - # (which projects a narrower `x`) or a fused norm. Both are handled by - # TileLang, or by the reference when TileLang is unavailable. + # aiter's fused delayed seam: post-mix, gate projection, and the + # collapse with the carried pre-mix and its RMSNorm. It always applies + # the norm and projects `residual` itself. + if ( + HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM + and x is None + and norm_weight is not None + and _aiter_mhc_supported(residual, norm_weight, supports_norm=True) + ): + residual_out = ( + torch.empty_like(residual) if sublayer_out is not None else None + ) + rest = torch.ops.vllm.mhc_fused_post_pre_delayed_rms_norm_aiter( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + sublayer_out, + post_layer_mix, + comb_res_mix, + norm_weight, + norm_eps, + residual_out, + ) + return (residual if residual_out is None else residual_out), *rest + # The unfused AITER delayed path drives mhc_pre_gemm_sqrsum against + # `residual` and folds no RMSNorm, so it cannot serve the model-entry + # broadcast (which projects a narrower `x`) or a fused norm the branch + # above did not take. Both are handled by TileLang, or by the reference + # when TileLang is unavailable. if ( x is None and norm_weight is None diff --git a/vllm/models/deepseek_v41/amd/model.py b/vllm/models/deepseek_v41/amd/model.py index 0cad301cb6..61c8d5de30 100644 --- a/vllm/models/deepseek_v41/amd/model.py +++ b/vllm/models/deepseek_v41/amd/model.py @@ -22,7 +22,11 @@ from vllm.model_executor.layers.fused_moe import ( ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.mhc import MHCPostOp, MHCPreDelayedOp +from vllm.model_executor.layers.mhc import ( + HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM, + MHCPostOp, + MHCPreDelayedOp, +) from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -251,6 +255,10 @@ class DeepseekV4DecoderLayer(nn.Module): ) self.mhc_pre_delayed = MHCPreDelayedOp() self.mhc_post = MHCPostOp() + # Where aiter's fused seam kernel runs (gfx950), it folds the following + # attn_norm / ffn_norm into its collapse, so the separate norms are + # skipped for the seams it takes. + self.fuse_seam_norm = HAS_AITER_MHC_FUSED_POST_PRE_DELAYED_RMS_NORM @staticmethod def _hc_collapse(x: torch.Tensor, pre_mix: torch.Tensor) -> torch.Tensor: @@ -269,6 +277,11 @@ class DeepseekV4DecoderLayer(nn.Module): engram_hashes: torch.Tensor | None = None, engram_mask: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + # Layer 0's attention seam projects the 2-D embedding with the folded + # hc_attn_fn_broadcast instead of the 4-stream residual with hc_attn_fn. + # The fused kernel only takes the latter, so that seam keeps the + # separate attn_norm. + fuse_attn_norm = self.fuse_seam_norm and not (residual is None and x.dim() == 2) # The reference collapses each sublayer's input with the *previous* # sublayer's pre-mix: attention uses the pre-mix carried in (identity # for the first layer), the FFN uses this layer's attention pre-mix. @@ -303,6 +316,8 @@ class DeepseekV4DecoderLayer(nn.Module): self.hc_post_alpha, self.hc_sinkhorn_iters, pre_mix=pre_mix, + norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None, + norm_eps=self.attn_norm.variance_epsilon, ) else: pre_args = ( @@ -328,7 +343,11 @@ class DeepseekV4DecoderLayer(nn.Module): engram_mask, ) residual, post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed( - residual, *pre_args, pre_mix=pre_mix + residual, + *pre_args, + pre_mix=pre_mix, + norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None, + norm_eps=self.attn_norm.variance_epsilon, ) else: ( @@ -344,8 +363,11 @@ class DeepseekV4DecoderLayer(nn.Module): sublayer_out=x, post_layer_mix=post_mix, comb_res_mix=res_mix, + norm_weight=self.attn_norm.weight if self.fuse_seam_norm else None, + norm_eps=self.attn_norm.variance_epsilon, ) - x = self.attn_norm(x) + if not fuse_attn_norm: + x = self.attn_norm(x) if self.use_sequence_parallel: x = sp_all_gather(x)[: positions.shape[0]] @@ -368,8 +390,11 @@ class DeepseekV4DecoderLayer(nn.Module): sublayer_out=x, post_layer_mix=post_mix, comb_res_mix=res_mix, + norm_weight=self.ffn_norm.weight if self.fuse_seam_norm else None, + norm_eps=self.ffn_norm.variance_epsilon, ) - x = self.ffn_norm(x) + if not self.fuse_seam_norm: + x = self.ffn_norm(x) x = self.ffn(x, input_ids) return x, residual, post_mix, res_mix, ffn_pre